293 lines
11 KiB
Go
293 lines
11 KiB
Go
package runtime
|
|
|
|
// This file is the end-to-end test for the headless / stdio run modes (US-020,
|
|
// #39). It drives RunHeadless over the real faux provider seam (no loop-internal
|
|
// mocking) and asserts the two output contracts — PrintMode's final text and
|
|
// StreamJSONMode's line-delimited JSON events — plus the success/failure signal
|
|
// that the CLI maps to a process exit code.
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/smallnest/pigo/internal/agentcore"
|
|
"github.com/smallnest/pigo/internal/provider"
|
|
)
|
|
|
|
// TestRunHeadlessPrintMode runs a text→tool→text scenario through RunHeadless in
|
|
// PrintMode and asserts that only the final assistant text reaches the writer,
|
|
// terminated by a newline, and that the run reports success (nil error).
|
|
func TestRunHeadlessPrintMode(t *testing.T) {
|
|
p := &fauxProvider{
|
|
name: "faux",
|
|
models: []provider.Model{{Provider: "faux", ID: "faux"}},
|
|
turns: []fauxTurn{
|
|
toolCallTurn("call-1", "echo", `{"msg":"hi"}`), // turn 1: tool call
|
|
textTurn("final answer"), // turn 2: final text
|
|
},
|
|
}
|
|
cfg := newFauxRunCfg(p, echoTool("echo", agentcore.ToolExecutionParallel, false))
|
|
var out bytes.Buffer
|
|
agentCtx := &agentcore.AgentContext{Messages: agentcore.MessageList{agentcore.UserMessage{RoleField: agentcore.RoleUser, Content: agentcore.ContentList{agentcore.NewTextContent("start")}}}}
|
|
|
|
err := RunHeadless(context.Background(), agentCtx, HeadlessConfig{Run: cfg, Mode: PrintMode, Out: &out})
|
|
if err != nil {
|
|
t.Fatalf("RunHeadless print mode: unexpected error %v", err)
|
|
}
|
|
got := out.String()
|
|
if got != "final answer\n" {
|
|
t.Errorf("print mode output = %q, want %q", got, "final answer\n")
|
|
}
|
|
}
|
|
|
|
// TestRunHeadlessStreamJSON runs the same scenario in StreamJSONMode and asserts
|
|
// every line is a valid JSON object carrying a "type" discriminant, that the run
|
|
// is bracketed by agent_start/agent_end, and that a tool execution is reported —
|
|
// the machine-readable protocol a parent process consumes.
|
|
func TestRunHeadlessStreamJSON(t *testing.T) {
|
|
p := &fauxProvider{
|
|
name: "faux",
|
|
models: []provider.Model{{Provider: "faux", ID: "faux"}},
|
|
turns: []fauxTurn{
|
|
toolCallTurn("call-1", "echo", `{"msg":"hi"}`),
|
|
textTurn("done"),
|
|
},
|
|
}
|
|
cfg := newFauxRunCfg(p, echoTool("echo", agentcore.ToolExecutionParallel, false))
|
|
var out bytes.Buffer
|
|
agentCtx := &agentcore.AgentContext{Messages: agentcore.MessageList{agentcore.UserMessage{RoleField: agentcore.RoleUser, Content: agentcore.ContentList{agentcore.NewTextContent("start")}}}}
|
|
|
|
if err := RunHeadless(context.Background(), agentCtx, HeadlessConfig{Run: cfg, Mode: StreamJSONMode, Out: &out}); err != nil {
|
|
t.Fatalf("RunHeadless stream-json: unexpected error %v", err)
|
|
}
|
|
|
|
var types []string
|
|
sc := bufio.NewScanner(&out)
|
|
for sc.Scan() {
|
|
line := sc.Bytes()
|
|
if len(bytes.TrimSpace(line)) == 0 {
|
|
continue
|
|
}
|
|
var env map[string]any
|
|
if err := json.Unmarshal(line, &env); err != nil {
|
|
t.Fatalf("stream-json line is not valid JSON: %q (%v)", line, err)
|
|
}
|
|
typ, ok := env["type"].(string)
|
|
if !ok || typ == "" {
|
|
t.Errorf("stream-json line missing type discriminant: %q", line)
|
|
}
|
|
types = append(types, typ)
|
|
}
|
|
if len(types) == 0 {
|
|
t.Fatal("stream-json produced no event lines")
|
|
}
|
|
if types[0] != agentcore.EventAgentStart || types[len(types)-1] != agentcore.EventAgentEnd {
|
|
t.Errorf("stream must be bracketed by agent_start/agent_end, got %v", types)
|
|
}
|
|
if !contains(types, agentcore.EventToolExecutionEnd) {
|
|
t.Errorf("expected a tool_execution_end event, got %v", types)
|
|
}
|
|
}
|
|
|
|
// TestRunHeadlessStreamJSONSessionID verifies that when RunConfig.SessionID is
|
|
// set, the first stream-json event (agent_start) carries it under "sessionId",
|
|
// so a consumer can associate the run's output with a session and resume it
|
|
// later (mirrors pi/Claude Code). When SessionID is empty the key is omitted.
|
|
func TestRunHeadlessStreamJSONSessionID(t *testing.T) {
|
|
run := func(sessionID string) map[string]any {
|
|
p := &fauxProvider{
|
|
name: "faux",
|
|
models: []provider.Model{{Provider: "faux", ID: "faux"}},
|
|
turns: []fauxTurn{textTurn("done")},
|
|
}
|
|
cfg := newFauxRunCfg(p)
|
|
cfg.SessionID = sessionID
|
|
var out bytes.Buffer
|
|
agentCtx := &agentcore.AgentContext{Messages: agentcore.MessageList{agentcore.UserMessage{RoleField: agentcore.RoleUser, Content: agentcore.ContentList{agentcore.NewTextContent("start")}}}}
|
|
if err := RunHeadless(context.Background(), agentCtx, HeadlessConfig{Run: cfg, Mode: StreamJSONMode, Out: &out}); err != nil {
|
|
t.Fatalf("RunHeadless stream-json: unexpected error %v", err)
|
|
}
|
|
sc := bufio.NewScanner(&out)
|
|
for sc.Scan() {
|
|
line := sc.Bytes()
|
|
if len(bytes.TrimSpace(line)) == 0 {
|
|
continue
|
|
}
|
|
var env map[string]any
|
|
if err := json.Unmarshal(line, &env); err != nil {
|
|
t.Fatalf("stream-json line is not valid JSON: %q (%v)", line, err)
|
|
}
|
|
if env["type"] == agentcore.EventAgentStart {
|
|
return env
|
|
}
|
|
}
|
|
t.Fatal("no agent_start event found")
|
|
return nil
|
|
}
|
|
|
|
first := run("sess-123")
|
|
if got, ok := first["sessionId"].(string); !ok || got != "sess-123" {
|
|
t.Errorf("agent_start sessionId = %v, want %q", first["sessionId"], "sess-123")
|
|
}
|
|
|
|
none := run("")
|
|
if _, present := none["sessionId"]; present {
|
|
t.Errorf("agent_start must omit sessionId when SessionID is empty, got %v", none["sessionId"])
|
|
}
|
|
}
|
|
|
|
// TestRunHeadlessReportsFailure verifies that a run whose final assistant message
|
|
// carries stopReason=error surfaces as an ErrRunFailed, so the CLI maps it to a
|
|
// non-zero exit code.
|
|
func TestRunHeadlessReportsFailure(t *testing.T) {
|
|
errPartial := agentcore.AssistantMessage{RoleField: agentcore.RoleAssistant}
|
|
errFinal := errPartial
|
|
errFinal.StopReason = agentcore.StopReasonError
|
|
errFinal.ErrorMessage = "boom"
|
|
p := &fauxProvider{
|
|
name: "faux",
|
|
turns: []fauxTurn{
|
|
{
|
|
provider.StreamStartEvent{Partial: errPartial},
|
|
provider.StreamDoneEvent{Message: errFinal},
|
|
},
|
|
},
|
|
}
|
|
cfg := newFauxRunCfg(p)
|
|
var out bytes.Buffer
|
|
agentCtx := &agentcore.AgentContext{Messages: agentcore.MessageList{agentcore.UserMessage{RoleField: agentcore.RoleUser}}}
|
|
|
|
err := RunHeadless(context.Background(), agentCtx, HeadlessConfig{Run: cfg, Mode: PrintMode, Out: &out})
|
|
if err == nil {
|
|
t.Fatal("run ending in stopReason=error must return a non-nil error")
|
|
}
|
|
var failed *ErrRunFailed
|
|
if !as(err, &failed) {
|
|
t.Fatalf("error = %T (%v), want *ErrRunFailed", err, err)
|
|
}
|
|
if !strings.Contains(failed.Error(), "boom") {
|
|
t.Errorf("error message = %q, want it to mention the failure reason", failed.Error())
|
|
}
|
|
}
|
|
|
|
// TestRunHeadlessNilWriter guards the misconfiguration path.
|
|
func TestRunHeadlessNilWriter(t *testing.T) {
|
|
p := &fauxProvider{turns: []fauxTurn{textTurn("x")}}
|
|
cfg := newFauxRunCfg(p)
|
|
agentCtx := &agentcore.AgentContext{Messages: agentcore.MessageList{agentcore.UserMessage{RoleField: agentcore.RoleUser}}}
|
|
if err := RunHeadless(context.Background(), agentCtx, HeadlessConfig{Run: cfg, Out: nil}); err == nil {
|
|
t.Fatal("nil output writer must be rejected")
|
|
}
|
|
}
|
|
|
|
// emitTool returns a tool that surfaces ev on the run stream via the run-level
|
|
// progress emitter the loop injects into ctx (WithProgressEmitter), then returns
|
|
// a trivial text result. This mirrors how a dispatched sub-agent surfaces a
|
|
// SubAgentProgressEvent up the parent stream.
|
|
func emitTool(name string, ev agentcore.AgentEvent) execTool {
|
|
return execTool{
|
|
name: name,
|
|
mode: agentcore.ToolExecutionParallel,
|
|
run: func(ctx context.Context, id string, args json.RawMessage, onUpdate agentcore.ToolUpdateFunc) (agentcore.AgentToolResult, error) {
|
|
if emit := agentcore.ProgressEmitterFromContext(ctx); emit != nil {
|
|
_ = emit(ctx, ev)
|
|
}
|
|
return agentcore.AgentToolResult{Content: agentcore.ContentList{agentcore.NewTextContent(name)}}, nil
|
|
},
|
|
}
|
|
}
|
|
|
|
// TestRunHeadlessSubAgentProgressToStderr verifies the D-9 contract: a
|
|
// SubAgentProgressEvent emitted during the run is rendered as a human-readable
|
|
// line to the progress writer (stderr) and is NEVER serialised onto stdout —
|
|
// neither the final result text nor the stream-json envelope stream may contain
|
|
// it. The event is injected via a faux tool whose execution fires it on the
|
|
// run's event stream (the same seam the loop uses).
|
|
func TestRunHeadlessSubAgentProgressToStderr(t *testing.T) {
|
|
const desc = "investigate the parser"
|
|
const activity = "Editing"
|
|
|
|
run := func(mode HeadlessMode) (stdout, stderr string) {
|
|
p := &fauxProvider{
|
|
name: "faux",
|
|
models: []provider.Model{{Provider: "faux", ID: "faux"}},
|
|
turns: []fauxTurn{
|
|
toolCallTurn("call-1", "task", `{"description":"investigate the parser"}`),
|
|
textTurn("done"),
|
|
},
|
|
}
|
|
// The tool emits a SubAgentProgressEvent onto the run stream, mimicking a
|
|
// dispatched sub-agent surfacing progress up the parent stream.
|
|
tool := emitTool("task", agentcore.SubAgentProgressEvent{
|
|
ToolCallID: "call-1",
|
|
Description: desc,
|
|
Activity: activity,
|
|
})
|
|
cfg := newFauxRunCfg(p, tool)
|
|
var out, prog bytes.Buffer
|
|
agentCtx := &agentcore.AgentContext{Messages: agentcore.MessageList{agentcore.UserMessage{RoleField: agentcore.RoleUser, Content: agentcore.ContentList{agentcore.NewTextContent("start")}}}}
|
|
if err := RunHeadless(context.Background(), agentCtx, HeadlessConfig{Run: cfg, Mode: mode, Out: &out, Progress: &prog}); err != nil {
|
|
t.Fatalf("RunHeadless: unexpected error %v", err)
|
|
}
|
|
return out.String(), prog.String()
|
|
}
|
|
|
|
for _, mode := range []struct {
|
|
name string
|
|
mode HeadlessMode
|
|
}{{"print", PrintMode}, {"stream-json", StreamJSONMode}} {
|
|
t.Run(mode.name, func(t *testing.T) {
|
|
stdout, stderr := run(mode.mode)
|
|
// (a) stderr carries the progress line with description + activity.
|
|
if !strings.Contains(stderr, desc) || !strings.Contains(stderr, activity) {
|
|
t.Errorf("stderr = %q, want it to contain description %q and activity %q", stderr, desc, activity)
|
|
}
|
|
// (b) stdout must not contain the progress event in any form.
|
|
if strings.Contains(stdout, "subagent_progress") {
|
|
t.Errorf("stdout must not contain the subagent_progress envelope, got %q", stdout)
|
|
}
|
|
if strings.Contains(stdout, desc) {
|
|
t.Errorf("stdout must not leak the progress description, got %q", stdout)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestWriteProgressLineEmptyDescription verifies the line degrades gracefully to
|
|
// the activity alone when the task supplied no description.
|
|
func TestWriteProgressLineEmptyDescription(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
writeProgressLine(&buf, agentcore.SubAgentProgressEvent{Activity: "Thinking"})
|
|
got := buf.String()
|
|
if !strings.Contains(got, "Thinking") {
|
|
t.Errorf("line = %q, want it to contain the activity", got)
|
|
}
|
|
if strings.Contains(got, "·") {
|
|
t.Errorf("line = %q, want no separator when description is empty", got)
|
|
}
|
|
}
|
|
|
|
// contains reports whether s contains v.
|
|
func contains(s []string, v string) bool {
|
|
for _, x := range s {
|
|
if x == v {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// as is a tiny errors.As shim kept local to avoid an extra import in a test that
|
|
// only ever unwraps one level.
|
|
func as(err error, target **ErrRunFailed) bool {
|
|
if e, ok := err.(*ErrRunFailed); ok {
|
|
*target = e
|
|
return true
|
|
}
|
|
return false
|
|
}
|