diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef582f7..582f106 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,6 +36,15 @@ go build ./examples/... Never commit `.env.live`, tokens, credentials, generated logs, or test output. Live tests must register cleanup immediately after creating a resource. +Managed execution tests log the user prompt and resource IDs. If a turn fails, +they also log observed public events before session cleanup: event IDs/types, +Read/Bash inputs, text results and stop reasons. The diagnostic tail is limited +to 128 events / 256 KiB with 4 KiB per text field; omissions are reported. +Reasoning and binary blocks are excluded, and the configured PAT and URLs are +redacted. Retain the `go test -json` output (`build/test-results/e2e.json` for +`make test-e2e`) to investigate after the test deletes the session. These logs +contain test-account content and must not be committed or published. + `make test-live-all` includes Forward hourly Usage, Credential merge patches and secret redaction, and Managed Session cancellation before and after sending a turn. Managed deployment tests cover scoped Run listing/retrieval. Active diff --git a/managed/execution_diagnostics_test.go b/managed/execution_diagnostics_test.go new file mode 100644 index 0000000..cef8bd9 --- /dev/null +++ b/managed/execution_diagnostics_test.go @@ -0,0 +1,142 @@ +package managed_test + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + "testing" +) + +const managedDiagnosticMaxEvents = 128 +const managedDiagnosticMaxBytes = 256 * 1024 + +var managedDiagnosticURL = regexp.MustCompile(`https?://[^\s"<>]+`) + +// Keep public turn evidence before cleanup deletes the session. Limit the tail +// so noisy streams cannot exhaust test memory or hide the final failure in logs. +type managedTurnDiagnostics struct { + secret string + events []string + bytes int + omitted int +} + +func (d *managedTurnDiagnostics) text(s string) string { + if d.secret != "" { + s = strings.ReplaceAll(s, d.secret, "[REDACTED]") + } + s = managedDiagnosticURL.ReplaceAllString(s, "[URL OMITTED]") + if len(s) > 4096 { + s = s[:4096] + "[TRUNCATED]" + } + return s +} + +func (d *managedTurnDiagnostics) observe(raw string) { + var event map[string]any + safe := map[string]any{} + if json.Unmarshal([]byte(raw), &event) != nil || event == nil { + safe["diagnostic"] = "invalid event JSON; payload omitted" + } else { + // Explicit fields exclude reasoning, binary content, auth headers and + // unknown payload extensions. Text is from this test account only. + for _, key := range []string{"id", "type", "processed_at", "name", "tool_use_id", "mcp_tool_use_id"} { + if value, ok := event[key].(string); ok { + safe[key] = d.text(value) + } + } + if value, ok := event["is_error"].(bool); ok { + safe["is_error"] = value + } + if reason, ok := event["stop_reason"].(map[string]any); ok { + if value, ok := reason["type"].(string); ok { + safe["stop_reason"] = d.text(value) + } + } else if value, ok := event["stop_reason"].(string); ok { + safe["stop_reason"] = d.text(value) + } + if input, ok := event["input"].(map[string]any); ok { + fields := map[string]string{} + for _, key := range []string{"file_path", "path", "command"} { + if value, ok := input[key].(string); ok { + fields[key] = d.text(value) + } + } + safe["input"] = fields + } + if content, ok := event["content"].([]any); ok { + var texts []string + for _, block := range content { + if block, ok := block.(map[string]any); ok && block["type"] == "text" { + if value, ok := block["text"].(string); ok { + texts = append(texts, value) + } + } + } + if len(texts) > 0 { + safe["text"] = d.text(strings.Join(texts, "\n")) + } + } + } + encoded, _ := json.Marshal(safe) + d.events = append(d.events, string(encoded)) + d.bytes += len(encoded) + for len(d.events) > managedDiagnosticMaxEvents || d.bytes > managedDiagnosticMaxBytes { + d.bytes -= len(d.events[0]) + d.events[0] = "" + d.events = d.events[1:] + d.omitted++ + } +} + +func TestManagedTurnDiagnostics(t *testing.T) { + d := managedTurnDiagnostics{secret: "test-pat-secret"} + for _, raw := range []string{ + `{"id":"tool-1","type":"agent.tool_use","name":"Read","input":{"file_path":"/data/.agent/awareness/sdk-e2e/proof.md","authorization":"secret-header"}}`, + `{"id":"result-1","type":"agent.tool_result","tool_use_id":"tool-1","is_error":false,"content":[{"type":"text","text":"SDK_E2E_MEMORY_TOKEN=random-marker"}]}`, + `{"id":"message-1","type":"agent.message","authorization":"secret-header","content":[{"type":"thinking","thinking":"private-reasoning"},{"type":"image","source":{"data":"private-image"}},{"type":"text","text":"Missing skill. test-pat-secret https://storage.example/file?signature=private-signature"}]}`, + `{"id":"idle-1","type":"session.status_idle","stop_reason":{"type":"end_turn"}}`, + `invalid payload test-pat-secret`, + } { + d.observe(raw) + } + output := strings.Join(d.events, "\n") + for _, want := range []string{"tool-1", "result-1", "random-marker", "Missing skill.", "end_turn", "/data/.agent/awareness/sdk-e2e/proof.md", "invalid event JSON", "[REDACTED]", "[URL OMITTED]"} { + if !strings.Contains(output, want) { + t.Errorf("diagnostics lost %q", want) + } + } + for _, secret := range []string{"test-pat-secret", "private-reasoning", "private-image", "private-signature", "secret-header"} { + if strings.Contains(output, secret) { + t.Errorf("diagnostics exposed %q", secret) + } + } + for _, event := range d.events { + if !json.Valid([]byte(event)) { + t.Fatal("diagnostics must retain valid JSON") + } + } +} + +func TestManagedTurnDiagnosticsBoundedTail(t *testing.T) { + for _, size := range []int{1, 10000} { + t.Run(fmt.Sprint(size), func(t *testing.T) { + d := managedTurnDiagnostics{} + for i := 0; i < 200; i++ { + raw, _ := json.Marshal(map[string]any{"id": fmt.Sprint(i), "type": "agent.message", "content": []any{map[string]any{"type": "text", "text": strings.Repeat("x", size)}}}) + d.observe(string(raw)) + } + d.observe(`{"id":"last","type":"session.status_idle","stop_reason":"end_turn"}`) + if d.omitted == 0 || d.bytes > managedDiagnosticMaxBytes || len(d.events) > managedDiagnosticMaxEvents { + t.Fatal("diagnostic tail exceeded its budget or did not report omitted events") + } + if !strings.Contains(d.events[len(d.events)-1], `"id":"last"`) { + t.Fatal("diagnostic tail lost the terminal event") + } + if size > 4096 && !strings.Contains(strings.Join(d.events, "\n"), "[TRUNCATED]") { + t.Fatal("oversized text was not marked as truncated") + } + }) + } +} diff --git a/managed/execution_live_test.go b/managed/execution_live_test.go index e1ca09a..ad3d2e3 100644 --- a/managed/execution_live_test.go +++ b/managed/execution_live_test.go @@ -21,8 +21,18 @@ func (s *managedScenarioSuite) waitTurn(t *testing.T, id, after string, expected ctx, cancel := context.WithTimeout(context.Background(), testsupport.ExecutionTimeout(t)) defer cancel() result := testsupport.TurnResult{LastID: after} + diagnostics := managedTurnDiagnostics{secret: os.Getenv("QODER_MANAGED_PAT")} + defer func() { + if t.Failed() { + t.Logf("failed turn session=%s after=%s last_event_id=%s complete=%t tool_used=%t expected_markers=%q omitted_events=%d", id, after, result.LastID, result.Complete, result.ToolUsed, expected, diagnostics.omitted) + for _, event := range diagnostics.events { + t.Logf("observed event: %s", event) + } + } + }() observe := func(raw string) bool { t.Helper() + diagnostics.observe(raw) if err := result.Observe(raw); err != nil { t.Fatalf("session=%s: %v", id, err) } @@ -80,6 +90,8 @@ func (s *managedScenarioSuite) sendTurn(t *testing.T, id, prompt string) string if res == nil || len(res.Data) != 1 || res.Data[0].ID == "" { t.Fatal("send did not return one user event ID") } + diagnostics := managedTurnDiagnostics{secret: os.Getenv("QODER_MANAGED_PAT")} + t.Logf("sent session=%s user_event_id=%s prompt=%q", id, res.Data[0].ID, diagnostics.text(prompt)) return res.Data[0].ID } func (s *managedScenarioSuite) cleanupSession(t *testing.T, id string) { @@ -126,24 +138,28 @@ func TestManagedExecutionE2ELive(t *testing.T) { _, err := s.client.Files.Delete(ctx, file.ID, managed.FileDeleteParams{}) return err }) - skillName := managedUnique("proof") + skillName := managedUnique("skill") + skillPath := "/data/.agent/skills/" + skillName + "/SKILL.md" + const memoryPath = "sdk-e2e/proof.md" + const memoryMountPath = "/data/.agent/awareness/" + memoryPath skill := liveResult(s.client.Skills.New(ctx, managed.SkillNewParams{Files: []io.Reader{convention.UploadFile{Name: skillName + "/SKILL.md", Reader: strings.NewReader(fmt.Sprintf("---\nname: %s\ndescription: Provides SDK_E2E_SKILL_TOKEN only when the current user message explicitly requests that token.\n---\nSDK_E2E_SKILL_TOKEN=%s\nInclude this exact token only when requested, using the response format specified by the current user message. Also include any other values the user requests.\n", skillName, skillToken))}}})).require(t) s.cleanup(t, "skill "+skill.ID, func(ctx context.Context) error { _, err := s.client.Skills.Delete(ctx, skill.ID, managed.SkillDeleteParams{}) return err }) store := s.createMemoryStore(t) - liveResult(s.client.MemoryStores.Memories.New(ctx, store.ID, managed.MemoryStoreMemoryNewParams{Path: "sdk-e2e/proof.md", Content: managed.String("SDK_E2E_MEMORY_TOKEN=" + memoryToken)})).require(t) - agentParams := liveJSON[managed.AgentNewParams](t, map[string]any{"name": managedUnique("proof"), "model": map[string]any{"id": os.Getenv("QODER_MANAGED_MODEL")}, "system": "Each user message is a separate SDK verification step. Follow the current user message and its response format; do not carry over instructions or answers from previous steps. Use tools only to access resources explicitly requested in the current message. Read skills only when that message explicitly requests a skill token. When asked to echo text already supplied in the message, reply directly without using tools. Return every requested value exactly. Do not guess missing values.", "tools": []any{map[string]any{"type": "agent_toolset_20260401"}}, "skills": []any{map[string]any{"type": "custom", "skill_id": skill.ID, "version": skill.LatestVersionID}}}) + liveResult(s.client.MemoryStores.Memories.New(ctx, store.ID, managed.MemoryStoreMemoryNewParams{Path: memoryPath, Content: managed.String("SDK_E2E_MEMORY_TOKEN=" + memoryToken)})).require(t) + agentParams := liveJSON[managed.AgentNewParams](t, map[string]any{"name": managedUnique("agent"), "model": map[string]any{"id": os.Getenv("QODER_MANAGED_MODEL")}, "system": "Each user message is a separate SDK verification step. Follow the current user message and its response format; do not carry over instructions or answers from previous steps. Use tools only to access resources explicitly requested in the current message. Read skills only when that message explicitly requests a skill token. When asked to echo text already supplied in the message, reply directly without using tools. Return every requested value exactly. Do not guess missing values.", "tools": []any{map[string]any{"type": "agent_toolset_20260401"}}, "skills": []any{map[string]any{"type": "custom", "skill_id": skill.ID, "version": skill.LatestVersionID}}}) agent := liveResult(s.client.Agents.New(ctx, agentParams)).require(t) s.cleanup(t, "agent "+agent.ID, func(ctx context.Context) error { _, err := s.client.Agents.Archive(ctx, agent.ID, managed.AgentArchiveParams{}) return err }) - sessionParams := liveJSON[managed.SessionNewParams](t, map[string]any{"agent": agent.ID, "environment_id": env.ID, "environment_variables": map[string]string{"SDK_E2E_VALUE": envToken}, "resources": []any{map[string]any{"type": "file", "file_id": file.ID, "mount_path": "/data/workspace/sdk-e2e.txt"}, map[string]any{"type": "memory_store", "memory_store_id": store.ID, "access": "read_only", "instructions": "Read sdk-e2e/proof.md when asked for SDK_E2E_MEMORY_TOKEN."}}}) + sessionParams := liveJSON[managed.SessionNewParams](t, map[string]any{"agent": agent.ID, "environment_id": env.ID, "environment_variables": map[string]string{"SDK_E2E_VALUE": envToken}, "resources": []any{map[string]any{"type": "file", "file_id": file.ID, "mount_path": "/data/workspace/sdk-e2e.txt"}, map[string]any{"type": "memory_store", "memory_store_id": store.ID, "access": "read_only", "instructions": "Read " + memoryMountPath + " when asked for SDK_E2E_MEMORY_TOKEN."}}}) session := liveResult(s.client.Sessions.New(ctx, sessionParams)).require(t) s.cleanupSession(t, session.ID) t.Logf("model=%s agent=%s file=%s skill=%s memory_store=%s", os.Getenv("QODER_MANAGED_MODEL"), agent.ID, file.ID, skill.ID, store.ID) + t.Logf("skill_name=%s skill_path=%s memory_path=%s", skillName, skillPath, memoryMountPath) echo := testsupport.Marker(t) for _, scenario := range []struct { name, prompt string @@ -152,7 +168,7 @@ func TestManagedExecutionE2ELive(t *testing.T) { }{ {"completion_and_sse", "Reply with exactly " + echo, []string{echo}, false, true}, {"file_and_environment_config", "Use tools to read /data/workspace/sdk-e2e.txt and the SDK_E2E_VALUE environment variable. Reply with both exact values on two lines: FILE= and ENV=.", []string{fileToken, envToken}, true, false}, - {"skill_and_memory", "Use skill " + skillName + " to obtain SDK_E2E_SKILL_TOKEN. Read sdk-e2e/proof.md from the mounted memory store to obtain SDK_E2E_MEMORY_TOKEN. Reply with both exact tokens on two lines: SKILL= and MEMORY=.", []string{skillToken, memoryToken}, true, false}, + {"skill_and_memory", "Use the Read tool to read " + skillPath + " for skill " + skillName + " and obtain SDK_E2E_SKILL_TOKEN. Also read the mounted memory file " + memoryMountPath + " to obtain SDK_E2E_MEMORY_TOKEN. Reply with both exact tokens on two lines: SKILL= and MEMORY=.", []string{skillToken, memoryToken}, true, false}, } { if !t.Run(scenario.name, func(t *testing.T) { after := s.sendTurn(t, session.ID, scenario.prompt)