feat(worker): repair red CI in rounds inside the attempt (CI repair, U5)

cmd/jig/ci_repair_test.go

@@ -0,0 +1,191 @@
+// ci_repair_test.go — one CI repair round through the real `jig worker`
+// wiring (plan U5's required end-to-end test). Everything is production
+// except the runtime (scripted) and the GitHub gateway (a fake): a live
+// `jig serve`, workerAttemptRunner with its per-attempt trace stream and
+// scratch, the publishing runner, and a real git remote. It proves what the
+// U3 review found broken: the continuation's scratch (handoff notes
+// included) and trace stream must outlive Execute until the round is done.
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/StructuPath/jig/internal/controlplane"
+	"github.com/StructuPath/jig/internal/protocol"
+	"github.com/StructuPath/jig/internal/worker"
+)
+
+const ciRepairDefinition = `name: ci-repair-e2e
+roster:
+  builder:
+    model: test-model
+    system_prompt: Build.
+    user_prompt: "Build."
+    env: [PATH]
+    writes: ["jig-note.txt", "jig-fix.txt"]
+phases:
+  - {name: build, kind: agent, owner: builder}
+  - name: verify
+    kind: code
+    command: "test -s jig-note.txt"
+acceptance: [all_phases_passed]
+publish:
+  ci:
+    wait: true
+    on_fail: {run: build, budget: 1}
+`
+
+// e2eGateway is the fake GitHub: CI is red on the first head it judges and
+// green on the next, and every poll runs the test's probe first.
+type e2eGateway struct {
+	mutex  sync.Mutex
+	first  string
+	probe  func(sha string, red bool)
+	polled []string
+}
+
+func (g *e2eGateway) FindOrCreatePullRequest(context.Context, worker.PullRequestRequest) (worker.PullRequest, error) {
+	return worker.PullRequest{Number: 1, URL: "https://github.com/example/repo/pull/1", State: "open"}, nil
+}
+
+func (g *e2eGateway) CommitChecks(_ context.Context, _ string, sha string) ([]worker.CICheck, error) {
+	g.mutex.Lock()
+	defer g.mutex.Unlock()
+	g.polled = append(g.polled, sha)
+	if g.first == "" {
+		g.first = sha
+	}
+	red := sha == g.first
+	g.probe(sha, red)
+	if red {
+		return []worker.CICheck{{Name: "lint", Verdict: worker.CIFail, Conclusion: "failure"}}, nil
+	}
+	return []worker.CICheck{{Name: "lint", Verdict: worker.CIPass, Conclusion: "success"}}, nil
+}
+
+func (g *e2eGateway) FailedCheckLogs(_ context.Context, _ string, checks []worker.CICheck) []worker.CICheck {
+	return checks
+}
+
+func TestACIRepairRoundRunsThroughTheRealWorkerWiring(t *testing.T) {
+	repo := initRepo(t)
+	serverData, workerData := t.TempDir(), t.TempDir()
+	scriptRuntime(t,
+		map[string]any{"files": map[string]any{"jig-note.txt": "note\n"},
+			"text": envelopeJSON(t, map[string]any{"status": "success", "summary": "wrote the note"})},
+		map[string]any{"files": map[string]any{"jig-fix.txt": "fixed\n"},
+			"text": envelopeJSON(t, map[string]any{"status": "success", "summary": "fixed lint"})},
+	)
+	base, stopServe := startServe(t, serverData, "--no-github-poll")
+	defer stopServe()
+
+	var definition protocol.Definition
+	if err := apiCall(context.Background(), base, "POST", "/api/definitions",
+		controlplane.DefinitionInput{Source: ciRepairDefinition}, &definition); err != nil {
+		t.Fatal(err)
+	}
+	var view controlplane.RunView
+	if err := apiCall(context.Background(), base, "POST", "/api/runs", controlplane.RunInvocation{
+		DefinitionID: definition.ID, Instructions: "write the note",
+		Targets: []controlplane.InvocationTarget{{Repository: repo}},
+	}, &view); err != nil {
+		t.Fatal(err)
+	}
+
+	// The handoff note is planted while CI is red — after Execute returned,
+	// before the round — and must still be there when CI judges the fix.
+	var probeErrors []string
+	note := ""
+	gateway := &e2eGateway{probe: func(sha string, red bool) {
+		handoffs, _ := filepath.Glob(filepath.Join(workerData, "scratch", "*", "handoff"))
+		if len(handoffs) != 1 {
+			probeErrors = append(probeErrors, fmt.Sprintf(
+				"CI poll on %.8s (red=%v) found %d scratch handoff dirs; the continuation's scratch is gone",
+				sha, red, len(handoffs)))
+			return
+		}
+		if red && note == "" {
+			note = filepath.Join(handoffs[0], "plan.md")
+			if err := os.WriteFile(note, []byte("the plan\n"), 0o600); err != nil {
+				probeErrors = append(probeErrors, err.Error())
+			}
+			return
+		}
+		if !red {
+			if _, err := os.Stat(note); err != nil {
+				probeErrors = append(probeErrors, "the handoff note did not survive the round: "+err.Error())
+			}
+		}
+	}}
+
+	ctx := context.Background()
+	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
+	selected, err := selectRuntime(ctx, "claude-code", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var host *worker.Worker
+	publisher := worker.NewPublishingRunner(
+		workerAttemptRunner(ctx, workerData, selected, logger, func() *worker.Worker { return host }),
+		gateway, worker.PublishOptions{
+			CommitAuthorName: "jig-test", CommitAuthorEmail: "jig@test",
+			CIPollInterval: 5 * time.Millisecond, CIRegistrationGrace: 20 * time.Millisecond,
+		})
+	host, err = worker.New(worker.Config{
+		ServerURL: base, DataDir: workerData, Capacity: 1,
+		Probes: []worker.RuntimeProbe{selected.Runtime}, Runner: publisher, Logger: logger,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	publisher.Bind(host)
+	if _, err := host.RegisterOnce(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	attempt, err := host.ClaimOnce(ctx)
+	if err != nil || attempt == nil {
+		t.Fatalf("claim: attempt=%v err=%v", attempt, err)
+	}
+	for _, problem := range probeErrors {
+		t.Error(problem)
+	}
+	if attempt.State != protocol.AttemptAccepted {
+		t.Fatalf("attempt = %s (%s)\n%s, want accepted after one repair round", attempt.State, attempt.Error, attempt.Result)
+	}
+	if len(gateway.polled) < 2 || note == "" {
+		t.Fatalf("CI polls = %d, note planted = %v: the round never ran", len(gateway.polled), note != "")
+	}
+
+	// The round's trace reached the control plane: the stream was still open.
+	var page struct {
+		Events []protocol.Event `json:"events"`
+	}
+	if err := apiCall(ctx, base, "GET", "/api/attempts/"+attempt.ID+"/events?limit=500", nil, &page); err != nil {
+		t.Fatal(err)
+	}
+	repairStarted, acceptances := false, 0
+	for _, event := range page.Events {
+		repairStarted = repairStarted || event.Name == "ci_repair_start"
+		if event.Name == "acceptance" {
+			acceptances++
+		}
+	}
+	if !repairStarted || acceptances != 2 {
+		t.Fatalf("control-plane trace has ci_repair_start=%v and %d acceptance event(s), want the round's events too",
+			repairStarted, acceptances)
+	}
+
+	// And once the attempt is over, its scratch is gone (KTD11).
+	if entries, _ := os.ReadDir(filepath.Join(workerData, "scratch")); len(entries) != 0 {
+		t.Fatalf("attempt scratch survived the attempt: %v", entries)
+	}
+}

docs/plans/2026-09-27-001-feat-ci-repair-plan.md

@@ -128,10 +128,19 @@ Round state lives in the worker's `PublishingRunner.Run` loop. The engine owns c
 
 ### U5. Worker: the round loop
 
-- **Files:** `internal/worker/publish.go`, new `internal/worker/publish_repair.go`, `internal/worker/publish_repair_test.go`
-- **Approach:** in `PublishingRunner.Run`, when `publish` returns `ci_failed` and the continuation is present with budget left: the head-moved check (KTD7), authorize the round, `RepairCI` under the attempt's own cancellable context, then commit and non-force push in the critical section, record the round, and call `awaitCI` again as a fresh `ci` step. `PublishSummary` gains `CIRepairs []{Round, HeadBefore, HeadAfter, FailedChecks, Outcome}`. Events: `ci_repair_round_start`, `ci_repair_round_end`, `ci_repair_exhausted`. Two traps from the U2 review: the head jig last pushed comes from `AttemptCIRepairs` (the last round's `head_after`, else the proof ref), because `RetryPublish` returns only `publish_records`; and an authorization's `Completed` is returned without comparing `head_before`, so the worker must check that the recorded round repaired the head it is holding before treating the round as done.
-- **Required end-to-end test:** one round must run through the real `jig worker` wiring (`cmd/jig/worker_test.go`'s scripted-runtime harness), asserting that the round's trace events reach the control plane and that the handoff notes were present during the round. U3 could only unit-test `DeferToContinuation`; this is the first place the host wiring in `cmd/jig/worker.go` can be proven.
-- **Tests (fake gateway plus scripted runtime):** red → fix → green ends `accepted` with one round recorded; red twice at budget 1 ends `accepted_unpublished` naming both heads; a person's push before round 1 means no round; non-force push rejected mid-round ends cleanly; hold tripped in a round pushes nothing; cancel mid-round leaves the pushed head intact.
+- **Files:** `internal/worker/publish.go` (`ciPolicy.repairBudget`, `PublishSummary.CIRepairs`, the hook in `PublishingRunner.Run`), new `internal/worker/publish_repair.go`, `internal/worker/publish_repair_test.go`, new `cmd/jig/ci_repair_test.go`.
+- **Approach (as built):** `PublishingRunner.Run` enters `repairCI` only when publish ended `ci_failed`, the outcome carries a continuation, and the frozen definition declares a repair budget. Each round:
+  1. reads the rounds from the ledger (`AttemptCIRepairs`), so the head jig last pushed is the last round's `head_after`, else the proof ref, never memory;
+  2. stops on `ci_repair_exhausted` past the budget, and on `ci_repair_head_moved` when CI was red on a head jig did not push (KTD7);
+  3. authorizes round N under a freshened lease, and refuses a `Completed` it did not list (`ci_repair_conflict`);
+  4. attaches log tails (`FailedCheckLogs`) and calls `RepairCI`;
+  5. stops without pushing on a round that is cancelled (`ci_repair_cancelled`), not accepted (`ci_repair_round_failed`), held (`ci_repair_held`), or unchanged (`ci_repair_no_change`);
+  6. **fences again** (freshen, then re-authorize) right before the push, because a round can run for tens of minutes and a lease lost meanwhile must never push; refuses a round whose own commits carry a path it never changed (`publish_staging_escape`, the committed twin of the staging check, e.g. a definition's `git add -A` sweeping in the chain's test output); then commits and pushes the round's paths non-force, proves the remote head, records the round, and moves `remote_ref` to it;
+  7. runs a fresh `ci` step on the new head, where green publishes and red loops.
+
+  Rounds run on the same uncancelled context as publish: cancellation reaches the engine through the attempt's cancel channel and the CI wait through the lease, and git and ledger writes always finish. `PublishSummary.CIRepairs` lists every round with its heads, failed checks, changed paths, acceptance evidence, and outcome, because the attempt's top-level result describes the chain, and a pushed round is described here. Trace events come from the engine (`ci_repair_start`, then the round's phase, gate, and acceptance events); the worker has no trace sink of its own, so the per-round outcome lives in the publish summary rather than a worker event. The publish-only retry calls `publish` directly and so never repairs (R11).
+- **End-to-end test (done):** `cmd/jig/ci_repair_test.go` runs one round through a live `jig serve`, `workerAttemptRunner`, the real publishing runner, a scripted runtime, and a fake gateway. It plants a handoff note while CI is red and asserts that it survives the round, that the round's events reach the control plane, and that the scratch is gone once the attempt ends. Reintroducing the U3 wiring bug fails it ("scratch is gone").
+- **Tests:** one round to green; two rounds chained through the ledger; budget exhausted; a person's push before a round; a person's push mid-round (the non-force push is rejected and nothing is recorded); held, failed, cancelled and no-change rounds push nothing; no continuation, or a continuation without `on_fail`, means plain `ci_failed`; the publish retry never repairs.
 
 ### U6. Docs and the stock factory
 

internal/worker/publish.go

@@ -125,8 +125,10 @@ type PublishSummary struct {
 	// that were red (or still pending at the timeout), bounded.
 	CIRef      string    `json:"ci_ref,omitempty"`
 	CIFailures []CICheck `json:"ci_failures,omitempty"`
-	Performed  []string  `json:"performed_steps,omitempty"`
-	Reused     []string  `json:"reused_steps,omitempty"`
+	// CIRepairs lists the CI repair rounds this publish ran, in order.
+	CIRepairs []CIRepairSummary `json:"ci_repairs,omitempty"`
+	Performed []string          `json:"performed_steps,omitempty"`
+	Reused    []string          `json:"reused_steps,omitempty"`
 }
 
 // Published reports whether the pipeline completed with remote proof — the
@@ -235,6 +237,9 @@ func (o PublishOptions) ciRegistrationGrace() time.Duration {
 type ciPolicy struct {
 	wait    bool
 	timeout time.Duration
+	// repairBudget is publish.ci.on_fail's budget; zero means red CI ends
+	// the attempt, as it always has.
+	repairBudget int
 }
 
 // ciPolicyFor reads the policy out of a run snapshot. A snapshot that does
@@ -246,7 +251,11 @@ func ciPolicyFor(snapshot string) ciPolicy {
 	if err != nil {
 		return ciPolicy{}
 	}
-	return ciPolicy{wait: spec.WaitsForCI(), timeout: spec.CITimeout()}
+	policy := ciPolicy{wait: spec.WaitsForCI(), timeout: spec.CITimeout()}
+	if policy.wait && spec.Publish.CI.OnFail != nil {
+		policy.repairBudget = spec.Publish.CI.OnFail.Budget
+	}
+	return policy
 }
 
 func (o PublishOptions) authorName() string {
@@ -349,8 +358,16 @@ func (r *PublishingRunner) Run(ctx context.Context, prepared *PreparedAttempt) O
 	// The critical section: an uncancellable context, and the attempt's
 	// cancellation signal deliberately unread. Every command inside still
 	// carries its own timeout, so "uncancellable" is not "unbounded".
+	ci := ciPolicyFor(prepared.Claim.Snapshot)
 	summary := worker.publish(context.WithoutCancel(ctx), r.gateway, r.options,
-		target, changedPathsFromResult(outcome.Result), ciPolicyFor(prepared.Claim.Snapshot))
+		target, changedPathsFromResult(outcome.Result), ci)
+	if summary.Code == "ci_failed" && outcome.Continuation != nil && ci.repairBudget > 0 {
+		// Rounds run on the same uncancelled context: cancellation reaches
+		// the engine through the attempt's cancel channel and the CI wait
+		// through the lease, while git and the ledger always finish.
+		summary = worker.repairCI(context.WithoutCancel(ctx), r.gateway, r.options,
+			target, outcome.Continuation, summary, ci)
+	}
 	return publishedOutcome(outcome, summary)
 }