diff --git a/README.md b/README.md index 8c1df99..de72472 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Claude Code mods (plugins with a hooks module), served as a local folder marketp | mod | version | what it does | | --- | --- | --- | | [harness](harness/) | 0.3.4 | The integrated harness: the user's Claude rules enforced at the tool call, session tooling and workflow automation (formerly house-rules) | -| [auto-claude](auto-claude/) | 1.1.0 | Unattended run of a repository through to live: plans the long-term goal, milestones and deploy; its own loop builds and judges each milestone in a worktree; ships each (push, PR, required CI, merge), deploys and smoke-checks it; a watchdog repairs or routes around stalls; replans until the final check passes. Absorbed goal-mode in 1.0.0; since 1.1.0 the session never rests: the next milestone is built, stacked, while a PR waits on CI | +| [auto-claude](auto-claude/) | 1.3.0 | Unattended run of a repository through to live: plans the long-term goal, milestones and deploy; its own loop builds and judges each milestone in a worktree; ships each (push, PR, required CI, merge), deploys and smoke-checks it; a watchdog repairs or routes around stalls; replans until the final check passes. Absorbed goal-mode in 1.0.0; since 1.1.0 the session never rests: the next milestone is built, stacked, while a PR waits on CI; since 1.3.0 a line can wait on a training run or cache build (`:: after:`, `wait_until`) without ending the run | | [model-tier](model-tier/) | 0.11.0 | Routes each main-loop turn to fable, opus, sonnet or haiku and a reasoning effort by task complexity, with a `set_tier` tool the model calls to correct either | | [refactor](refactor/) | 0.1.3 | Enforces the architecture a repo declares in `.claude/architecture.json` at Edit/Write (layer direction, forbidden imports), notes code smells with the refactoring.guru technique to apply, `/arch audit` for violations and import cycles | diff --git a/auto-claude/.claude-plugin/plugin.json b/auto-claude/.claude-plugin/plugin.json index e5d3bce..14ce462 100644 --- a/auto-claude/.claude-plugin/plugin.json +++ b/auto-claude/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "auto-claude", - "version": "1.2.0", + "version": "1.3.0", "description": "Finishes a repository's work unattended and live: Claude plans the long-term end state, its milestones and how it deploys; the built-in loop builds each milestone in its own worktree and judges it; auto-claude ships it (push, PR, CI, merge when green), deploys and smoke-checks it, and a watchdog keeps the run moving until the final check passes. Absorbs goal-mode's loop (1.0.0).", "author": { "name": "unohee" @@ -29,9 +29,15 @@ "maxHours": { "type": "number", "title": "Max hours per run", - "description": "Stops and reports after this many hours", + "description": "Stops and reports after this many hours of working time (waiting on :: after: lines is not counted, up to maxWaitHours)", "default": 24 }, + "maxWaitHours": { + "type": "number", + "title": "Max waiting hours per run", + "description": "Hours of waiting on :: after: lines (training, cache builds) left out of maxHours", + "default": 72 + }, "ciPollSec": { "type": "number", "title": "CI poll interval (s)", diff --git a/auto-claude/README.md b/auto-claude/README.md index ac6bb60..3f82966 100644 --- a/auto-claude/README.md +++ b/auto-claude/README.md @@ -31,7 +31,8 @@ goal-mode merged into auto-claude in 1.0.0 (see "From goal-mode" below). Its loo - a final check; - a Deploy line; - at least 2 milestones, with a check on every line and no duplicates; - - no milestone that says it is merged, pushed or deployed. That happens after the milestone is met. + - no milestone that says it is merged, pushed or deployed. That happens after the milestone is met; + - no start condition in prose ("start only after X shows DONE"). It goes in `:: after:` (see "Waiting on compute"). 2. **Build.** When nothing is being built or shipped, auto-claude starts the next pending line itself. Lines with `:: base:` go first. - The line gets its own worktree, `worktrees/goal-NN-…` on branch `goal/NN-…`, made from `origin/`, or from the line's `:: base:`. - Claude gets a kickoff prompt. At every stop, the loop runs the line's check in the worktree and refuses uncommitted changes. Then an independent evaluator (`claude-sonnet-5-5`) judges the milestone from the transcript. @@ -45,7 +46,7 @@ goal-mode merged into auto-claude in 1.0.0 (see "From goal-mode" below). Its loo - When CI fails or the PR conflicts, auto-claude appends a fix goal built on the milestone's branch (`:: base: goal/NN-…`), and the loop builds that next. After 3 failures the milestone is held. - **Deploy.** After each merge, as the plan says (see "Deploying"). - **The session never rests through CI.** While a PR waits on CI, the next milestone starts at once, **stacked** on that PR's branch: the commit it starts from is recorded. Up to 2 milestones are built ahead this way. They ship in order. When the one before lands (squashed) or is held, the stacked branch runs `git rebase --onto origin/ `, which drops its parent's commits, and is then pushed like any other. A rebased branch is pushed with `--force-with-lease`, since the session may have pushed it before. A stacked branch that does not rebase cleanly gets a fix goal that does the rebase by hand. While a broken deploy is being repaired, nothing new is stacked and nothing stacked ships: the repair, built from the default branch, lands and deploys first. A fix goal (`:: base:`) always starts on its own base. -4. **Finish or replan.** When the queue is done, auto-claude runs the final check on a fresh checkout of the merged default branch. +4. **Finish or replan.** When the queue is done, auto-claude runs the final check on a fresh checkout of the merged default branch. Lines still waiting on their `:: after:` mean the queue is not done: no final check runs and no round is spent until they have started. - If it passes, auto-claude deploys once more if a deploy is still due. Then it writes a `## Result` section and stops. - If it fails, Claude appends new milestones from the check's output (round 2, and so on, up to `maxRounds`). @@ -107,7 +108,7 @@ A run keeps the repository's Linear project in step, through the session's own ` - `rm -rf` outside the worktree; - edits in the repository outside the worktree; - any regex in `.claude/goal-loop.deny`. -- **Caps.** A run stops and reports after `maxMerges` (20) merges, after `maxHours` (24), or after `maxRounds` (3) rounds without the final check passing. +- **Caps.** A run stops and reports after `maxMerges` (20) merges, after `maxHours` (24) of working time, or after `maxRounds` (3) rounds without the final check passing. Time spent with nothing to do but wait on `:: after:` lines does not count toward `maxHours`, up to `maxWaitHours` (72) of it. - **Kill switch.** Create `.claude/auto-claude.stop`, or run `/auto-claude stop`. Either one ends the loop's milestone as well: it is no longer judged, and its guards lift. `/auto-claude start` and `resume` move a stop file left from an earlier run to `.claude/auto-claude.stop.cleared`. `resume` goes back to the phase the run stopped in: planning stays planning. ## Nothing waits on the person @@ -117,7 +118,9 @@ A run never asks the person to look at `status` or to run a command. When someth | what stops | what happens | | --- | --- | | A milestone is open, but the session has had no turn for 1 min: the model ended its turn to "wait", a prompt was dropped, an interrupt, or a stop the judge could not read | The milestone's prompt is sent again, after 1, 3 and 6 min of quiet. At 12 min, the milestone is blocked and the next one starts. A stop being judged, or a retry pause, is not quiet. Turns are tracked with `turn.start` / `turn.complete`. | -| The model waits inside a turn | Refused while a run is on: `sleep` of 60 s or more (or a computed duration), `gh run watch`, `gh pr checks --watch`, `Monitor`, `CronCreate` and `ScheduleWakeup`. auto-claude watches CI, merges and deploys itself, and prompts the session when there is work. Shorter sleeps (a check waiting for a server) pass. | +| The model waits inside a turn | Refused while a run is on: `sleep` of 60 s or more (or a computed duration), `gh run watch`, `gh pr checks --watch`, `Monitor`, `CronCreate` and `ScheduleWakeup`. auto-claude watches CI, merges and deploys itself, and prompts the session when there is work. The refusal points to `wait_until` for a job with its own clock. Shorter sleeps (a check waiting for a server) pass. | +| A milestone needs a training run or a cache build to finish | The line waits on its `:: after:` command, polled every 3 min, while other lines are built (see "Waiting on compute"). The run neither ends nor counts it as idle. | +| A waited-on job died, or its wait runs past its `within` | `:: alive:` failing 2 polls in a row, or the wait passing its `within`, sends the model one prompt with the last poll's output to look at the job and relaunch it or fix the line. Past twice its `within`, the line is held. | | A shipping error to diagnose while the session builds another milestone | The diagnosis prompt waits until that milestone is done, so the stop judge never judges the wrong work. Nothing is held before the model was asked and had 10 min to act (after an hour unasked, the hold goes ahead). | | Planning ends without `auto_plan_ready` | A reminder 1 min after that turn ended, listing what the plan files lack (15 min at most otherwise). After 3, the run stops. | | An SSH remote refuses the push (`Permission denied (publickey)`) | The push goes to the https URL of the same repository. | @@ -131,10 +134,29 @@ A run never asks the person to look at `status` or to run a command. When someth `/auto-claude status` ends with a `stalled:` line naming the cause whenever the run is not moving. Only the owning session receives the unattended rules. Progress shows in the status line piece (`auto·R1·M3/7·PR#12`) and in notifications. +## Waiting on compute + +A milestone may need something with its own clock first: a training run, a cache build, or a file another milestone lands on the default branch. The driver watches it; the model never schedules its own wake-up. + +``` +- [ ] record the M21 cache :: check: pytest -q tests/test_m21.py :: after: grep -q "CACHE DONE" ~/data/m21/cache.log :: alive: tmux has-session -t m21cache :: within: 8h +- [ ] the M22 verdict :: check: python -m src.m22.check final :: after: git cat-file -e origin/main:docs/m22/runs.json +``` + +- **`:: after: `** must exit 0 before the line starts. auto-claude runs it from the repository root every 3 min (30 s timeout), fetching first when it names `origin/`. Lines that wait are skipped; the others are built meanwhile. +- **`:: alive: `** (optional) exits 0 while the awaited job runs. Failing 2 polls in a row asks the model, once, to look at the job. +- **`:: within: <90m | 12h | 2d>`** (optional): past it the model is asked to look; past twice it the line is held. A line without one is still asked about, once, after 12 h, and never held. +- `after` and `alive` run from the repository root, never from a worktree, both when polled and when `wait_until` checks them. +- The fields go after the check, in this order, with `:: base:` last. +- **`mcp__auto-claude__wait_until`** does the same from inside a milestone: `{ until, alive?, within?, reason }`. It is refused if `until` already passes or the worktree has uncommitted changes. At the next stop the line goes back to the queue with `:: after:` and `:: base: `, so it restarts from the work built so far. A milestone that was stacked keeps the commit it was stacked on, so its restarted branch is rebased off the parent like any stacked one. +- `/auto-claude status` lists each wait, how long it has waited and its last poll; the status line piece shows `·waitN`. + +Measured in ArtifactNet (2026-11-09 to 11-15): lines carried prose start conditions the driver could not read, so each was started at once and deferred by the model. With every line deferred, the final check failed and a round was spent; three rounds went in 25 minutes, the run stopped, and nothing resumed it when the cache finished hours later. + ## The loop - **Stuck.** 5 unmet checks in one turn, two unreadable verdicts, or an evaluator that fails: the loop waits 2 minutes, then sends a "change your approach" prompt, at most twice. After that the line is marked `[!]` and the run moves on. -- **Deferring.** The model may call `mcp__auto-claude__defer_goal` for an external blocker after 2 failed checks. The deferral is applied at the next stop. +- **Deferring.** The model may call `mcp__auto-claude__defer_goal` for an external blocker after 2 failed checks. The deferral is applied at the next stop. Waiting on compute is not a blocker: it uses `wait_until`. - **Evaluator.** Each call takes about 2–20 s. It is billed to the session's credentials but does not appear in `/cost`. A transcript over the 4 MiB read cap is judged by a fork of the session instead. - **Files.** `.claude/goals.md`, `.claude/goal-loop.log` (JSONL, its last 2000 lines) and `.claude/goal-loop.deny` are excluded from git through `.git/info/exclude`, along with `worktrees/`. - **`--queue`.** `/auto-claude start --queue` builds the queue you wrote, line by line, with the same loop. Nothing is pushed, merged or deployed, and the branches are left for review, as goal-mode's `loop on` did. @@ -147,4 +169,4 @@ A run never asks the person to look at `status` or to run a command. When someth - **Session goals** (`set_goal`, `/goal-mode set|auto|clear` and the outcome nudges) are gone. Use Claude Code's `/goal` and `ProposeGoal`. - **A run already going** when 1.0 loads keeps its milestone. The loop's state lived in goal-mode's store, which this plugin cannot read, so the open milestone is found again from the last `start` in `.claude/goal-loop.log` and prompted again. New branches number above the highest existing `goal/NN-*`. -Options: `ciMode` (`auto` | `github` | `local`), `maxRounds`, `maxMerges`, `maxHours` and `ciPollSec`. +Options: `ciMode` (`auto` | `github` | `local`), `maxRounds`, `maxMerges`, `maxHours`, `maxWaitHours` and `ciPollSec`. diff --git a/auto-claude/hooks/auto.test.ts b/auto-claude/hooks/auto.test.ts index 9f99da7..f2dec0f 100644 --- a/auto-claude/hooks/auto.test.ts +++ b/auto-claude/hooks/auto.test.ts @@ -75,6 +75,8 @@ function world(on: On, store: Record = {}) { // Every `git rebase`, and whether a stacked branch's rebase --onto conflicts. rebases: [] as string[], rebaseFails: false, + // Exits of particular `bash -lc ` runs (a wait's after and alive), over checkExit. + cmdExit: new Map(), clock: mock.clock(on, { now: 1_000 }), } const removed = (path: string) => [...w.gone].some(g => path === g || path.startsWith(`${g}/`)) @@ -182,6 +184,7 @@ function world(on: On, store: Record = {}) { w.adds.push(a) w.gone.delete(String(e.argv[5])) } + if (e.argv[0] === 'bash' && w.cmdExit.has(String(e.argv[2]))) return ok('', w.cmdExit.get(String(e.argv[2]))!) if (e.argv[0] === 'bash' && cwd.endsWith('/worktrees/auto-deploy')) return ok('', String(e.argv[2]).startsWith('curl') ? w.smokeExit : w.deployExit) if (e.argv[0] === 'bash') return ok('', cwd.endsWith('/worktrees/auto-verify') && w.freshExit !== undefined ? w.freshExit : w.checkExit) if (a.includes('status --porcelain') && !a.includes('--untracked-files')) return ok(w.dirty) @@ -1266,3 +1269,195 @@ describe('Linear sync', () => { expect((await cmd($, 'status')).text).toContain('linear: off') }) }) + +describe('waits on compute (:: after:, wait_until)', () => { + const AFTER = 'grep -q "CACHE DONE" /data/cache.log' + const ALIVE = 'tmux has-session -t cache' + const WAITQ = `- [ ] add() returns a+b :: check: python3 -m unittest -q test_add\n- [ ] record the cache :: check: python3 -m unittest -q test_cache :: after: ${AFTER} :: alive: ${ALIVE}\n` + const finals = (w: W) => w.runs.filter(r => r.includes('worktree add --detach /repo/worktrees/auto-final')).length + + test('only waiting lines left: no final check, no round spent, no stop, no idle nudge; it starts when its after passes (ArtifactNet 2026-11-15)', { timeoutMs: 60_000 }, async ($, on) => { + const w = world(on) + w.cmdExit.set(AFTER, 1) + w.cmdExit.set(ALIVE, 0) + await $.session.start({ cwd: '/repo', surface: 'terminal', isInteractive: true }) + await cmd($, 'start') + w.files.set('/repo/.claude/auto-plan.md', PLAN) + w.files.set(GOALS, WAITQ) + expect(String((await ready($)).result)).toContain('Plan accepted') + await w.clock.advance(1_200) + // The waiting line is not started first, though it is not first to wait. + expect(w.adds).toEqual([`git -C /repo worktree add ${WT1} -b ${BR1} origin/main`]) + await meet($, w) + await w.clock.advance(TICK) + expect(w.pushes.some(p => p.includes('HEAD:main') && p.includes(WT1))).toBe(true) + // Five hours of a cache build: the run waits, polled every 3 min. + for (let i = 0; i < 10; i++) await w.clock.advance(30 * MIN) + expect(finals(w)).toBe(0) + expect(w.adds).toHaveLength(1) + expect(w.notifies.some(n => n.includes('idle') || n.includes('stopped') || n.includes('finished'))).toBe(false) + expect(w.runs.filter(r => r === `bash -lc ${AFTER}`).length).toBeGreaterThan(50) + const status = (await cmd($, 'status')).text + expect(status).toContain('auto-claude on · phase run · round 1') + expect(status).toMatch(/waiting 5\.\d h: record the cache on `grep -q "CACHE DONE"/) + expect(status).not.toContain('stalled') + // The cache is done: the next poll starts the line. + w.cmdExit.set(AFTER, 0) + await w.clock.advance(3 * MIN + TICK) + expect(w.notifies.some(n => n.includes('wait over after 5.') && n.includes('record the cache'))).toBe(true) + expect(w.adds.at(-1)).toBe('git -C /repo worktree add /repo/worktrees/goal-02-record-the-cache -b goal/02-record-the-cache origin/main') + await meet($, w) + for (let i = 0; i < 3; i++) await w.clock.advance(TICK) + expect(finals(w)).toBe(1) + expect(w.notifies.at(-1)).toContain('finished: the final check passed after 1 round(s). 2 merged, 0 held') + }) + + test('the time cap leaves the waiting out: a run that waits 30 h is not stopped at 24 h', { timeoutMs: 60_000 }, async ($, on) => { + const w = world(on) + w.cmdExit.set(AFTER, 1) + w.cmdExit.set(ALIVE, 0) + await $.session.start({ cwd: '/repo', surface: 'terminal', isInteractive: true }) + await cmd($, 'start') + w.files.set('/repo/.claude/auto-plan.md', PLAN) + w.files.set(GOALS, WAITQ) + await ready($) + await w.clock.advance(1_200) + await meet($, w) + for (let i = 0; i < 30; i++) await w.clock.advance(60 * MIN) + expect(w.notifies.some(n => n.includes('time cap'))).toBe(false) + expect((await cmd($, 'status')).text).toContain('auto-claude on') + }) + + test('a job that died: alive failing twice asks the model once, with the poll output; once it runs again it may be asked again', { timeoutMs: 60_000 }, async ($, on) => { + const w = world(on) + w.cmdExit.set(AFTER, 1) + w.cmdExit.set(ALIVE, 0) + await $.session.start({ cwd: '/repo', surface: 'terminal', isInteractive: true }) + await cmd($, 'start') + w.files.set('/repo/.claude/auto-plan.md', PLAN) + w.files.set(GOALS, WAITQ) + await ready($) + await w.clock.advance(1_200) + await meet($, w) + await w.clock.advance(10 * MIN) + expect(w.prompts.some(p => p.includes('A wait looks dead'))).toBe(false) + w.cmdExit.set(ALIVE, 1) + await w.clock.advance(7 * MIN) + const asked = w.prompts.filter(p => p.includes('A wait looks dead')) + expect(asked).toHaveLength(1) + expect(asked[0]).toContain(`alive \`${ALIVE}\` exited 1`) + expect(asked[0]).toContain('Relaunch it if it died') + expect(w.notifies.some(n => n.includes('a waited-on job looks dead'))).toBe(true) + await w.clock.advance(30 * MIN) + expect(w.prompts.filter(p => p.includes('A wait looks dead'))).toHaveLength(1) + expect((await cmd($, 'status')).text).toContain('alive failing') + }) + + test('a wait past its within asks the model once; past twice it the line is held and the run goes on', { timeoutMs: 60_000 }, async ($, on) => { + const w = world(on) + w.cmdExit.set(AFTER, 1) + await $.session.start({ cwd: '/repo', surface: 'terminal', isInteractive: true }) + await cmd($, 'start') + w.files.set('/repo/.claude/auto-plan.md', PLAN) + w.files.set(GOALS, `- [ ] add() returns a+b :: check: python3 -m unittest -q test_add\n- [ ] record the cache :: check: true :: after: ${AFTER} :: within: 30m\n`) + await ready($) + await w.clock.advance(1_200) + await meet($, w) + await w.clock.advance(36 * MIN) + expect(w.prompts.filter(p => p.includes('A wait looks late'))).toHaveLength(1) + await w.clock.advance(30 * MIN) + expect(w.files.get(GOALS)).toMatch(/- \[!\] record the cache .*twice its within/) + for (let i = 0; i < 3; i++) await w.clock.advance(TICK) + expect(finals(w)).toBe(1) + expect((await cmd($, 'status')).text).toContain('1 merged, 1 held') + }) + + test('wait_until: refused when done already or uncommitted; applied at the next stop, the line waits on its branch, others go on, then it resumes there', { timeoutMs: 60_000 }, async ($, on) => { + const w = world(on) + const UNTIL = 'test -f /data/m22/final.json' + await planned($, w) + const call = (input: Record) => $.tool.call({ tool: 'mcp__auto-claude__wait_until', ...input } as never) + w.cmdExit.set(UNTIL, 0) + expect((await call({ until: UNTIL, reason: 'M22 training' })).deny).toContain('already exits 0') + w.cmdExit.set(UNTIL, 1) + expect((await call({ until: UNTIL })).deny).toContain('reason is required') + expect((await call({ until: UNTIL, reason: 'x', within: 'soon' })).deny).toContain('within must be a duration') + w.dirty = ' M calc.py\n' + expect((await call({ until: UNTIL, reason: 'M22 training' })).deny).toContain('Commit your work first') + w.dirty = '' + expect(String((await call({ until: UNTIL, alive: 'tmux has-session -t m22', within: '12h', reason: 'M22 training' })).result)).toContain('Wait recorded') + // The stop puts the line back to wait instead of judging it. + expect((await $.classic.Stop(STOP)).block).toBeUndefined() + expect(w.evaluations).toBe(0) + const line = parseQueue(w.files.get(GOALS) ?? '')[0]! + expect(line).toMatchObject({ mark: ' ', condition: 'add() returns a+b', after: UNTIL, alive: 'tmux has-session -t m22', withinMs: 12 * 3_600_000, base: BR1 }) + expect(w.files.get(LOG)).toContain('"event":"waiting"') + await w.clock.advance(TICK) + expect(w.notifies.some(n => n.includes('waiting: add() returns a+b'))).toBe(true) + // The other milestone is built meanwhile. + expect(w.adds.at(-1)).toBe(`git -C /repo worktree add ${WT2} -b ${BR2} origin/main`) + await meet($, w) + for (let i = 0; i < 12; i++) await w.clock.advance(TICK) + expect(finals(w)).toBe(0) + w.cmdExit.set(UNTIL, 0) + await w.clock.advance(3 * MIN + TICK) + expect(w.adds.at(-1)).toBe(`git -C /repo worktree add /repo/worktrees/goal-03-add-returns-a-b -b goal/03-add-returns-a-b ${BR1}`) + expect(w.prompts.at(-1)).toContain('Milestone #3: add() returns a+b') + }) + + test('a wait with no within or alive is still asked about once after 12 h, and never held', { timeoutMs: 60_000 }, async ($, on) => { + const w = world(on) + w.cmdExit.set(AFTER, 1) + await $.session.start({ cwd: '/repo', surface: 'terminal', isInteractive: true }) + await cmd($, 'start') + w.files.set('/repo/.claude/auto-plan.md', PLAN) + w.files.set(GOALS, `- [ ] add() returns a+b :: check: python3 -m unittest -q test_add\n- [ ] record the cache :: check: true :: after: ${AFTER}\n`) + await ready($) + await w.clock.advance(1_200) + await meet($, w) + for (let i = 0; i < 11; i++) await w.clock.advance(60 * MIN) + expect(w.prompts.some(p => p.includes('A wait looks late'))).toBe(false) + for (let i = 0; i < 2; i++) await w.clock.advance(60 * MIN) + const asked = w.prompts.filter(p => p.includes('A wait looks late')) + expect(asked).toHaveLength(1) + expect(asked[0]).toContain('it has no within of its own') + for (let i = 0; i < 24; i++) await w.clock.advance(60 * MIN) + expect(w.prompts.filter(p => p.includes('A wait looks late'))).toHaveLength(1) + expect(w.files.get(GOALS)).toMatch(/- \[ \] record the cache/) + expect(finals(w)).toBe(0) + }) + + test('a stacked milestone that waits keeps the commit it was stacked on: restarted, it is rebased off its parent before it ships', { timeoutMs: 60_000 }, async ($, on) => { + const w = world(on) + const UNTIL = 'test -f /data/sub.ready' + w.cmdExit.set(UNTIL, 1) + w.github = true + w.checks = '[{"bucket":"pending"}]' + await planned($, w) + await meet($, w) + // Milestone 2 is stacked on milestone 1, whose CI is pending. + expect(w.adds.at(-1)).toBe(`git -C /repo worktree add ${WT2} -b ${BR2} ${BR1}`) + expect(String((await $.tool.call({ tool: 'mcp__auto-claude__wait_until', until: UNTIL, reason: 'sub needs its data' } as never)).result)).toContain('Wait recorded') + await $.classic.Stop(STOP) + await w.clock.advance(TICK) + // Milestone 1 lands meanwhile. + w.checks = '[{"bucket":"pass"}]' + await w.clock.advance(10_000) + expect(w.merged).toHaveLength(1) + w.cmdExit.set(UNTIL, 0) + await w.clock.advance(3 * MIN + TICK) + const WT3 = '/repo/worktrees/goal-03-sub-returns-a-b' + expect(w.adds.at(-1)).toBe(`git -C /repo worktree add ${WT3} -b goal/03-sub-returns-a-b ${BR2}`) + await meet($, w) + await w.clock.advance(10_000) + expect(w.rebases).toContain(`git -C ${WT3} rebase --onto origin/main tip-of-${BR1}`) + expect(w.merged).toHaveLength(2) + }) + + test('wait_until is only for a milestone being built', async ($, on) => { + const w = world(on) + await $.session.start({ cwd: '/repo', surface: 'terminal', isInteractive: true }) + expect((await $.tool.call({ tool: 'mcp__auto-claude__wait_until', until: 'false', reason: 'x' } as never)).deny).toContain('only for milestones') + expect(w.prompts).toEqual([]) + }) +}) diff --git a/auto-claude/hooks/loop.ts b/auto-claude/hooks/loop.ts index fbeb4fc..f5a9b67 100644 --- a/auto-claude/hooks/loop.ts +++ b/auto-claude/hooks/loop.ts @@ -14,10 +14,15 @@ import type { Goal, LoopCurrent, LoopRecord, LoopView } from '../types' import { clip, evaluate } from './evaluator' import type { StopFields } from './evaluator' import { checkBash, checkPath, uncommitted, WAIT_MAX_S, waitsLong } from './guard' -import { markItem, nextPending, parseQueue, slugify } from './queue' +import { markItem, nextPending, parseDuration, parseQueue, requeueItem, slugify, waitingItems } from './queue' +import type { QueueItem } from './queue' export const DEFER_TOOL = 'defer_goal' export const DEFER_TOOL_ID = 'mcp__auto-claude__defer_goal' +export const WAIT_TOOL = 'wait_until' +export const WAIT_TOOL_ID = 'mcp__auto-claude__wait_until' +// How long wait_until's own probe of `until` may run. +const WAIT_PROBE_MS = 30_000 const MAX_UNMET_PER_TURN = 5 const MAX_CHECKS = 30 @@ -53,7 +58,7 @@ export function setOnFinish(f: (() => void) | undefined): void { onFinish = f } -export type LogEvent = 'start' | 'resume' | 'unmet' | 'met' | 'retry' | 'blocked' | 'deferred' | 'denied' | 'queue-empty' | 'error' +export type LogEvent = 'start' | 'resume' | 'unmet' | 'met' | 'retry' | 'blocked' | 'deferred' | 'denied' | 'queue-empty' | 'waiting' | 'error' const loopKey = (root: string) => `loop:${root}` export const queuePath = (root: string) => `${root}/.claude/goals.md` @@ -90,17 +95,21 @@ export function maxBranchSeq(refs: string): number { // Takes the next pending queue line, makes its worktree from `base` (or the // line's own `:: base:`), sets the goal and sends the kickoff. Undefined when -// the queue has no pending line (logged as queue-empty) or none could start. -export async function startNext($: EngineInterface, root: string, base: string, remote: string): Promise { +// the queue has no pending line (logged as queue-empty), when every pending +// line still waits on its `:: after:` (`ready`, the driver's polls), or when +// none could start. +export async function startNext($: EngineInterface, root: string, base: string, remote: string, ready?: (item: QueueItem) => boolean): Promise { let rec: LoopRecord = (await readLoop($, root)) ?? { repoRoot: root, seq: 0, retries: 0 } if (rec.current !== undefined) return rec.current const refs = await run($, ['git', '-C', root, 'for-each-ref', '--format=%(refname:short)', 'refs/heads/goal/'], root, 15_000).catch(() => undefined) rec = { ...rec, seq: Math.max(rec.seq, maxBranchSeq(refs?.stdout ?? '')) } for (let attempt = 0; attempt < 5; attempt++) { - const item = nextPending(parseQueue((await readText($, queuePath(root))) ?? '')) + const items = parseQueue((await readText($, queuePath(root))) ?? '') + const item = nextPending(items, ready) if (item === undefined) { await writeLoop($, { ...rec, current: undefined, retries: 0 }) - await appendLog($, root, { event: 'queue-empty' }) + // Lines still waiting: the queue is not empty, and the driver polls them. + if (waitingItems(items, ready ?? (i => i.after === undefined)).length === 0) await appendLog($, root, { event: 'queue-empty' }) return undefined } const seq = rec.seq + 1 @@ -209,6 +218,10 @@ async function judge($: EngineInterface, { rec, vi await finishGoal($, rec, { mark: '!', note: `deferred ${isoNow()}: ${goal.deferPending}`, event: 'deferred', reason: goal.deferPending }) return below } + if (goal.waitPending !== undefined) { + await finishWait($, rec, goal.waitPending) + return below + } // 1. The queue's own check, run deterministically in the worktree. let evidence: string | undefined @@ -296,6 +309,49 @@ async function finishGoal($: EngineInterface, rec: LoopRecord, { mark, note, eve onFinish?.() } +// wait_until applied: the line goes back to the queue, waiting on its +// command, and restarts from the branch built so far once that passes. The +// session is free for other lines meanwhile. +async function finishWait($: EngineInterface, rec: LoopRecord, wait: NonNullable): Promise { + const current = rec.current + if (current === undefined) return + const text = await readText($, queuePath(rec.repoRoot)) + if (text !== undefined) { + await $.fs.write(queuePath(rec.repoRoot), requeueItem(text, current.condition, wait, current.branch, `waiting since ${isoNow()}: ${wait.reason}`)).catch(() => undefined) + } + await appendLog($, rec.repoRoot, { event: 'waiting', goal: current.condition, reason: clip(wait.reason, 300), branch: current.branch, after: wait.after }) + $.ui.toast(`auto-claude: milestone waits on \`${clip(wait.after, 60)}\`: ${clip(current.condition, 60)}`) + await update($, goalAtom, () => null) + await update($, viewAtom, () => null) + await writeLoop($, { ...rec, current: undefined, retries: 0, wakes: 0 }) + onFinish?.() +} + +// wait_until: the milestone needs something with its own clock (a training +// run, a cache build, a file another milestone lands). Recorded only; the next +// Stop puts the line back to wait, so the driver, not the model, watches it. +export async function waitCall($: EngineInterface, input: Record): Promise<{ deny: string } | { result: string }> { + const text = (k: string) => (typeof input[k] === 'string' ? (input[k] as string).trim() : '') + const until = text('until') + const alive = text('alive') + const within = text('within') + const reason = text('reason') + const goal = await read($, goalAtom) + const view = await read($, viewAtom) + if (goal === null || view === null) return { deny: `${WAIT_TOOL_ID} is only for milestones of a running auto-claude loop.` } + if (until === '') return { deny: 'until is required: a shell command that exits 0 once what you wait for is done (e.g. grep -q "DONE" ~/data/x/cache.log).' } + if (reason === '') return { deny: 'reason is required.' } + if (within !== '' && parseDuration(within) === undefined) return { deny: `within must be a duration like 90m, 12h or 2d, not "${within}".` } + // Run where the driver polls it: the repository root (the worktree may be gone by then). + const probe = await run($, ['bash', '-lc', until], view.repoRoot, WAIT_PROBE_MS).catch(error => ({ exitCode: -1, stdout: '', stderr: String(error) })) + if (probe.exitCode === 0) return { deny: `\`${until}\` already exits 0: what it waits for is done. Keep working on the milestone.` } + const status = await run($, ['git', '-C', view.worktree, 'status', '--porcelain'], view.worktree, 60_000).catch(() => undefined) + const dirty = status === undefined ? [] : uncommitted(status.stdout) + if (dirty.length > 0) return { deny: `Commit your work first: the milestone restarts from ${view.branch} once the wait is over.\n${clip(dirty.join('\n'), 400)}` } + await update($, goalAtom, g => (g === null ? null : { ...g, waitPending: { after: until, ...(alive !== '' ? { alive } : {}), ...(within !== '' ? { within } : {}), reason } })) + return { result: `Wait recorded. End the turn now. auto-claude polls \`${clip(until, 120)}\` every few minutes and restarts this milestone from ${view.branch} when it exits 0; other milestones go on meanwhile.${alive !== '' ? ' If the alive command fails, you are asked to look.' : ''}` } +} + // defer_goal: recorded only; the next Stop marks the goal and moves on, so a // new goal is never judged against this turn's transcript. export async function deferCall($: EngineInterface, reasonRaw: unknown): Promise<{ deny: string } | { result: string }> { @@ -317,7 +373,7 @@ export async function deferCall($: EngineInterface, reasonRaw: unknown): Promise // A run never rests inside a turn either: auto-claude watches CI, merges and // deploys itself, and wakes the session when it has work for it. const WAIT_TOOLS: ReadonlySet = new Set(['CronCreate', 'ScheduleWakeup', 'Monitor']) -const WAIT_REFUSAL = `auto-claude run: do not wait or schedule a wake-up. auto-claude watches CI, merges and deploys by itself and prompts this session when there is work; keep working on the milestone, or end the turn if it is done (sleeps under ${WAIT_MAX_S} s are fine).` +const WAIT_REFUSAL = `auto-claude run: do not wait or schedule a wake-up inside a turn. auto-claude watches CI, merges and deploys by itself. To wait on a training run, a cache build or another job, call ${WAIT_TOOL_ID} with a shell command that exits 0 once it is done, then end the turn: the driver polls it and prompts this session again. Otherwise keep working on the milestone, or end the turn if it is done (sleeps under ${WAIT_MAX_S} s are fine).` export function guardRefusal(e: Record, view: LoopView | null, runOn: boolean): string | undefined { const tool = String(e.tool) @@ -338,8 +394,9 @@ export const LOOP_POLICY = `# auto-claude milestone — ACTIVE You are building one milestone of an unattended auto-claude run. Nobody is watching this session. - Do not ask questions (AskUserQuestion is refused) and do not enter plan mode. Decide with a stated assumption and record it in the commit message. - A "CONFIDENCE HALT" becomes a call to ${DEFER_TOOL_ID} with the reason, and only for external blockers (credentials, a human decision, missing access) after at least ${MIN_CHECKS_TO_DEFER} failed checks. Otherwise keep working. +- Waiting on compute (a training run, a cache build, a job you launched, a file another milestone lands) is not a blocker: commit, call ${WAIT_TOOL_ID} with a shell command that exits 0 once it is done (and an alive command that fails if the job died), then end the turn. Never defer for it. - Work only inside the milestone's worktree; commit after each meaningful step. Push only your goal/* branch, never main or with force. auto-claude pushes, merges and deploys for you. -- Never wait inside a turn (no long sleep, CI watch, Monitor, cron or scheduled wake-up). While an earlier milestone's CI runs you are already building the next one; when this one is done, end the turn and the next prompt comes by itself.` +- Never wait inside a turn (no long sleep, CI watch, Monitor, cron or scheduled wake-up); ${WAIT_TOOL_ID} is how a milestone waits. While an earlier milestone's CI runs you are already building the next one; when this one is done, end the turn and the next prompt comes by itself.` export function kickoffText(current: LoopCurrent, seq: number): string { return [ @@ -351,6 +408,7 @@ export function kickoffText(current: LoopCurrent, seq: number): string { 'Commit after each meaningful step: the milestone is not accepted with uncommitted changes.', `You may push only this branch (git push -u origin ${current.branch}); never main/master, never with force. auto-claude opens the PR, merges and deploys once the milestone is met.`, `Nobody is watching: do not ask questions. Decide with a stated assumption and record it in the commit message. For an external blocker (credentials, a human decision, missing access), call ${DEFER_TOOL_ID} after at least ${MIN_CHECKS_TO_DEFER} failed checks.`, + `If it needs a job to finish first (training, a cache build), launch or check the job, commit, and call ${WAIT_TOOL_ID} with a shell command that exits 0 once it is done; auto-claude polls it and resumes this milestone.`, ].join('\n') } diff --git a/auto-claude/hooks/plan.ts b/auto-claude/hooks/plan.ts index 6eb7b5e..ea46cee 100644 --- a/auto-claude/hooks/plan.ts +++ b/auto-claude/hooks/plan.ts @@ -61,6 +61,14 @@ export const MIN_MILESTONES = 2 // Words that put shipping into a milestone's end state, which the loop judges // before anything is pushed (measured: "…, merged to main" stayed unmet). +// A start condition in prose ("(start only after X shows DONE)"): the driver +// cannot read it, so it started the line at once and the model deferred it, +// which spent the run's rounds while the awaited job still ran (measured in +// ArtifactNet, 2026-11-09 and 11-15). It belongs in `:: after: `. +// Only the parenthesised prefix seen there: an end state that merely says +// "the server starts only after migrations" is not a wait. +const PROSE_WAIT = /^\([^)]*\bstarts? only (after|when|once)\b/i + const SHIPPING = /\b(merged|merge[sd]? (in)?to|pushed|push(ed)? to|pull request|PR (is )?(open|opened|created|merged)|deployed|is live)\b|머지|병합|푸시|배포(됨|되었|된|완료)/i // The plan Claude wrote is the contract the run holds it to: a final check, @@ -81,7 +89,11 @@ export function validatePlan(plan: string | undefined, queue: string | undefined for (const item of pending) { if (SHIPPING.test(item.condition)) errors.push(`a milestone says what is true on its own branch; merging, pushing, PRs and deploying happen after it is met: "${item.condition.slice(0, 80)}"`) if (item.check === undefined) errors.push(`milestone without a check: "${item.condition.slice(0, 80)}"`) - if (item.base !== undefined) errors.push(`milestones take no ":: base:" (auto-claude sets the base): "${item.condition.slice(0, 80)}"`) + // A line put back to wait (wait_until) keeps the branch it was built on. + if (item.base !== undefined && item.after === undefined) errors.push(`milestones take no ":: base:" (auto-claude sets the base): "${item.condition.slice(0, 80)}"`) + if (item.after === undefined && PROSE_WAIT.test(item.condition)) { + errors.push(`a start condition in prose is never checked, so the line starts at once: write it as ":: after: " after the check (e.g. :: after: grep -q DONE ~/data/x/cache.log): "${item.condition.slice(0, 80)}"`) + } } const seen = new Set() for (const item of items) { @@ -119,6 +131,7 @@ export function kickoffText(hint: string, base: string, noRemote = false, fixedD 'Aim at the long-term end state of the whole work, not a first step. Then write two files in the repository root:', '1. .claude/auto-plan.md: ## Goal (the end state, with its quality bar), a line `Final check: `, the deploy lines below, ## Milestones (ordered, why each), ## Decisions (empty for now).', `2. .claude/goals.md: one line per milestone, in order: \`- [ ] :: check: \`. Each milestone is one reviewable PR, built in its own worktree from ${base} after the earlier milestones are merged, so each check must pass from such a checkout. State only what is true on the milestone's own branch: never \"merged\", \"pushed\", \"PR opened\" or \"deployed\" (that happens after it is met).`, + 'A milestone that needs something with its own clock first (a training run, a cache build, a file an earlier milestone lands on the default branch) adds, after its check, `:: after: ` and optionally `:: alive: ` and `:: within: <12h>`. Examples: `:: after: grep -q "CACHE DONE" ~/data/m21/cache.log :: alive: tmux has-session -t m21cache :: within: 8h`, `:: after: git cat-file -e origin/main:docs/m22/final.json`. auto-claude polls it, builds other milestones meanwhile and starts the line when it passes; the run does not end while lines wait. Never write such a condition in prose.', ...deployRules(fixedDeploy), ...(linearBrief !== undefined ? [linearBrief] : []), `Then call mcp__auto-claude__${READY_TOOL}. It checks the files and starts the run: auto-claude builds each milestone in a worktree, pushes it, opens a PR, waits for CI, merges when green and deploys.`, @@ -146,6 +159,7 @@ export const POLICY = [ '- Decide every choice yourself: the recommended option, or the safest when it is plainly wrong; write it with its reason under ## Decisions in .claude/auto-plan.md.', '- Starting milestones, pushing, PRs, merging and deploying are auto-claude\'s: never run gh pr merge, a release or a deploy yourself, and never ask the person to run /auto-claude, /goal or merge anything. Commit your milestone on its goal/* branch.', '- When you are waiting on something (CI, the next milestone, a merge), just end the turn: auto-claude carries on by itself and prompts you when there is work. Never end a turn asking the person to check status.', + '- Waiting on compute (training, a cache build, a job you launched) is not a blocker and never a reason to defer: inside a milestone call mcp__auto-claude__wait_until with a shell command that exits 0 once it is done; in goals.md write `:: after: ` on the line. auto-claude polls it and resumes the work itself.', '- auto-claude sets each milestone as the goal itself: never call ProposeGoal or /goal during the run (a session goal would hold the loop).', '- An irreversible step (deleting data, credentials, payment, a new public release) is never taken: call mcp__auto-claude__defer_goal with the reason instead.', '- Linear issues named at the start of a milestone ("[AX-12] ...") are the driver\'s to move and comment on: read them, never change their state or comment on them, and never create Linear issues during a run (a new issue in the project becomes a milestone of this run).', diff --git a/auto-claude/hooks/pure.test.ts b/auto-claude/hooks/pure.test.ts index 2504b0d..e1c0bcc 100644 --- a/auto-claude/hooks/pure.test.ts +++ b/auto-claude/hooks/pure.test.ts @@ -7,8 +7,8 @@ import { checkBash, checkPath, deploysSomething, uncommitted, waitsLong } from ' import { guardRefusal, maxBranchSeq } from './loop' import type { ShipState } from '../types' import { deployFixLine, deployOf, finalCheckOf, fixLine, kickoffText, POLICY, validatePlan } from './plan' -import { markItem, nextPending, parseQueue, slugify, summarize } from './queue' -import { capReached, causeOf, ciAction, httpsOf, idleDue, inFlight, isWaiting, mergeable, noRequired, onlyCancelled, parseChecks, prNumber, progressSig, readLog, repoSlug, shipRemedy, startBase, unfinished, MAX_AHEAD, wakeDue } from './ship' +import { markItem, nextPending, parseDuration, parseQueue, requeueItem, slugify, summarize, waitingItems } from './queue' +import { capReached, onlyWaiting, causeOf, ciAction, httpsOf, idleDue, inFlight, isWaiting, mergeable, noRequired, onlyCancelled, parseChecks, prNumber, progressSig, readLog, repoSlug, shipRemedy, startBase, unfinished, MAX_AHEAD, wakeDue } from './ship' const PLAN = '## Goal\nadd and sub work\n\nFinal check: `python3 -m unittest -q`\nDeploy: `bash scripts/deploy.sh`\nSmoke check: `curl -fsS http://localhost:8080/health`\nCodeQL: python\n\n## Milestones\n1. add\n2. sub\n\n## Decisions\n' const QUEUE = '- [ ] add() returns a+b :: check: python3 -m unittest -q test_add\n- [ ] sub() returns a-b :: check: python3 -m unittest -q test_sub\n' @@ -463,3 +463,64 @@ test('a comment written before the run started is not a change, whichever issues const fresh = { 'A-29': [{ body: 'new ask', createdAt: '2026-11-13T15:05:00Z' }] } expect(planSync(issues, fresh, seen, new Set(), new Set()).append.map(i => i.id)).toEqual(['A-29']) }) + +describe('waits (:: after:)', () => { + const WAITING = '- [ ] record the cache :: check: pytest -q :: after: grep -q "CACHE DONE" ~/data/c.log :: alive: tmux has-session -t c :: within: 12h\n- [ ] b :: check: y\n' + + test('a waiting line parses its after, alive and within; the condition and check stay as they were', () => { + expect(parseQueue(WAITING)[0]).toEqual({ line: 0, mark: ' ', condition: 'record the cache', check: 'pytest -q', after: 'grep -q "CACHE DONE" ~/data/c.log', alive: 'tmux has-session -t c', withinMs: 12 * 3_600_000 }) + expect(parseQueue('- [ ] a :: check: x :: after: test -f y :: base: goal/03-a ')[0]).toEqual({ line: 0, mark: ' ', condition: 'a', check: 'x', after: 'test -f y', base: 'goal/03-a', note: 'waiting' }) + expect(parseDuration('90m')).toBe(90 * MIN) + expect(parseDuration('2d')).toBe(2 * 86_400_000) + expect(parseDuration('soon')).toBeUndefined() + }) + + test('a waiting line starts only once ready; by default never', () => { + const items = parseQueue(WAITING) + expect(nextPending(items)?.condition).toBe('b') + expect(nextPending(items, i => i.after === undefined || i.condition === 'record the cache')?.condition).toBe('record the cache') + expect(waitingItems(items, i => i.after === undefined).map(i => i.condition)).toEqual(['record the cache']) + // A waiting line with a base goes first once ready, like a fix goal. + const based = parseQueue('- [ ] b :: check: y\n- [ ] a :: check: x :: after: true :: base: goal/03-a\n') + expect(nextPending(based)?.condition).toBe('b') + expect(nextPending(based, () => true)?.condition).toBe('a') + }) + + test('requeueItem puts a line back to wait on the branch built so far; markItem still finds it', () => { + const text = requeueItem('- [ ] a :: check: x\n- [ ] b :: check: y\n', 'a', { after: 'grep -q DONE\n log', alive: 'tmux has-session -t t', within: '8h' }, 'goal/03-a', 'waiting since now: cache -->') + const item = parseQueue(text)[0]! + expect(item).toMatchObject({ mark: ' ', condition: 'a', check: 'x', after: 'grep -q DONE log', alive: 'tmux has-session -t t', withinMs: 8 * 3_600_000, base: 'goal/03-a' }) + expect(item.note).toContain('cache - ->') + expect(parseQueue(markItem(text, 'a', 'x', 'done'))[0]).toMatchObject({ mark: 'x', condition: 'a' }) + }) + + test('a prose start condition is refused at planning; a line put back to wait may keep its base', () => { + const prose = validatePlan(PLAN, '- [ ] (start only after ~/data/c.log shows "DONE") record it :: check: x\n- [ ] b :: check: y\n', true) + expect(!prose.ok && prose.errors.join(' ')).toContain(':: after:') + expect(validatePlan(PLAN, '- [ ] (start only after ~/data/c.log shows "DONE") record it :: check: x :: after: grep -q DONE ~/data/c.log\n- [ ] b :: check: y\n', true).ok).toBe(true) + expect(validatePlan(PLAN, '- [ ] a :: check: x :: after: test -f y :: base: goal/03-a\n- [ ] b :: check: y\n', false).ok).toBe(true) + expect(validatePlan(PLAN, '- [ ] a :: check: x :: base: goal/03-a\n- [ ] b :: check: y\n', false).ok).toBe(false) + // The prefixes measured in ArtifactNet; an end state that only mentions an order is not a wait. + expect(validatePlan(PLAN, '- [ ] (round 2; GPU parts start only when no other heavy job holds vela) x :: check: x\n- [ ] b :: check: y\n', true).ok).toBe(false) + expect(validatePlan(PLAN, '- [ ] the server starts only after migrations finish :: check: x\n- [ ] b :: check: y\n', true).ok).toBe(true) + }) + + test('only waiting is not idle, and waiting time is left out of the time cap up to its own cap', () => { + const r = { started: 2, finished: 2, ships: [], waitCount: 1 } + expect(onlyWaiting(r)).toBe(true) + expect(isWaiting(r, 0)).toBe(true) + expect(onlyWaiting({ ...r, started: 3 })).toBe(false) + expect(onlyWaiting({ ...r, waitCount: 0 })).toBe(false) + const caps = { maxRounds: 3, maxMerges: 20, maxHours: 24, maxWaitHours: 72 } + const H = 3_600_000 + expect(capReached({ merges: 0, startedAt: 0, waitedMs: 10 * H }, caps, 30 * H)).toBeUndefined() + expect(capReached({ merges: 0, startedAt: 0, waitedMs: 10 * H }, caps, 34 * H)).toContain('not counting 10 h of waiting') + expect(capReached({ merges: 0, startedAt: 0, waitedMs: 200 * H }, caps, 96 * H)).toContain('time cap') + }) + + test('every text that steers the model names wait_until for compute, not defer', () => { + expect(POLICY).toContain('wait_until') + expect(guardRefusal({ tool: 'Monitor' }, null, true)).toContain('mcp__auto-claude__wait_until') + expect(kickoffText('', 'origin/main')).toContain(':: after:') + }) +}) diff --git a/auto-claude/hooks/queue.ts b/auto-claude/hooks/queue.ts index 4c9270c..204473d 100644 --- a/auto-claude/hooks/queue.ts +++ b/auto-claude/hooks/queue.ts @@ -7,6 +7,16 @@ // - [ ] fix add() :: check: pytest -q tests/test_counter.py // - [x] fix add() :: check: pytest -q // - [ ] make CI green on PR #4 :: check: gh pr checks 4 :: base: goal/03-x (worktree starts from that ref) +// +// A line that waits on something with its own clock (a training run, a cache +// build, a file another milestone lands) names it as shell commands after its +// check, in this order, each optional, `base` always last: +// +// - [ ] record the M21 cache :: check: pytest -q :: after: grep -q DONE ~/data/cache.log :: alive: tmux has-session -t cache :: within: 12h +// +// `after` must exit 0 before the line starts; the driver polls it. `alive` +// failing says the awaited job died; `within` is how long the wait may take +// before the model is asked to look (`90m`, `12h`, `2d`). export type QueueMark = ' ' | 'x' | '!' @@ -17,6 +27,10 @@ export type QueueItem = { check?: string // The ref the goal's worktree starts from, over .claude/goal-loop.base. base?: string + // A wait: the line starts only once `after` exits 0. + after?: string + alive?: string + withinMs?: number note?: string } @@ -24,6 +38,26 @@ const ITEM = /^\s*[-*] \[([ xX!])\] (.*)$/ const NOTE = /\s*\s*$/ const CHECK = ' :: check:' const BASE = /\s+:: base:\s*(\S+)\s*$/ +const WITHIN = /\s+:: within:\s*(\S+)\s*$/ +const ALIVE = ' :: alive:' +const AFTER = ' :: after:' + +// `90m`, `12h`, `2d`, `1.5h` in milliseconds; undefined for anything else. +export function parseDuration(text: string): number | undefined { + const m = text.trim().match(/^(\d+(?:\.\d+)?)\s*([mhd])$/) + if (m === null) return undefined + const unit = m[2] === 'm' ? 60_000 : m[2] === 'h' ? 3_600_000 : 86_400_000 + const ms = Number(m[1]) * unit + return ms > 0 ? ms : undefined +} + +// Takes the last ` :: :` field off the end of a line body. +function takeLast(body: string, marker: string): { body: string; value?: string } { + const at = body.lastIndexOf(marker) + if (at < 0) return { body } + const value = body.slice(at + marker.length).trim() + return { body: body.slice(0, at), value: value === '' ? undefined : value } +} export function parseQueue(text: string): QueueItem[] { const items: QueueItem[] = [] @@ -44,20 +78,49 @@ export function parseQueue(text: string): QueueItem[] { base = baseMatch[1] body = body.slice(0, baseMatch.index) } + let withinMs: number | undefined + const withinMatch = body.match(WITHIN) + if (withinMatch !== null) { + withinMs = parseDuration(withinMatch[1] ?? '') + body = body.slice(0, withinMatch.index) + } + const alive = takeLast(body, ALIVE) + body = alive.body + const after = takeLast(body, AFTER) + body = after.body const at = body.indexOf(CHECK) const condition = (at < 0 ? body : body.slice(0, at)).trim() const check = at < 0 ? undefined : body.slice(at + CHECK.length).trim() || undefined if (condition === '') return - items.push({ line, mark, condition, check, ...(base !== undefined ? { base } : {}), note }) + items.push({ + line, + mark, + condition, + check, + ...(after.value !== undefined ? { after: after.value } : {}), + ...(alive.value !== undefined ? { alive: alive.value } : {}), + ...(withinMs !== undefined ? { withinMs } : {}), + ...(base !== undefined ? { base } : {}), + note, + }) }) return items } -// A line built on another branch (`:: base:`, a driver's fix goal) goes first: -// it repairs work already shipped, which later lines may build on. -export function nextPending(items: readonly QueueItem[]): QueueItem | undefined { - return items.find(item => item.mark === ' ' && item.base !== undefined) ?? items.find(item => item.mark === ' ') +// The next line to build. A line built on another branch (`:: base:`, a +// driver's fix goal) goes first: it repairs work already shipped, which later +// lines may build on. `ready` says whether a waiting line's `after` passed; +// by default a waiting line is never ready, so a caller that does not poll +// waits never starts one early. +export function nextPending(items: readonly QueueItem[], ready: (item: QueueItem) => boolean = item => item.after === undefined): QueueItem | undefined { + const open = items.filter(item => item.mark === ' ' && ready(item)) + return open.find(item => item.base !== undefined) ?? open[0] +} + +// Pending lines still waiting on their `after`. +export function waitingItems(items: readonly QueueItem[], ready: (item: QueueItem) => boolean): QueueItem[] { + return items.filter(item => item.mark === ' ' && item.after !== undefined && !ready(item)) } // Marks the first pending line whose condition matches, so a person editing @@ -69,7 +132,29 @@ export function markItem(text: string, condition: string, mark: 'x' | '!', note: const raw = lines[target.line] ?? '' const withoutNote = raw.replace(NOTE, '') const marked = withoutNote.replace(/\[[ xX!]\]/, `[${mark}]`) - lines[target.line] = `${marked} ` + lines[target.line] = `${marked} ` + + return lines.join('\n') +} + +export type Wait = { after: string; alive?: string; within?: string } + +// Puts a pending line back to wait: it keeps its condition and check, waits on +// `after`, and starts again from `base` (the branch it was built on so far). +export function requeueItem(text: string, condition: string, wait: Wait, base: string, note: string): string { + const lines = text.split('\n') + const target = parseQueue(text).find(item => item.mark === ' ' && item.condition === condition) + if (target === undefined) return text + const raw = lines[target.line] ?? '' + const bullet = raw.match(/^(\s*[-*]) \[/)?.[1] ?? '-' + const fields = [ + target.check !== undefined ? `${CHECK} ${target.check}` : '', + `${AFTER} ${oneLine(wait.after)}`, + wait.alive !== undefined && wait.alive.trim() !== '' ? `${ALIVE} ${oneLine(wait.alive)}` : '', + wait.within !== undefined && wait.within.trim() !== '' ? ` :: within: ${wait.within.trim()}` : '', + ` :: base: ${base}`, + ].join('') + lines[target.line] = `${bullet} [ ] ${target.condition}${fields} ` return lines.join('\n') } @@ -90,3 +175,11 @@ export function slugify(condition: string): string { return slug === '' ? 'goal' : slug } + +function cleanNote(note: string): string { + return note.replace(/--!?>/g, '- ->').replace(/\n/g, ' ') +} + +function oneLine(text: string): string { + return text.replace(/\s*\n\s*/g, ' ').trim() +} diff --git a/auto-claude/hooks/register.ts b/auto-claude/hooks/register.ts index c2dd540..11ad2f5 100644 --- a/auto-claude/hooks/register.ts +++ b/auto-claude/hooks/register.ts @@ -1,6 +1,6 @@ import type { EngineInterface, Register, Timer } from 'claude-code' -import type { AutoRecord, ShipItem } from '../types' +import type { AutoRecord, ShipItem, WaitState } from '../types' import { deployFixLine, finalCheckOf, fixLine, kickoffText, parseQueue, planNudgeText, POLICY, READY_TOOL, replanText, validatePlan } from './plan' import { segment } from './segment' import { DEPLOY_FILE, parseDeploy, specOfPlan } from './deploy' @@ -8,9 +8,10 @@ import type { DeploySpec } from './deploy' import { deployMerged } from './deploy-run' import { bootstrapRepo } from './bootstrap' import { ensureLines, FILES, fetchRemote, LOCAL, out, readText, refOf, repoRoot, run, tell } from './io' -import { appendLog, blockCurrent, DEFER_TOOL, DEFER_TOOL_ID, deferCall, guardRefusal, LOOP_EXCLUDES, LOOP_POLICY, loopView, MIN_CHECKS_TO_DEFER, onStop, readLoop, resumeLoop, setOnFinish, startNext, stopLoop, watch, writeLoop } from './loop' -import { capReached, causeOf, ciAction, diagnose, httpsOf, idleDue, inFlight, isWaiting, LOOP_WAKES, MAX_IDLE_NUDGES, MAX_RED, mergeable, noRequired, onlyCancelled, parseChecks, prNumber, progressSig, readLog, repoSlug, shipRemedy, startBase, summary, unfinished, wakeDue } from './ship' -import { markItem, nextPending } from './queue' +import { appendLog, blockCurrent, DEFER_TOOL, DEFER_TOOL_ID, deferCall, WAIT_TOOL, WAIT_TOOL_ID, waitCall, guardRefusal, LOOP_EXCLUDES, LOOP_POLICY, loopView, MIN_CHECKS_TO_DEFER, onStop, readLoop, resumeLoop, setOnFinish, startNext, stopLoop, watch, writeLoop } from './loop' +import { capReached, causeOf, ciAction, diagnose, httpsOf, idleDue, inFlight, isWaiting, onlyWaiting, LOOP_WAKES, MAX_IDLE_NUDGES, MAX_RED, mergeable, noRequired, onlyCancelled, parseChecks, prNumber, progressSig, readLog, repoSlug, shipRemedy, startBase, summary, unfinished, wakeDue } from './ship' +import { markItem, nextPending, waitingItems } from './queue' +import type { QueueItem } from './queue' import { baseline, getIssue, isOpen, lineIssues, onHeld, onMerged, onStart, projectOf, readChanges, SYNC_MS } from './linear' import type { StallFacts } from './ship' import type { Caps } from './ship' @@ -54,6 +55,15 @@ const NO_CI_GRACE_MS = 90_000 const MAX_DEPLOY_FAILS = 3 // Milestones held in a row for one cause before the run stops (ship.ts causeOf). const MAX_HOLD_STREAK = 2 +// A waiting line's `:: after:` is run this often, each run cut off after +// WAIT_PROBE_MS; `:: alive:` failing this many polls in a row means the job +// it waits for died. +const WAIT_POLL_MS = 3 * 60_000 +const WAIT_PROBE_MS = 30_000 +const ALIVE_FAILS = 2 +// A wait with no `within` of its own still goes to the model after this long, +// so a job that died without an alive check is not waited on in silence. +const WAIT_ASK_MS = 12 * 3_600_000 let ticker: Timer | undefined let ticking = false @@ -154,7 +164,8 @@ async function show($: EngineInterface, r: AutoRecord | undefined) { done = items.filter(i => i.mark !== ' ').length } const pr = inFlight(r.ships)?.pr - await segment($, `auto·R${r.round}·${r.phase === 'plan' ? 'plan' : `M${done}/${total}`}${pr !== undefined ? `·PR#${pr}` : ''}`) + const waiting = (r.waitCount ?? 0) > 0 ? `·wait${r.waitCount}` : '' + await segment($, `auto·R${r.round}·${r.phase === 'plan' ? 'plan' : `M${done}/${total}`}${pr !== undefined ? `·PR#${pr}` : ''}${waiting}`) } // The repository of the session's directory, or of its nearest ancestor that @@ -198,7 +209,7 @@ async function startWanted($: EngineInterface, r: AutoRecord, base: string, stac if ((await readLoop($, r.root))?.current !== undefined) return const tip = stacked ? (await run($, ['git', '-C', r.root, 'rev-parse', '--verify', '--quiet', `${base}^{commit}`], r.root, 15_000)).stdout.trim() : '' if (stacked && tip === '') return - const current = await startNext($, r.root, base, r.remote) + const current = await startNext($, r.root, base, r.remote, readyOf(r)) if (current !== undefined && stacked) await writeRecord($, { ...r, onto: { ...r.onto, [current.branch]: tip } }) } @@ -595,12 +606,17 @@ async function treat($: EngineInterface, r: AutoRecord, item: ShipItem, now: num // keeps building through the CI (ship.ts startBase). async function nextOrFinish($: EngineInterface, r: AutoRecord, caps: Caps): Promise { if (r.phase !== 'run') return - const next = nextPending(parseQueue((await readText($, FILES(r.root).queue)) ?? '')) + const items = parseQueue((await readText($, FILES(r.root).queue)) ?? '') + const next = nextPending(items, readyOf(r)) if (next !== undefined) { const at = startBase(r, next, refOf(r)) if (at !== undefined) await startWanted($, r, at.base, at.stacked) return } + // Lines still waiting on their `:: after:`: the run is not done, and no + // round is spent on a final check that cannot pass yet. pollWaits starts + // them when what they wait for is done. + if (waitingItems(items, readyOf(r)).length > 0) return const building = (await readLoop($, r.root))?.current !== undefined if (unfinished(r.ships).length === 0 && !building && r.started <= r.finished) { // The run is not done while Linear has changes it has not queued. @@ -611,6 +627,97 @@ async function nextOrFinish($: EngineInterface, r: AutoRecord, caps: Caps): Prom } } +// A pending line may start: it waits on nothing, or its `:: after:` passed. +function readyOf(r: Pick): (item: QueueItem) => boolean { + return item => item.after === undefined || r.waits?.[item.condition]?.ok === true +} + +// Polls the `:: after:` of each waiting line (every WAIT_POLL_MS), and its +// `:: alive:`: the driver, not the model, watches a training run or a cache +// build, so a run never ends, or sits stopped, while compute it waits for is +// still running. A wait that looks dead (alive failing) or late (past its +// `within`) goes to the model once; past twice its `within` it is held. +async function pollWaits($: EngineInterface, r: AutoRecord): Promise { + if (r.phase !== 'run') return r + const now = await $.clock.now() + const building = (await readLoop($, r.root))?.current?.condition + const items = parseQueue((await readText($, FILES(r.root).queue)) ?? '').filter(i => i.mark === ' ' && i.after !== undefined && i.condition !== building) + const old = r.waits ?? {} + const waits: Record = {} + let held = 0 + let fetched = false + for (const item of items) { + const after = item.after! + let w: WaitState = old[item.condition] ?? { since: now } + if (w.ok !== true && now - (w.checkedAt ?? 0) >= WAIT_POLL_MS) { + // A wait on what reached the remote (`git cat-file -e origin/main:x`) sees it fetched. + if (!fetched && r.remote !== LOCAL && after.includes(`${r.remote}/`)) { + await fetchRemote($, r, r.root) + fetched = true + } + const ran = await run($, ['bash', '-lc', after], r.root, WAIT_PROBE_MS).catch(error => ({ exitCode: -1, stdout: '', stderr: String(error) })) + w = { ...w, checkedAt: now } + if (ran.exitCode === 0) { + w = { ...w, ok: true, out: undefined } + await tell($, r, `wait over after ${hours(now - w.since)}: ${item.condition.slice(0, 80)}; it starts next`) + } else { + w = { ...w, out: `\`${after.slice(0, 120)}\` exited ${ran.exitCode}${out(ran) !== '' ? `: ${out(ran).slice(0, 300)}` : ''}` } + if (item.alive !== undefined) { + const alive = await run($, ['bash', '-lc', item.alive], r.root, WAIT_PROBE_MS).catch(error => ({ exitCode: -1, stdout: '', stderr: String(error) })) + w = alive.exitCode === 0 + ? { ...w, aliveFails: 0, askedDead: false } + : { ...w, aliveFails: (w.aliveFails ?? 0) + 1, out: `alive \`${item.alive.slice(0, 120)}\` exited ${alive.exitCode}${out(alive) !== '' ? `: ${out(alive).slice(0, 200)}` : ''}; ${w.out}` } + } + const waited = now - w.since + if (item.withinMs !== undefined && waited > 2 * item.withinMs) { + const text = (await readText($, FILES(r.root).queue)) ?? '' + await $.fs.write(FILES(r.root).queue, markItem(text, item.condition, '!', `held ${new Date(now).toISOString()}: waited ${hours(waited)}, twice its within; ${w.out ?? ''}`)) + await tell($, r, `held: ${item.condition.slice(0, 80)} waited ${hours(waited)}, twice its within; ${(w.out ?? '').slice(0, 150)}; going on`) + held++ + continue + } + const dead = (w.aliveFails ?? 0) >= ALIVE_FAILS + const late = waited > (item.withinMs ?? WAIT_ASK_MS) + if (dead && w.askedDead !== true) { + if (await askWait($, r, item, w, 'dead')) w = { ...w, askedDead: true } + } else if (late && w.askedLate !== true) { + if (await askWait($, r, item, w, 'late')) w = { ...w, askedLate: true } + } + } + } + waits[item.condition] = w + } + const waitCount = Object.values(waits).filter(w => w.ok !== true).length + let next: AutoRecord = { ...r, waits, waitCount, held: r.held + held } + // Time with nothing to do but wait is left out of the time cap. + if (onlyWaiting(next)) next = { ...next, waitedMs: (r.waitedMs ?? 0) + (r.waitTickAt !== undefined ? now - r.waitTickAt : 0), waitTickAt: now } + else next = { ...next, waitTickAt: undefined } + return next +} + +const hours = (ms: number) => (ms >= 3_600_000 ? `${(ms / 3_600_000).toFixed(1)} h` : `${Math.round(ms / 60_000)} min`) + +// A wait that looks dead or late goes to the model, which can look at the job +// (logs, tmux, the GPU) and relaunch it or fix the line. Not while it builds a +// milestone: asked at a later poll instead. +async function askWait($: EngineInterface, r: AutoRecord, item: QueueItem, w: WaitState, why: 'dead' | 'late'): Promise { + if ((await readLoop($, r.root))?.current !== undefined) return false + const since = hours((await $.clock.now()) - w.since) + const text = [ + `[auto-claude] A wait looks ${why === 'dead' ? 'dead' : 'late'}: "${item.condition.slice(0, 160)}" has waited ${since} on \`${item.after ?? ''}\`.`, + why === 'dead' + ? `Its alive check \`${item.alive ?? ''}\` failed ${w.aliveFails ?? 0} polls in a row: the job it waits for may have died (a crash, a reboot).` + : item.withinMs !== undefined + ? `That is longer than its within (${hours(item.withinMs)}).` + : `No line waits this long unasked (${hours(WAIT_ASK_MS)}); it has no within of its own.`, + `Last poll: ${(w.out ?? '').slice(0, 600)}`, + 'Find out from here whether the job is still running (its log, tmux, the GPU). Relaunch it if it died, or fix the line\'s `:: after:` / `:: alive:` in .claude/goals.md if they test the wrong thing. Do not wait for it and do not defer it: end the turn, and auto-claude keeps polling. Note what you found under ## Decisions in .claude/auto-plan.md.', + ].join('\n') + await tell($, r, `${why === 'dead' ? 'a waited-on job looks dead' : 'a wait is late'} (${since}): ${item.condition.slice(0, 80)}; the model was asked to look`) + $.prompt.submit({ text }).catch(() => {}) + return true +} + async function tick($: EngineInterface, caps: Caps, pollMs: number): Promise { if (ticking || activeRoot === undefined) return ticking = true @@ -647,6 +754,7 @@ async function tick($: EngineInterface, caps: Caps, pollMs: number): Promise 0) r = { ...r, linear: await onStart($, r.linear, ids) } } else if (e.event === 'met' && e.goal !== undefined && e.branch !== undefined) { @@ -706,6 +825,15 @@ async function readEvents($: EngineInterface, r: AutoRecord): Promise 0) r = { ...r, linear: await onHeld($, r.linear, ids, `${e.event}${e.reason !== undefined ? `: ${e.reason}` : ''}`) } + } else if (e.event === 'waiting') { + // Back in the queue, waiting on its command: finished as a build, not held. + r = { ...r, finished: r.finished + 1 } + const sha = e.branch === undefined ? undefined : r.onto?.[e.branch] + if (e.goal !== undefined && e.branch !== undefined && sha !== undefined) { + const { [e.branch]: _left, ...rest } = r.onto ?? {} + r = { ...r, onto: rest, waitOnto: { ...r.waitOnto, [e.goal]: sha } } + } + await tell($, r, `waiting: ${(e.goal ?? '').slice(0, 80)}${e.reason !== undefined ? ` (${e.reason.slice(0, 100)})` : ''}; auto-claude polls it and goes on with other milestones`) } else if (e.event === 'queue-empty') { r = { ...r, queueEmpty: true } } @@ -933,6 +1061,20 @@ async function stallLine($: EngineInterface, r: AutoRecord): Promise { + const now = await $.clock.now() + return queue + .filter(i => i.mark === ' ' && i.after !== undefined) + .map(i => { + const w = r.waits?.[i.condition] + if (w === undefined) return `- waiting: ${i.condition.slice(0, 60)} on \`${i.after!.slice(0, 80)}\` (not polled yet)` + if (w.ok === true) return `- wait over: ${i.condition.slice(0, 60)} (starts next)` + const checked = w.checkedAt !== undefined ? `, last poll ${Math.round((now - w.checkedAt) / 60_000)} min ago` : '' + return `- waiting ${hours(now - w.since)}: ${i.condition.slice(0, 60)} on \`${i.after!.slice(0, 80)}\`${checked}${(w.aliveFails ?? 0) > 0 ? `, alive failing ${w.aliveFails}x` : ''}` + }) +} + async function statusText($: EngineInterface): Promise { const root = activeRoot ?? (await sessionRoot($)).root const r = root === undefined ? undefined : await readRecord($, root) @@ -950,6 +1092,7 @@ async function statusText($: EngineInterface): Promise { deploys, `queue: ${queue.filter(i => i.mark === 'x').length} done, ${queue.filter(i => i.mark === '!').length} blocked, ${queue.filter(i => i.mark === ' ').length} pending · ${summary(r.merges, r.held, r.ships)}`, current !== undefined ? `building: ${current.condition.slice(0, 80)} (${current.branch})` : undefined, + ...(await waitLines($, r, queue)), ...r.ships.filter(s => s.state !== 'merged').map(s => `- ${s.state}${s.pr !== undefined ? ` PR #${s.pr}` : ''}: ${s.goal.slice(0, 70)}${s.note ? ` (${s.note.slice(0, 80)})` : ''}`), r.linear !== undefined ? `linear: project ${r.linear.project} · ${Object.keys(r.linear.seen).length} issue(s) covered · last sync ${r.linear.syncedAt !== undefined ? `${Math.round(((await $.clock.now()) - r.linear.syncedAt) / 60_000)} min ago` : 'never'}${r.linear.error !== undefined ? ` · error: ${r.linear.error}` : ''}` @@ -975,7 +1118,7 @@ async function recoverLoop($: EngineInterface, r: AutoRecord): Promise { continue } if (e.event === 'start' && e.goal !== undefined && e.branch !== undefined) open = { goal: e.goal, worktree: e.worktree, branch: e.branch } - else if ((e.event === 'met' || e.event === 'blocked' || e.event === 'deferred') && e.goal === open?.goal) open = undefined + else if ((e.event === 'met' || e.event === 'blocked' || e.event === 'deferred' || e.event === 'waiting') && e.goal === open?.goal) open = undefined } if (open === undefined || open.worktree === undefined) { await writeLoop($, { repoRoot: r.root, seq: 0, retries: 0 }) @@ -994,6 +1137,7 @@ export const register: Register = (on, options) => { maxRounds: Number(options.maxRounds ?? 3), maxMerges: Number(options.maxMerges ?? 20), maxHours: Number(options.maxHours ?? 24), + maxWaitHours: Number(options.maxWaitHours ?? 72), } const pollMs = Math.max(Number(options.ciPollSec ?? 30), 5) * 1000 const ciMode = typeof options.ciMode === 'string' ? options.ciMode : 'auto' @@ -1009,10 +1153,25 @@ export const register: Register = (on, options) => { }) await $.tool.register({ name: DEFER_TOOL, - description: `auto-claude run only: give up on the current milestone because of an external blocker (credentials, a human decision, missing access), so it is marked blocked and the run moves on. Allowed only after at least ${MIN_CHECKS_TO_DEFER} failed checks; stop right after calling it.`, + description: `auto-claude run only: give up on the current milestone because of an external blocker (credentials, a human decision, missing access), so it is marked blocked and the run moves on. Allowed only after at least ${MIN_CHECKS_TO_DEFER} failed checks; stop right after calling it. Waiting on compute (training, a cache build) is not a blocker: use ${WAIT_TOOL_ID}.`, inputSchema: { type: 'object', properties: { reason: { type: 'string', description: 'The external blocker, in one or two sentences.' } }, required: ['reason'] }, isDeferred: false, }) + await $.tool.register({ + name: WAIT_TOOL, + description: 'auto-claude run only: the current milestone needs a job with its own clock to finish first (a training run, a cache build, a file another milestone lands). Commit, call this with a shell command that exits 0 once it is done, then end the turn. auto-claude polls it every few minutes, builds other milestones meanwhile, and restarts this one from its branch when it passes.', + inputSchema: { + type: 'object', + properties: { + until: { type: 'string', description: 'Shell command, run from the repository root (not the worktree), that exits 0 once what you wait for is done, e.g. grep -q "CACHE DONE" ~/data/m21/cache.log or git cat-file -e origin/main:docs/m22/final.json.' }, + alive: { type: 'string', description: 'Optional shell command that exits 0 while the awaited job still runs, e.g. tmux has-session -t m21cache. Failing twice in a row means it died, and you are asked to look.' }, + within: { type: 'string', description: 'Optional: how long it should take at most (90m, 12h, 2d). Past it you are asked to look; past twice it the milestone is held.' }, + reason: { type: 'string', description: 'What it waits for, in one sentence.' }, + }, + required: ['until', 'reason'], + }, + isDeferred: false, + }) // A fresh load knows nothing of the turns before it: quiet counts from now. Object.assign(watch, { turnActive: false, lastTurnEndAt: undefined, judging: false, retryUntil: 0, loadedAt: await $.clock.now() }) tickNow = () => tick($, caps, pollMs) @@ -1079,6 +1238,8 @@ export const register: Register = (on, options) => { // @ts-expect-error TS2589 in claude-code's Matcher> on('tool.call', { tool: DEFER_TOOL_ID }, async ($, e) => deferCall($, (e as unknown as { reason?: unknown }).reason)) + on('tool.call', { tool: WAIT_TOOL_ID }, async ($, e) => waitCall($, e as unknown as Record)) + on('tool.call', { tool: READY_TOOL_ID }, async ($): Promise<{ result: string }> => { const root = activeRoot const r = root === undefined ? undefined : await readRecord($, root) diff --git a/auto-claude/hooks/ship.ts b/auto-claude/hooks/ship.ts index cfc9f6c..8cc7380 100644 --- a/auto-claude/hooks/ship.ts +++ b/auto-claude/hooks/ship.ts @@ -117,14 +117,26 @@ export function summary(merges: number, held: number, ships: readonly ShipItem[] return `${merges} merged, ${held} held${prs.length > 0 ? ` (PRs ${prs.join(' ')})` : ''}` } -export type Caps = { maxRounds: number; maxMerges: number; maxHours: number } +export type Caps = { maxRounds: number; maxMerges: number; maxHours: number; maxWaitHours?: number } -export function capReached(r: { merges: number; startedAt: number }, caps: Caps, now: number): string | undefined { +// The time cap counts the run's working time: time spent with nothing to do +// but wait on a line's `:: after:` (a training run) is left out, up to +// maxWaitHours of it, so a run waiting on a day-long job is not cut off. +export function capReached(r: { merges: number; startedAt: number; waitedMs?: number }, caps: Caps, now: number): string | undefined { if (r.merges >= caps.maxMerges) return `the merge cap (${caps.maxMerges}) was reached` - if (now - r.startedAt >= caps.maxHours * 3600_000) return `the time cap (${caps.maxHours} h) was reached` + const waited = Math.min(r.waitedMs ?? 0, (caps.maxWaitHours ?? 0) * 3600_000) + if (now - r.startedAt - waited >= caps.maxHours * 3600_000) { + return `the time cap (${caps.maxHours} h${waited > 0 ? `, not counting ${Math.round(waited / 3600_000)} h of waiting` : ''}) was reached` + } return undefined } +// Only waits are left: nothing is being built or shipped, and at least one +// pending line waits on its `:: after:`. The run is waiting, not idle. +export function onlyWaiting(r: Pick): boolean { + return (r.waitCount ?? 0) > 0 && r.started <= r.finished && unfinished(r.ships).length === 0 +} + // Why a run that is on is not moving, with its evidence; undefined when it // moves or nothing points to a stall. Worst first: an orphaned owner moves // nothing at all. @@ -321,10 +333,12 @@ export function startBase(r: Pick, next: { base? } // Waiting on something that has its own clock is not idle: the loop building a -// goal (while the session works on it), or a PR's CI pending with nothing -// wrong, for up to CI_STUCK_MS. -export function isWaiting(r: Pick, now: number, quietMs?: number): boolean { +// goal (while the session works on it), a PR's CI pending with nothing +// wrong, for up to CI_STUCK_MS, or queue lines waiting on their `:: after:` +// with nothing else to do (the driver polls them; see register.ts pollWaits). +export function isWaiting(r: Pick, now: number, quietMs?: number): boolean { if (r.started > r.finished) return quietMs === undefined || quietMs < LOOP_QUIET_MS + if (onlyWaiting(r)) return true const item = inFlight(r.ships) return item?.state === 'ci' && item.note === undefined && ciStuck(r, now) === undefined } diff --git a/auto-claude/types/index.d.ts b/auto-claude/types/index.d.ts index 75c125f..df62835 100644 --- a/auto-claude/types/index.d.ts +++ b/auto-claude/types/index.d.ts @@ -118,6 +118,31 @@ export type AutoRecord = { idleNudgedAt?: number // Linear sync; unset when the repository maps to no project. linear?: LinearSync + // Queue lines waiting on their `:: after:` command, by condition (register.ts pollWaits). + waits?: Record + // How many pending lines wait, as of the last tick: while only waits are left the run is not idle. + waitCount?: number + // Time spent with nothing to do but wait, left out of the time cap; and the tick it was last counted at. + waitedMs?: number + waitTickAt?: number + // A stacked milestone put back to wait keeps the commit it was stacked on, + // by condition, so the branch it restarts on is rebased off its parent too. + waitOnto?: Record +} + +// One waiting queue line, as the driver polls it. +export type WaitState = { + since: number + checkedAt?: number + // `after` exited 0: the line may start. + ok?: boolean + // The last output of a failing `after` or `alive`, for the model and status. + out?: string + // `alive` failures in a row. + aliveFails?: number + // The model was asked to look (the job looks dead, or `within` passed), once each. + askedDead?: boolean + askedLate?: boolean } // A goal of the loop: one queue line being built in its own worktree. @@ -131,6 +156,8 @@ export type Goal = { recentReasons: string[] lastReason?: string deferPending?: string + // wait_until: applied at the next Stop, like deferPending. + waitPending?: { after: string; alive?: string; within?: string; reason: string } } // What the guards need while a goal runs in this session. diff --git a/shared/queue.ts b/shared/queue.ts index 7c2c3b9..76bb684 100644 --- a/shared/queue.ts +++ b/shared/queue.ts @@ -6,6 +6,16 @@ // - [ ] fix add() :: check: pytest -q tests/test_counter.py // - [x] fix add() :: check: pytest -q // - [ ] make CI green on PR #4 :: check: gh pr checks 4 :: base: goal/03-x (worktree starts from that ref) +// +// A line that waits on something with its own clock (a training run, a cache +// build, a file another milestone lands) names it as shell commands after its +// check, in this order, each optional, `base` always last: +// +// - [ ] record the M21 cache :: check: pytest -q :: after: grep -q DONE ~/data/cache.log :: alive: tmux has-session -t cache :: within: 12h +// +// `after` must exit 0 before the line starts; the driver polls it. `alive` +// failing says the awaited job died; `within` is how long the wait may take +// before the model is asked to look (`90m`, `12h`, `2d`). export type QueueMark = ' ' | 'x' | '!' @@ -16,6 +26,10 @@ export type QueueItem = { check?: string // The ref the goal's worktree starts from, over .claude/goal-loop.base. base?: string + // A wait: the line starts only once `after` exits 0. + after?: string + alive?: string + withinMs?: number note?: string } @@ -23,6 +37,26 @@ const ITEM = /^\s*[-*] \[([ xX!])\] (.*)$/ const NOTE = /\s*\s*$/ const CHECK = ' :: check:' const BASE = /\s+:: base:\s*(\S+)\s*$/ +const WITHIN = /\s+:: within:\s*(\S+)\s*$/ +const ALIVE = ' :: alive:' +const AFTER = ' :: after:' + +// `90m`, `12h`, `2d`, `1.5h` in milliseconds; undefined for anything else. +export function parseDuration(text: string): number | undefined { + const m = text.trim().match(/^(\d+(?:\.\d+)?)\s*([mhd])$/) + if (m === null) return undefined + const unit = m[2] === 'm' ? 60_000 : m[2] === 'h' ? 3_600_000 : 86_400_000 + const ms = Number(m[1]) * unit + return ms > 0 ? ms : undefined +} + +// Takes the last ` :: :` field off the end of a line body. +function takeLast(body: string, marker: string): { body: string; value?: string } { + const at = body.lastIndexOf(marker) + if (at < 0) return { body } + const value = body.slice(at + marker.length).trim() + return { body: body.slice(0, at), value: value === '' ? undefined : value } +} export function parseQueue(text: string): QueueItem[] { const items: QueueItem[] = [] @@ -43,20 +77,49 @@ export function parseQueue(text: string): QueueItem[] { base = baseMatch[1] body = body.slice(0, baseMatch.index) } + let withinMs: number | undefined + const withinMatch = body.match(WITHIN) + if (withinMatch !== null) { + withinMs = parseDuration(withinMatch[1] ?? '') + body = body.slice(0, withinMatch.index) + } + const alive = takeLast(body, ALIVE) + body = alive.body + const after = takeLast(body, AFTER) + body = after.body const at = body.indexOf(CHECK) const condition = (at < 0 ? body : body.slice(0, at)).trim() const check = at < 0 ? undefined : body.slice(at + CHECK.length).trim() || undefined if (condition === '') return - items.push({ line, mark, condition, check, ...(base !== undefined ? { base } : {}), note }) + items.push({ + line, + mark, + condition, + check, + ...(after.value !== undefined ? { after: after.value } : {}), + ...(alive.value !== undefined ? { alive: alive.value } : {}), + ...(withinMs !== undefined ? { withinMs } : {}), + ...(base !== undefined ? { base } : {}), + note, + }) }) return items } -// A line built on another branch (`:: base:`, a driver's fix goal) goes first: -// it repairs work already shipped, which later lines may build on. -export function nextPending(items: readonly QueueItem[]): QueueItem | undefined { - return items.find(item => item.mark === ' ' && item.base !== undefined) ?? items.find(item => item.mark === ' ') +// The next line to build. A line built on another branch (`:: base:`, a +// driver's fix goal) goes first: it repairs work already shipped, which later +// lines may build on. `ready` says whether a waiting line's `after` passed; +// by default a waiting line is never ready, so a caller that does not poll +// waits never starts one early. +export function nextPending(items: readonly QueueItem[], ready: (item: QueueItem) => boolean = item => item.after === undefined): QueueItem | undefined { + const open = items.filter(item => item.mark === ' ' && ready(item)) + return open.find(item => item.base !== undefined) ?? open[0] +} + +// Pending lines still waiting on their `after`. +export function waitingItems(items: readonly QueueItem[], ready: (item: QueueItem) => boolean): QueueItem[] { + return items.filter(item => item.mark === ' ' && item.after !== undefined && !ready(item)) } // Marks the first pending line whose condition matches, so a person editing @@ -68,7 +131,29 @@ export function markItem(text: string, condition: string, mark: 'x' | '!', note: const raw = lines[target.line] ?? '' const withoutNote = raw.replace(NOTE, '') const marked = withoutNote.replace(/\[[ xX!]\]/, `[${mark}]`) - lines[target.line] = `${marked} ` + lines[target.line] = `${marked} ` + + return lines.join('\n') +} + +export type Wait = { after: string; alive?: string; within?: string } + +// Puts a pending line back to wait: it keeps its condition and check, waits on +// `after`, and starts again from `base` (the branch it was built on so far). +export function requeueItem(text: string, condition: string, wait: Wait, base: string, note: string): string { + const lines = text.split('\n') + const target = parseQueue(text).find(item => item.mark === ' ' && item.condition === condition) + if (target === undefined) return text + const raw = lines[target.line] ?? '' + const bullet = raw.match(/^(\s*[-*]) \[/)?.[1] ?? '-' + const fields = [ + target.check !== undefined ? `${CHECK} ${target.check}` : '', + `${AFTER} ${oneLine(wait.after)}`, + wait.alive !== undefined && wait.alive.trim() !== '' ? `${ALIVE} ${oneLine(wait.alive)}` : '', + wait.within !== undefined && wait.within.trim() !== '' ? ` :: within: ${wait.within.trim()}` : '', + ` :: base: ${base}`, + ].join('') + lines[target.line] = `${bullet} [ ] ${target.condition}${fields} ` return lines.join('\n') } @@ -89,3 +174,11 @@ export function slugify(condition: string): string { return slug === '' ? 'goal' : slug } + +function cleanNote(note: string): string { + return note.replace(/--!?>/g, '- ->').replace(/\n/g, ' ') +} + +function oneLine(text: string): string { + return text.replace(/\s*\n\s*/g, ' ').trim() +}