From 6b798177ab4bab2009b46480faf34b180279f8de Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 16:23:45 -0700 Subject: [PATCH 01/21] fix: reclaim delivered captures according to readability --- docs/decisions/0005-output-budget-protocol.md | 22 +- docs/decisions/0006-output-containment.md | 40 ++- .../0008-shell-capture-snapshot-authority.md | 8 +- docs/safety/hooks.md | 11 +- src/autoskillit/hooks/_capture/_delivery.py | 3 +- src/autoskillit/hooks/_capture/_ledger.py | 2 + .../hooks/_capture/_lifecycle_policy.py | 16 +- .../hooks/_capture/_lifecycle_record.py | 25 ++ src/autoskillit/hooks/_capture/_reconcile.py | 4 +- src/autoskillit/hooks/_capture/_runner.py | 8 +- .../hooks/_capture/_sweep_cursor.py | 20 +- .../hooks/_capture_lifecycle/_store.py | 15 +- tests/hooks/test_capture_artifacts.py | 134 ++++++- tests/hooks/test_capture_lifecycle.py | 4 +- .../hooks/test_capture_sustained_admission.py | 338 ++++++++++++++++++ tests/hooks/test_capture_verified_reader.py | 330 ++++++++++++++++- tests/hooks/test_shell_capture_conformance.py | 46 +-- 17 files changed, 932 insertions(+), 94 deletions(-) create mode 100644 tests/hooks/test_capture_sustained_admission.py diff --git a/docs/decisions/0005-output-budget-protocol.md b/docs/decisions/0005-output-budget-protocol.md index d900d65ba2..fe54b550ba 100644 --- a/docs/decisions/0005-output-budget-protocol.md +++ b/docs/decisions/0005-output-budget-protocol.md @@ -80,15 +80,19 @@ exact pull closure. It does not masquerade as the tool's normal operational fail | `MAX_MCP_OUTPUT_TOKENS = 50_000` | Keep Claude's independently defined setting separate. It has no shared source of truth with Codex result or history limits and does not control Claude Code's observed disk-persistence gate. Claude's native Bash spill behavior covers shell output on that backend. | The shell capture hook uses `shell_max_inline_bytes = 12_000` as the inline threshold: -commands whose combined output fits within that budget are inlined in full (artifact -retained for one hour after durable finalization); larger outputs are captured losslessly -to the same descriptor-owned artifact with a bounded head/tail slice and provenance -marker inlined. A per-artifact writer lease protects quiet live producers. Every valid -installed runner performs one bounded tail sweep after producer resources release, and -the cleanup-only `capture_lifecycle_hook.py` performs the same sweep at `SessionStart` -for interactive and headless sessions. Eligible artifacts are reclaimed on the next -enabled trusted trigger through identity-revalidated quarantine deletion. Cleanup -failure is fail-open and cannot replace the command's stdout, stderr, or exit status. +commands whose combined output fits within that budget are inlined in full; larger +outputs are captured losslessly to the descriptor-owned artifact with a bounded +head/tail slice and provenance marker inlined. Retention follows readability: after +delivery completes, captures with no outstanding bearer become eligible +immediately; a delivered capture with a +published reference remains readable until expiry; unsettled deliveries and failed +artifacts keep the one-hour hold. A per-artifact writer lease protects quiet live +producers. Every valid installed runner performs one bounded tail sweep after producer +resources release, and the cleanup-only `capture_lifecycle_hook.py` performs the same +sweep at `SessionStart` for interactive and headless sessions. Eligible artifacts are +reclaimed on the next enabled trusted trigger through identity-revalidated quarantine +deletion. Cleanup failure is fail-open and cannot replace the command's stdout, +stderr, or exit status. `small_file_max_bytes` was removed — it existed solely for the classifier's literal-small-JSONL exception, which is no longer needed. diff --git a/docs/decisions/0006-output-containment.md b/docs/decisions/0006-output-containment.md index 9c7acfc4e6..5f452b09f3 100644 --- a/docs/decisions/0006-output-containment.md +++ b/docs/decisions/0006-output-containment.md @@ -99,10 +99,24 @@ process-lifetime contract is defined against that host model in ADR-0008. The lifecycle store records `RESERVED`, `STAGED`, `PUBLISHED_WRITING`, `FINALIZED`, `FAILED`, `ABANDONED`, `DELETING`, `DELETED`, and `TAMPERED`. The immutable FINAL, reference, delivery, and reader semantics belong to -ADR-0008. A finalized or failed artifact becomes eligible one hour after its -terminal transition; an abandoned producer becomes eligible one hour after its -durable creation time. Eligibility is reclaimed only on the next enabled, -trusted trigger, not by a wall-clock scheduler. +ADR-0008. Retention follows readability: after delivery completes, captures +with no outstanding bearer become eligible immediately; a delivered capture +with a published reference remains readable until expiry; unsettled deliveries +and failed artifacts keep the one-hour hold. + +| FINALIZED record | Eligibility time | Why | +|---|---|---| +| Delivered with no outstanding bearer (`not_requested`, `unavailable`, `expired`, `revoked`) | Immediately after the delivered transition commits; its due timestamp is `manifest.finalized_at` | Inline bytes have already been delivered and no bearer can open the artifact | +| Delivered with a `published` reference | `manifest.reference_expiry` | The bearer remains readable until expiry | +| Any other FINALIZED state (undelivered, delivery `failed`/`unknown`, or reference `unknown`) | `manifest.retention_deadline` | Preserve the one-hour forensic hold | + +`FAILED` artifacts, including pre-FINAL failures, remain eligible one hour after +their terminal transition; abandoned producers remain eligible one hour after +durable creation time. A FINALIZED record with no outstanding bearer is not due +until delivery completes, even though its due timestamp is `manifest.finalized_at`. + +Eligibility is reclaimed only on the next enabled, trusted trigger, not by a +wall-clock scheduler. Admission participates in that trigger-owned model. When store-wide due debt reaches the assist threshold, the admitting runner performs one debt-scaled @@ -122,6 +136,19 @@ at least one installed reclamation owner retains a positive directory-scan budget. Removing either condition requires a new ownership decision; the read-only doctor must not become a scheduler implicitly. +The producer's own runner tail tries its just-delivered inline capture first, +before the cursor-rotated backlog, within the existing `RUNNER_TAIL_BUDGET`; +this moves the same required deletion work earlier without increasing total +deletion work or adding an owner or scheduler. + +Eligibility is distinct from access and deletion. Inline output has already +been delivered as in-memory bytes and grants no later artifact-opening +authority; `open_verified_capture()` requires a published bearer. A holder of a +delivered published reference may open it until expiry. Expiry prevents new +authorized opens but does not erase previously delivered bytes. An already-open +verified reader's shared lease prevents reclamation until it closes, after which +identity checks still have to pass before deletion. + Deletion is confined to the shared store's identity-revalidated quarantine transaction. Only lifecycle-recorded `shell_[0-9a-f]{16}.log` artifacts are eligible. Fresh records, live writers, nonmatching names, symlinks, FIFOs, @@ -215,5 +242,6 @@ are rejected in favor of the safe production default. - Complete Codex shell output is captured to a descriptor-anchored carrier. Oversized replay reports an opaque V2 reference or an unavailable status, never an authoritative pathname. -- One-hour lifecycle reclamation is owned by the installed runner tail and the - cleanup-only `SessionStart` hook, with bounded retry and quarantine recovery. +- Readability-derived lifecycle reclamation is owned by the installed runner + tail and the cleanup-only `SessionStart` hook, with bounded retry and + quarantine recovery; unsettled artifacts retain the one-hour hold. diff --git a/docs/decisions/0008-shell-capture-snapshot-authority.md b/docs/decisions/0008-shell-capture-snapshot-authority.md index e25d943871..ae3b346c6c 100644 --- a/docs/decisions/0008-shell-capture-snapshot-authority.md +++ b/docs/decisions/0008-shell-capture-snapshot-authority.md @@ -184,9 +184,11 @@ nonblocking exclusive lease and defers while the producer or any reader remains live. The existing runner-tail and cleanup-only `SessionStart` owners remain the only owners of the single `shell-captures` lifecycle resource. -References expire no later than the one-hour retention deadline. Expiry stops new -resolution, as does an explicit revoked state; cleanup removes an eligible carrier -only after leases release and descriptor-relative identity checks pass. +After the `delivered` transition commits, a capture with no outstanding bearer +becomes eligible for reclamation immediately. A published reference expires no +later than the one-hour retention deadline. Expiry stops new resolution, as does +an explicit revoked state; cleanup removes an eligible carrier only after producer +and reader leases release and descriptor-relative identity checks pass. ### Ledger, filesystem, and durability boundary diff --git a/docs/safety/hooks.md b/docs/safety/hooks.md index 2b3c150035..0bb0807af6 100644 --- a/docs/safety/hooks.md +++ b/docs/safety/hooks.md @@ -195,9 +195,9 @@ is implemented. | Publication | Oversized FINAL issues one bearer token and stores only its bound hash; publication verifies that tuple and can return published or unavailable but cannot mint another token | | Delivery | Checked write-all rejects invalid/no-progress writes and flushes hook stdout before `delivered`; this boundary does not prove host, UI, history, or model visibility | | Reader | `open_verified_capture()` authenticates the token, takes a shared lease, revalidates the descriptor, and exposes exact bounded reads without a path, descriptor, or write API | -| Retention | References expire no later than the one-hour retention deadline; active producer-exclusive and reader-shared leases block cleanup | +| Retention | After delivery completes, captures with no outstanding bearer become eligible immediately; a delivered capture with a published reference remains readable until expiry; unsettled deliveries and failed artifacts keep the one-hour hold; active producer-exclusive and reader-shared leases block cleanup | | Ownership | The producer transfers the retained carrier while preserving exclusive ownership through initial publication and delivery | -| Cleanup — installed runner | Every valid run/reject invocation performs one bounded sweep after producer resources release | +| Cleanup — installed runner | Every valid run/reject invocation performs one bounded sweep after producer resources release and prioritizes its just-delivered inline capture before the cursor-rotated backlog within the existing `RUNNER_TAIL_BUDGET` | | Cleanup — session lifecycle | `capture_lifecycle_hook.py` performs the same bounded sweep at `SessionStart` in both interactive and headless sessions | | Naming contract | `shell_[0-9a-f]{16}.log` — files not matching this pattern are never deleted | | Safety | Capture components reject symlinks; artifacts reject collisions, hardlinks, unsafe modes, and identity changes; a hostile same-UID process that ignores advisory locks is excluded | @@ -212,9 +212,10 @@ it with capped backoff. FINAL also carries independent reference `revoked`) and delivery (`not_attempted`, `attempting`, `delivered`, `failed`, `unknown`) states. Lost pre-delivery tokens become unavailable; an interrupted attempting delivery -becomes unknown and is never re-emitted. Finalized and failed records use their -terminal transition as the retention clock; abandoned records use their durable -creation time. +becomes unknown and is never re-emitted. FINALIZED records derive eligibility +from readability as described in the Retention row above; `FAILED` records keep +their one-hour hold from terminal transition and abandoned records keep their +one-hour hold from durable creation time. Eligibility is not a wall-clock scheduler. Deletion occurs on the next enabled, trusted installed runner-tail or cleanup-only `SessionStart` trigger. If hooks diff --git a/src/autoskillit/hooks/_capture/_delivery.py b/src/autoskillit/hooks/_capture/_delivery.py index b736e3a25c..7f24e4daf8 100644 --- a/src/autoskillit/hooks/_capture/_delivery.py +++ b/src/autoskillit/hooks/_capture/_delivery.py @@ -306,11 +306,12 @@ def _delivery_transition( raise lifecycle_error("capture delivery transition predecessor changed") if not _ledger.is_delivery_successor(expected, target, allow_same=False): raise lifecycle_error("capture delivery transition is not allowed") - return replace( + candidate = replace( record, delivery_status=target, revision=record.revision + 1, ) + return replace(candidate, next_attempt_at=_ledger.readable_until(candidate)) def transition_delivery( diff --git a/src/autoskillit/hooks/_capture/_ledger.py b/src/autoskillit/hooks/_capture/_ledger.py index 2752569f98..8697d8b499 100644 --- a/src/autoskillit/hooks/_capture/_ledger.py +++ b/src/autoskillit/hooks/_capture/_ledger.py @@ -43,6 +43,7 @@ CaptureTransitionCommittedError, adopted_orphan_record, legacy_record_from_dict, + readable_until, record_from_dict, record_to_dict, same_record, @@ -79,6 +80,7 @@ "is_retention_successor", "is_state_successor", "legacy_record_from_dict", + "readable_until", "record_from_dict", "record_to_dict", "same_record", diff --git a/src/autoskillit/hooks/_capture/_lifecycle_policy.py b/src/autoskillit/hooks/_capture/_lifecycle_policy.py index 3a8d95aca1..a80795cec1 100644 --- a/src/autoskillit/hooks/_capture/_lifecycle_policy.py +++ b/src/autoskillit/hooks/_capture/_lifecycle_policy.py @@ -16,6 +16,7 @@ "CaptureSnapshotStatus", "CaptureState", "CaptureStatus", + "REFERENCE_LIFETIME_SECONDS", "ReclaimKind", "STATE_RECLAIMABILITY", "StateReclaimabilityDef", @@ -264,10 +265,11 @@ def is_retention_successor( # Reclaimability declarations # --------------------------------------------------------------------------- -# The canonical sweep-grace constant. Formerly ``_RETENTION_SECONDS`` in -# ``_capture_lifecycle.py``; moved here so that the declaration and the -# enforcement share one value. +# Shared by the reclaimability declaration and lifecycle-store enforcement. SWEEP_GRACE_SECONDS: float = 3600.0 +REFERENCE_LIFETIME_SECONDS = 1800.0 +if SWEEP_GRACE_SECONDS < REFERENCE_LIFETIME_SECONDS: + raise AssertionError("capture retention must cover the replay-reference lifetime") class ReclaimKind(StrEnum): @@ -310,11 +312,9 @@ class StateReclaimabilityDef(NamedTuple): state=CaptureState.FINALIZED, kind=ReclaimKind.SWEEP_AFTER_GRACE, duration_seconds=SWEEP_GRACE_SECONDS, - # Coupling invariant: FINALIZED's sweep grace must never be set below - # the replay-reference window (_REFERENCE_LIFETIME_SECONDS = 1800s in - # _capture_lifecycle.py), or sweeps would delete artifacts whose issued - # references are still valid. - rationale="completed capture — reclaimable after retention window", + # Delivery sets the due time from _lifecycle_record.readable_until; + # undelivered and unsettled captures retain the forensic grace. + rationale="completed capture — reclaimable when delivered bytes are no longer readable", ), CaptureState.FAILED: StateReclaimabilityDef( state=CaptureState.FAILED, diff --git a/src/autoskillit/hooks/_capture/_lifecycle_record.py b/src/autoskillit/hooks/_capture/_lifecycle_record.py index b7045aa85e..163b0e0417 100644 --- a/src/autoskillit/hooks/_capture/_lifecycle_record.py +++ b/src/autoskillit/hooks/_capture/_lifecycle_record.py @@ -79,6 +79,7 @@ "is_retention_successor", "is_state_successor", "legacy_record_from_dict", + "readable_until", "record_from_dict", "record_to_dict", "same_record", @@ -138,6 +139,30 @@ def sha256(self) -> str: return self.manifest.sha256 if self.manifest is not None else "" +_NO_BEARER_REFERENCES = frozenset( + { + CaptureReferenceStatus.NOT_REQUESTED, + CaptureReferenceStatus.UNAVAILABLE, + CaptureReferenceStatus.EXPIRED, + CaptureReferenceStatus.REVOKED, + } +) + + +def readable_until(record: CaptureLifecycleRecord) -> float: + """Return the last authorized read time for a finalized capture.""" + if record.state is not CaptureState.FINALIZED or record.manifest is None: + raise LedgerCodecError("readability requires a finalized capture manifest") + manifest = record.manifest + if record.delivery_status is CaptureDeliveryStatus.DELIVERED: + if record.reference_status in _NO_BEARER_REFERENCES: + return manifest.finalized_at + if record.reference_status is CaptureReferenceStatus.PUBLISHED: + assert manifest.reference_expiry is not None + return manifest.reference_expiry + return manifest.retention_deadline + + def same_record( expected: CaptureLifecycleRecord, current: CaptureLifecycleRecord | None, diff --git a/src/autoskillit/hooks/_capture/_reconcile.py b/src/autoskillit/hooks/_capture/_reconcile.py index 43dd61f24e..b19649d930 100644 --- a/src/autoskillit/hooks/_capture/_reconcile.py +++ b/src/autoskillit/hooks/_capture/_reconcile.py @@ -102,6 +102,8 @@ def _failure_blocker(reason: CaptureFailureReason) -> CleanupBlocker: def reconcile_capture_store( project_cwd: str, budget: SweepBudgetSpec, + *, + priority_capture_id: str | None = None, ) -> CaptureCleanupOutcome: if not isinstance(project_cwd, str) or not project_cwd or type(budget) is not SweepBudgetSpec: return _failure_outcome(CleanupBlocker.FILESYSTEM_AUTHORITY) @@ -110,7 +112,7 @@ def reconcile_capture_store( with _authority.open_capture_lifecycle( project_cwd, create=False, lock_wait=lock_wait_for_budget(budget) ) as lifecycle: - return lifecycle.sweep(budget) + return lifecycle.sweep(budget, priority_capture_id=priority_capture_id) except _authority.CaptureStoreAbsentError: return CaptureCleanupOutcome(blocker=CleanupBlocker.STORE_ABSENT) except LockContended: diff --git a/src/autoskillit/hooks/_capture/_runner.py b/src/autoskillit/hooks/_capture/_runner.py index 09ceb91c2e..9579081676 100644 --- a/src/autoskillit/hooks/_capture/_runner.py +++ b/src/autoskillit/hooks/_capture/_runner.py @@ -570,7 +570,7 @@ def _emit_runner_tail_crash_diagnostic() -> None: ) -def _sweep_after_runner(requested_cwd: str) -> None: +def _sweep_after_runner(requested_cwd: str, capture_id: str) -> None: global _BYTE_PRESSURE_OBSERVED byte_pressure_observed = _BYTE_PRESSURE_OBSERVED _BYTE_PRESSURE_OBSERVED = False @@ -580,7 +580,9 @@ def _sweep_after_runner(requested_cwd: str) -> None: if byte_pressure_observed else _capture_reconcile.RUNNER_TAIL_BUDGET ) - outcome = _capture_reconcile.reconcile_capture_store(requested_cwd, budget) + outcome = _capture_reconcile.reconcile_capture_store( + requested_cwd, budget, priority_capture_id=capture_id + ) _capture_reconcile.emit_owner_diagnostic( outcome, owner=_RUNNER_TAIL_OWNER, write=sys.stderr.write ) @@ -620,5 +622,5 @@ def _main(argv: list[str] | None = None) -> int: ) ) finally: - _sweep_after_runner(request.cwd) + _sweep_after_runner(request.cwd, request.capture_id) return user_result diff --git a/src/autoskillit/hooks/_capture/_sweep_cursor.py b/src/autoskillit/hooks/_capture/_sweep_cursor.py index e92f7171b6..3effb0d6b0 100644 --- a/src/autoskillit/hooks/_capture/_sweep_cursor.py +++ b/src/autoskillit/hooks/_capture/_sweep_cursor.py @@ -6,7 +6,7 @@ import json import math import os -from collections.abc import Collection, Iterable +from collections.abc import Collection, Iterable, Mapping from dataclasses import dataclass from enum import StrEnum from typing import Protocol @@ -265,16 +265,28 @@ def bounded_due_keys( return due, complete, inspected, rebuild_key +def _priority_records( + records: Mapping[str, SweepRecord], priority_capture_id: str | None +) -> Iterable[SweepRecord]: + if priority_capture_id is not None and priority_capture_id in records: + yield records[priority_capture_id] + yield from ( + record for capture_id, record in records.items() if capture_id != priority_capture_id + ) + + def select_due_keys( store: _store_port.SweepStorePort, now: float, max_records: int, terminal_states: Collection[object], + *, + priority_capture_id: str | None = None, ) -> tuple[list[DueKey], bool, bool]: with store._locked(): records, compaction_epoch, _size = store._load_locked() due, complete, inspected, rebuild_key = bounded_due_keys( - records.values(), + _priority_records(records, priority_capture_id), now, terminal_states, max_records, @@ -307,8 +319,10 @@ def select_due_keys( if repaired: store._sweep_cursor_writes += 1 store._sweep_records_inspected += inspected + ordered = rotate_after(due, cursor.due_key) + ordered.sort(key=lambda key: key.capture_id != priority_capture_id) return ( - rotate_after(due, cursor.due_key), + ordered, complete, repaired or (repair_needed and bool(due)), ) diff --git a/src/autoskillit/hooks/_capture_lifecycle/_store.py b/src/autoskillit/hooks/_capture_lifecycle/_store.py index a6afa825db..cefb54c358 100644 --- a/src/autoskillit/hooks/_capture_lifecycle/_store.py +++ b/src/autoskillit/hooks/_capture_lifecycle/_store.py @@ -100,9 +100,7 @@ MAX_LEDGER_BYTES = _capture_ledger.MAX_LEDGER_BYTES # Single source of truth: the reclaimability declaration owns the sweep grace. _RETENTION_SECONDS = float(_capture_lifecycle_policy.SWEEP_GRACE_SECONDS) -_REFERENCE_LIFETIME_SECONDS = 1800.0 -if _RETENTION_SECONDS < _REFERENCE_LIFETIME_SECONDS: - raise AssertionError("capture retention must cover the replay-reference lifetime") +_REFERENCE_LIFETIME_SECONDS = _capture_lifecycle_policy.REFERENCE_LIFETIME_SECONDS _MAX_RETRY_SECONDS = 3600.0 _COMPACTION_THRESHOLD_BYTES = 31 * 1024 * 1024 // 8 _MAX_COMPACTION_BYTES = 4 * 1024 * 1024 @@ -683,12 +681,15 @@ def _due_keys( self, now: float, max_records: int, + *, + priority_capture_id: str | None = None, ) -> tuple[list[DueKey], bool, bool]: return _capture_sweep_cursor.select_due_keys( self, now, max_records, TERMINAL_STATES, + priority_capture_id=priority_capture_id, ) def _advance_sweep_cursor(self, due_key: DueKey) -> None: @@ -731,6 +732,8 @@ def _scan_and_adopt_orphans(self) -> _capture_orphan_scan.OrphanAdoptionOutcome: def sweep( self, budget: SweepBudgetSpec, + *, + priority_capture_id: str | None = None, ) -> CaptureCleanupOutcome: if type(budget) is not SweepBudgetSpec: raise CaptureLifecycleError("cleanup requires one SweepBudgetSpec") @@ -738,12 +741,16 @@ def sweep( self._sweep_started_monotonic = self._monotonic() self._sweep_records_inspected = self._sweep_replay_bytes = 0 self._sweep_transitions = self._sweep_cursor_writes = 0 + + def due_keys(now: float, max_records: int) -> tuple[list[DueKey], bool, bool]: + return self._due_keys(now, max_records, priority_capture_id=priority_capture_id) + try: return _capture_sweep.run_bounded_sweep( budget=budget, monotonic=self._monotonic, wall_clock=self._wall_clock, - due_keys=self._due_keys, + due_keys=due_keys, before_attempt=self._advance_sweep_cursor, sweep_one=self._sweep_one, work_counters=lambda: _capture_orphan_scan.sweep_work_counters(self), diff --git a/tests/hooks/test_capture_artifacts.py b/tests/hooks/test_capture_artifacts.py index a0c2e81aa8..3d3d0343d7 100644 --- a/tests/hooks/test_capture_artifacts.py +++ b/tests/hooks/test_capture_artifacts.py @@ -232,7 +232,14 @@ def _create_artifact(anchor, root, capture_id: str = _CAPTURE_ID): return create_capture_artifact(root, capture_id, lifecycle) -def _finalize_artifact(anchor, root, artifact, data: bytes = b"captured"): +def _finalize_artifact( + anchor, + root, + artifact, + data: bytes = b"captured", + *, + issue_reference: bool = True, +): lifecycle = CaptureLifecycleStore.from_open_authorities( anchor, root, @@ -253,7 +260,16 @@ def _finalize_artifact(anchor, root, artifact, data: bytes = b"captured"): finalized_at=1_000_000.0, retention_deadline=1_003_600.0, ) - return lifecycle.commit_verified_snapshot(verified, issue_reference=True) + return lifecycle.commit_verified_snapshot(verified, issue_reference=issue_reference) + + +def _seed_due_inline_capture(anchor, root, capture_id: str) -> None: + artifact = _create_artifact(anchor, root, capture_id) + try: + _finalize_artifact(anchor, root, artifact, data=b"due", issue_reference=False) + finally: + artifact.close_artifact_fd() + artifact.release_lease() def _issue_artifact(anchor, root, artifact, data: bytes = b"captured"): @@ -1368,9 +1384,10 @@ def test_valid_reject_runs_one_runner_tail_sweep( ) -> None: events: list[str] = [] - def reconcile(requested_cwd, budget): + def reconcile(requested_cwd, budget, *, priority_capture_id=None): assert requested_cwd == "/abs/project" assert budget is capture_reconcile.RUNNER_TAIL_BUDGET + assert priority_capture_id == _CAPTURE_ID events.append("reconcile") return CaptureCleanupOutcome() @@ -1388,10 +1405,10 @@ def reconcile(requested_cwd, budget): def test_runner_tail_consumes_byte_pressure_budget( monkeypatch: pytest.MonkeyPatch, ) -> None: - budgets = [] + reconciliations = [] - def reconcile(_requested_cwd, budget): - budgets.append(budget) + def reconcile(_requested_cwd, budget, *, priority_capture_id=None): + reconciliations.append((budget, priority_capture_id)) return CaptureCleanupOutcome() monkeypatch.setattr(capture_runner, "_BYTE_PRESSURE_OBSERVED", True) @@ -1401,13 +1418,95 @@ def reconcile(_requested_cwd, budget): reconcile, ) - capture_runner._sweep_after_runner("/abs/project") - capture_runner._sweep_after_runner("/abs/project") + capture_runner._sweep_after_runner("/abs/project", _CAPTURE_ID) + capture_runner._sweep_after_runner("/abs/project", _CAPTURE_ID) + + assert reconciliations == [ + (capture_types.TRANSITION_RESCUE_BUDGET, _CAPTURE_ID), + (capture_reconcile.RUNNER_TAIL_BUDGET, _CAPTURE_ID), + ] + + +def test_runner_tail_reclaims_its_own_inline_capture( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], +) -> None: + project = tmp_path / "project" + project.mkdir() + process = _FakeCaptureProcess(b"inline") + monkeypatch.setattr(capture_runner, "_spawn_bash", lambda *_args, **_kwargs: process) + + assert capture_runner._main(_runner_args(cwd=str(project))) == 0 - assert budgets == [ - capture_types.TRANSITION_RESCUE_BUDGET, - capture_reconcile.RUNNER_TAIL_BUDGET, + captured = capfd.readouterr() + record = _capture_record(project) + assert captured.out == "inline" + assert record.state is CaptureState.DELETED + assert not (_capture_dir(project) / f"shell_{_CAPTURE_ID}.log").exists() + + +def test_runner_tail_reclaims_own_capture_before_backlog( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + project = tmp_path / "project" + project.mkdir() + anchor, root = _open_authority(project) + backlog_ids = [ + f"{index:016x}" for index in range(capture_reconcile.RUNNER_TAIL_BUDGET.max_attempts + 1) ] + try: + for capture_id in backlog_ids: + _seed_due_inline_capture(anchor, root, capture_id) + finally: + root.close() + anchor.close() + + process = _FakeCaptureProcess(b"inline") + monkeypatch.setattr(capture_runner, "_spawn_bash", lambda *_args, **_kwargs: process) + + assert capture_runner._main(_runner_args(cwd=str(project))) == 0 + + record = _capture_record(project) + assert record.state is CaptureState.DELETED + assert not (_capture_dir(project) / f"shell_{_CAPTURE_ID}.log").exists() + + +def test_runner_tail_retains_referenced_capture( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], +) -> None: + project = tmp_path / "project" + project.mkdir() + expected = b"published-output" + process = _FakeCaptureProcess(expected) + monkeypatch.setattr(capture_runner, "_spawn_bash", lambda *_args, **_kwargs: process) + monkeypatch.setattr( + capture_runner, + "read_capture_policy", + lambda _anchor: CapturePolicy(inline_bytes=8), + ) + + assert capture_runner._main(_runner_args(cwd=str(project))) == 0 + + marker = _single_v2_marker(capfd.readouterr().out) + record = _capture_record(project) + artifact_path = _capture_dir(project) / f"shell_{_CAPTURE_ID}.log" + assert marker.reference_status == "published" + assert marker.reference is not None + assert record.state is CaptureState.FINALIZED + assert record.reference_status is CaptureReferenceStatus.PUBLISHED + assert record.delivery_status is CaptureDeliveryStatus.DELIVERED + assert artifact_path.is_file() + with open_capture_lifecycle( + str(project), + create=False, + lock_wait=capture_types.HOT_PATH_LOCK_WAIT, + ) as lifecycle: + with lifecycle.open_verified_capture(marker.reference) as reader: + assert reader.read(0, len(expected)) == expected def test_runner_tail_preserves_dispatch_result_and_order( @@ -1419,7 +1518,8 @@ def dispatch(*_args) -> int: events.append("dispatch") return 37 - def reconcile(_requested_cwd, _budget): + def reconcile(_requested_cwd, _budget, *, priority_capture_id=None): + assert priority_capture_id == _CAPTURE_ID events.append("reconcile") return CaptureCleanupOutcome() @@ -1440,7 +1540,7 @@ def test_runner_tail_cleanup_failure_does_not_replace_user_result( ) -> None: monkeypatch.setattr(capture_runner, "_dispatch_runner", lambda *_args: 23) - def fail_reconcile(_requested_cwd, _budget): + def fail_reconcile(_requested_cwd, _budget, **_kwargs): raise RuntimeError("🔥" * 512) monkeypatch.setattr( @@ -1465,7 +1565,7 @@ def test_runner_tail_reports_sweep_outcome_errors( monkeypatch.setattr( capture_reconcile, "reconcile_capture_store", - lambda requested_cwd, budget: CaptureCleanupOutcome( + lambda _requested_cwd, _budget, **_kwargs: CaptureCleanupOutcome( errors=2, remaining_due=1, blocker=CleanupBlocker.LEDGER_INTEGRITY, @@ -1489,7 +1589,7 @@ def test_runner_tail_deferred_outcome_emits_nothing_per_command( monkeypatch.setattr( capture_reconcile, "reconcile_capture_store", - lambda requested_cwd, budget: CaptureCleanupOutcome( + lambda _requested_cwd, _budget, **_kwargs: CaptureCleanupOutcome( examined=2, deleted=2, remaining_due=3, @@ -1514,7 +1614,7 @@ def test_runner_tail_still_sweeps_after_unexpected_dispatch_exception( def fail_dispatch(*_args): raise RuntimeError("fault injection") - def reconcile(_requested_cwd, _budget): + def reconcile(_requested_cwd, _budget, **_kwargs): swept.append(True) return CaptureCleanupOutcome() @@ -1535,7 +1635,7 @@ def test_malformed_runner_invocation_reconciles_only_with_absolute_cwd( ) -> None: reconciled: list[str] = [] - def reconcile(requested_cwd, _budget): + def reconcile(requested_cwd, _budget, **_kwargs): reconciled.append(requested_cwd) return CaptureCleanupOutcome() diff --git a/tests/hooks/test_capture_lifecycle.py b/tests/hooks/test_capture_lifecycle.py index 205362a364..2c57d715b5 100644 --- a/tests/hooks/test_capture_lifecycle.py +++ b/tests/hooks/test_capture_lifecycle.py @@ -3607,8 +3607,8 @@ def test_sweep_reserves_one_attempt_after_discovery_consumes_elapsed_budget( anchor, root, store = _open_store(project, clock) real_due_keys = store._due_keys - def slow_discovery(now: float, max_records: int): - result = real_due_keys(now, max_records) + def slow_discovery(now: float, max_records: int, *, priority_capture_id: str | None = None): + result = real_due_keys(now, max_records, priority_capture_id=priority_capture_id) clock.advance(0.051) return result diff --git a/tests/hooks/test_capture_sustained_admission.py b/tests/hooks/test_capture_sustained_admission.py new file mode 100644 index 0000000000..4b0d48092a --- /dev/null +++ b/tests/hooks/test_capture_sustained_admission.py @@ -0,0 +1,338 @@ +"""Sustained capture admission under readability-based retention.""" + +from __future__ import annotations + +import math +import os +from dataclasses import replace +from pathlib import Path + +import pytest + +import autoskillit.hooks._capture._lifecycle_policy as capture_lifecycle_policy +import autoskillit.hooks._capture._reconcile as capture_reconcile +import autoskillit.hooks._capture_lifecycle as capture_lifecycle +from autoskillit.hooks._capture._snapshot import ( + CaptureMeasurement, + CommandOutcome, + verify_capture_snapshot, +) +from autoskillit.hooks._capture._types import ( + HOT_PATH_LOCK_WAIT, + CaptureCapacitySpec, +) +from autoskillit.hooks._capture_artifacts import ( + create_capture_artifact, + open_capture_root, + open_project_anchor, +) + +pytestmark = [pytest.mark.layer("hooks"), pytest.mark.medium] + + +class _Clock: + def __init__(self, value: float = 1_000_000.0) -> None: + self.value = value + + def wall(self) -> float: + return self.value + + def monotonic(self) -> float: + return self.value + + def advance(self, seconds: float) -> None: + self.value += seconds + + +def _open_store( + project: Path, + clock: _Clock, + *, + capacity: CaptureCapacitySpec | None = None, +): + project.mkdir() + anchor = open_project_anchor(str(project)) + try: + root = open_capture_root(anchor, create=True) + except BaseException: + anchor.close() + raise + try: + store = capture_lifecycle.CaptureLifecycleStore.from_open_authorities( + anchor, + root, + wall_clock=clock.wall, + monotonic=clock.monotonic, + lock_wait=HOT_PATH_LOCK_WAIT, + capacity=capacity, + ) + except BaseException: + root.close() + anchor.close() + raise + return anchor, root, store + + +def _close_artifact(artifact) -> None: + try: + artifact.close_artifact_fd() + finally: + artifact.release_lease() + + +def _begin_capture( + store, + anchor, + root, + clock: _Clock, + capture_id: str, + *, + issue_reference: bool = False, +): + artifact = create_capture_artifact(root, capture_id, store) + try: + data = b"captured output" + os.write(artifact.fd, data) + snapshot = verify_capture_snapshot( + fd=artifact.fd, + capture_id=artifact.authority.capture_id, + incarnation=artifact.authority.incarnation, + project_identity=(anchor.identity.device, anchor.identity.inode), + root_identity=(root.identity.device, root.identity.inode), + carrier_name=artifact.name, + carrier_identity=(artifact.identity.device, artifact.identity.inode), + measurement=CaptureMeasurement.from_bytes(data, inline_bytes=len(data)), + command_outcome=CommandOutcome.exited(0), + expected_revision=artifact.authority.expected_revision, + finalized_at=clock.wall(), + retention_deadline=clock.wall() + 3600.0, + ) + finalized = store.commit_verified_snapshot( + snapshot, + issue_reference=issue_reference, + ) + return artifact, finalized + except BaseException: + _close_artifact(artifact) + raise + + +def _deliver_capture(store, value) -> None: + store.transition_delivery( + value, + expected=capture_lifecycle.CaptureDeliveryStatus.NOT_ATTEMPTED, + target=capture_lifecycle.CaptureDeliveryStatus.ATTEMPTING, + ) + store.transition_delivery( + value, + expected=capture_lifecycle.CaptureDeliveryStatus.ATTEMPTING, + target=capture_lifecycle.CaptureDeliveryStatus.DELIVERED, + ) + + +def _observe_capacity_and_deletions( + store, + monkeypatch: pytest.MonkeyPatch, + active_ids: set[str], + reference_expiries: dict[str, float], + clock: _Clock, +) -> list[capture_lifecycle.CaptureCapacityReason]: + debt_reasons = [] + real_admission_reason = store._admission_reason + + def observe_admission_reason(records, candidate, compaction_epoch, now): + decision = real_admission_reason(records, candidate, compaction_epoch, now) + if decision.reason in { + capture_lifecycle.CaptureCapacityReason.RECLAMATION_DEBT_ASSIST, + capture_lifecycle.CaptureCapacityReason.RECLAMATION_DEBT_STALL, + }: + debt_reasons.append(decision.reason) + return decision + + real_sweep_one = store._sweep_one + + def observe_sweep_one(capture_id: str): + result = real_sweep_one(capture_id) + if result[0] is capture_lifecycle.SweepAttempt.DELETED: + if capture_id in reference_expiries: + assert clock.wall() >= reference_expiries[capture_id] + active_ids.discard(capture_id) + return result + + monkeypatch.setattr(store, "_admission_reason", observe_admission_reason) + monkeypatch.setattr(store, "_sweep_one", observe_sweep_one) + return debt_reasons + + +def _capacity_for_measured_frame(tmp_path: Path, clock: _Clock) -> CaptureCapacitySpec: + anchor, root, store = _open_store(tmp_path / "bootstrap", clock) + artifact, _finalized = _begin_capture( + store, + anchor, + root, + clock, + "ffffffffffffffff", + ) + try: + frame_record = store.get_record("ffffffffffffffff") + assert isinstance(frame_record, capture_lifecycle.CaptureLifecycleRecord) + assert frame_record.state is capture_lifecycle.CaptureState.FINALIZED + frame = len( + capture_lifecycle._capture_ledger.encode_frame( + capture_lifecycle._capture_lifecycle_record.record_to_dict(frame_record), + compaction_epoch=1, + ) + ) + finally: + _close_artifact(artifact) + root.close() + anchor.close() + + base = CaptureCapacitySpec() + low_bytes = frame * 150 + high_bytes = low_bytes + 4096 + return replace( + base, + max_tombstones=32, + compaction_low_bytes=low_bytes, + compaction_high_bytes=high_bytes, + hard_ledger_bytes=high_bytes + base.recovery_headroom_bytes, + ) + + +def _produce_capture( + store, + anchor, + root, + clock: _Clock, + capture_id: str, + *, + reference: bool, + active_ids: set[str], + reference_expiries: dict[str, float], +): + active_ids.add(capture_id) + artifact, finalized = _begin_capture( + store, + anchor, + root, + clock, + capture_id, + issue_reference=reference, + ) + try: + value = store.publish_reference(finalized) if reference else finalized + if reference: + record = store.get_record(capture_id) + assert record is not None + assert record.manifest is not None + assert record.manifest.reference_expiry is not None + reference_expiries[capture_id] = record.manifest.reference_expiry + _deliver_capture(store, value) + finally: + _close_artifact(artifact) + + +def _sweep_capture(store, capture_id: str) -> None: + store.sweep( + capture_reconcile.RUNNER_TAIL_BUDGET, + priority_capture_id=capture_id, + ) + + +def test_issue_5282_inline_burst_does_not_exhaust_admission( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _Clock() + capacity = _capacity_for_measured_frame(tmp_path, clock) + anchor, root, store = _open_store(tmp_path / "workload", clock, capacity=capacity) + active_ids: set[str] = set() + reference_expiries: dict[str, float] = {} + debt_reasons = _observe_capacity_and_deletions( + store, + monkeypatch, + active_ids, + reference_expiries, + clock, + ) + max_non_deleted = 0 + capture_number = 0 + try: + producers = 4 + for _ in range(5400 // 30): + clock.advance(30) + tick_capture_ids = [] + for _ in range(producers): + capture_id = f"{capture_number:016x}" + _produce_capture( + store, + anchor, + root, + clock, + capture_id, + reference=capture_number % 12 == 0, + active_ids=active_ids, + reference_expiries=reference_expiries, + ) + tick_capture_ids.append(capture_id) + capture_number += 1 + max_non_deleted = max(max_non_deleted, len(active_ids)) + for capture_id in tick_capture_ids: + _sweep_capture(store, capture_id) + max_non_deleted = max(max_non_deleted, len(active_ids)) + + referenced_per_second = 1 / 90 + allowed_non_deleted = ( + math.ceil(referenced_per_second * capture_lifecycle_policy.REFERENCE_LIFETIME_SECONDS) + + producers + + 1 + ) + assert capture_number == 720 + assert max_non_deleted <= allowed_non_deleted + assert not debt_reasons + finally: + root.close() + anchor.close() + + +def test_referenced_load_never_reclaims_before_expiry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _Clock() + capacity = _capacity_for_measured_frame(tmp_path, clock) + anchor, root, store = _open_store(tmp_path / "workload", clock, capacity=capacity) + active_ids: set[str] = set() + reference_expiries: dict[str, float] = {} + debt_reasons = _observe_capacity_and_deletions( + store, + monkeypatch, + active_ids, + reference_expiries, + clock, + ) + max_non_deleted = 0 + try: + for capture_number in range(5400 // 30): + clock.advance(30) + capture_id = f"{capture_number:016x}" + _produce_capture( + store, + anchor, + root, + clock, + capture_id, + reference=True, + active_ids=active_ids, + reference_expiries=reference_expiries, + ) + _sweep_capture(store, capture_id) + max_non_deleted = max(max_non_deleted, len(active_ids)) + + max_live_references = math.ceil(capture_lifecycle_policy.REFERENCE_LIFETIME_SECONDS / 30) + assert max_non_deleted <= max_live_references + 1 + assert not debt_reasons + finally: + root.close() + anchor.close() diff --git a/tests/hooks/test_capture_verified_reader.py b/tests/hooks/test_capture_verified_reader.py index b31fd52856..fa07989a5f 100644 --- a/tests/hooks/test_capture_verified_reader.py +++ b/tests/hooks/test_capture_verified_reader.py @@ -12,13 +12,17 @@ import threading import time from contextlib import ExitStack +from dataclasses import replace from pathlib import Path import pytest import autoskillit.hooks._capture._cleanup as capture_cleanup import autoskillit.hooks._capture._descriptor as capture_descriptor +import autoskillit.hooks._capture._lifecycle_policy as capture_lifecycle_policy +import autoskillit.hooks._capture._lifecycle_record as capture_lifecycle_record import autoskillit.hooks._capture._reader as capture_reader +import autoskillit.hooks._capture._reconcile as capture_reconcile import autoskillit.hooks._capture._runner as capture_runner import autoskillit.hooks._capture_lifecycle as capture_lifecycle from autoskillit.hooks._capture._reader import ( @@ -45,12 +49,34 @@ CaptureLifecycleError, CaptureLifecycleStore, CaptureReferenceStatus, + CaptureState, ) from tests.conftest import production_interpreter_env pytestmark = [pytest.mark.layer("hooks"), pytest.mark.medium] _CAPTURE_ID = "0123456789abcdef" +_READABLE_BEARER_CELLS = tuple( + (reference_status, delivery_status) + for reference_status in ( + CaptureReferenceStatus.ISSUED, + CaptureReferenceStatus.PUBLISHED, + CaptureReferenceStatus.UNKNOWN, + ) + for delivery_status in CaptureDeliveryStatus + if reference_status is not CaptureReferenceStatus.ISSUED + or delivery_status is CaptureDeliveryStatus.NOT_ATTEMPTED +) +_UNSETTLED_DELIVERY_CELLS = tuple( + (reference_status, delivery_status) + for reference_status in CaptureReferenceStatus + for delivery_status in CaptureDeliveryStatus + if delivery_status is not CaptureDeliveryStatus.DELIVERED + and not ( + reference_status is CaptureReferenceStatus.ISSUED + and delivery_status is not CaptureDeliveryStatus.NOT_ATTEMPTED + ) +) + ((CaptureReferenceStatus.UNKNOWN, CaptureDeliveryStatus.DELIVERED),) class _Clock: @@ -91,12 +117,13 @@ def _finalize( data: bytes, *, issue_reference: bool = True, + capture_id: str = _CAPTURE_ID, ): with ExitStack() as cleanup: anchor, root, store = _open_store(project, clock) cleanup.callback(anchor.close) cleanup.callback(root.close) - artifact = create_capture_artifact(root, _CAPTURE_ID, store) + artifact = create_capture_artifact(root, capture_id, store) cleanup.callback(artifact.release_lease) cleanup.callback(artifact.close_artifact_fd) os.write(artifact.fd, data) @@ -142,6 +169,47 @@ def _close_artifact(artifact) -> None: artifact.release_lease() +def _deliver(store, value): + store.transition_delivery( + value, + expected=CaptureDeliveryStatus.NOT_ATTEMPTED, + target=CaptureDeliveryStatus.ATTEMPTING, + ) + return store.transition_delivery( + value, + expected=CaptureDeliveryStatus.ATTEMPTING, + target=CaptureDeliveryStatus.DELIVERED, + ) + + +def _record_for_readability_cell( + project: Path, + clock: _Clock, + reference_status: CaptureReferenceStatus, + delivery_status: CaptureDeliveryStatus, +): + anchor, root, store, artifact, _finalized = _finalize( + project, + clock, + b"captured", + issue_reference=reference_status is not CaptureReferenceStatus.NOT_REQUESTED, + ) + try: + record = store.get_record(_CAPTURE_ID) + assert record is not None + candidate = replace( + record, + reference_status=reference_status, + delivery_status=delivery_status, + ) + capture_lifecycle_record.validate_record(candidate) + return candidate + finally: + _close_artifact(artifact) + root.close() + anchor.close() + + def test_published_reference_returns_self_contained_bounded_reader( tmp_path: Path, ) -> None: @@ -927,3 +995,263 @@ def test_unavailable_reference_cannot_be_resolved(tmp_path: Path) -> None: finally: root.close() anchor.close() + + +@pytest.mark.parametrize( + ("reference_status", "delivery_status"), + _READABLE_BEARER_CELLS, +) +def test_readable_until_never_precedes_an_outstanding_bearer( + tmp_path: Path, + reference_status: CaptureReferenceStatus, + delivery_status: CaptureDeliveryStatus, +) -> None: + record = _record_for_readability_cell( + tmp_path / "project", + _Clock(), + reference_status, + delivery_status, + ) + + assert record.manifest is not None + assert record.manifest.reference_expiry is not None + readable_until = capture_lifecycle_record.readable_until(record) + assert readable_until >= record.manifest.reference_expiry + assert readable_until <= record.manifest.retention_deadline + + +@pytest.mark.parametrize( + "reference_status", + ( + CaptureReferenceStatus.NOT_REQUESTED, + CaptureReferenceStatus.UNAVAILABLE, + CaptureReferenceStatus.EXPIRED, + CaptureReferenceStatus.REVOKED, + ), +) +def test_readable_until_releases_delivered_capture_without_bearer( + tmp_path: Path, + reference_status: CaptureReferenceStatus, +) -> None: + record = _record_for_readability_cell( + tmp_path / "project", + _Clock(), + reference_status, + CaptureDeliveryStatus.DELIVERED, + ) + + assert record.manifest is not None + readable_until = capture_lifecycle_record.readable_until(record) + assert readable_until == record.manifest.finalized_at + assert readable_until <= record.manifest.retention_deadline + + +@pytest.mark.parametrize( + ("reference_status", "delivery_status"), + _UNSETTLED_DELIVERY_CELLS, +) +def test_readable_until_holds_unsettled_delivery( + tmp_path: Path, + reference_status: CaptureReferenceStatus, + delivery_status: CaptureDeliveryStatus, +) -> None: + record = _record_for_readability_cell( + tmp_path / "project", + _Clock(), + reference_status, + delivery_status, + ) + + assert record.manifest is not None + readable_until = capture_lifecycle_record.readable_until(record) + assert readable_until == record.manifest.retention_deadline + assert readable_until <= record.manifest.retention_deadline + + +def test_success_path_hold_is_bounded_by_reference_lifetime(tmp_path: Path) -> None: + clock = _Clock() + anchor, root, store, artifact, _finalized, published = _publish( + tmp_path / "project", + clock, + b"published", + ) + try: + _deliver(store, published) + record = store.get_record(_CAPTURE_ID) + assert record is not None + assert record.manifest is not None + assert ( + capture_lifecycle_record.readable_until(record) - record.manifest.finalized_at + <= capture_lifecycle_policy.REFERENCE_LIFETIME_SECONDS + ) + finally: + _close_artifact(artifact) + root.close() + anchor.close() + + +def test_delivered_inline_capture_is_due_immediately(tmp_path: Path) -> None: + clock = _Clock() + anchor, root, store, artifact, finalized = _finalize( + tmp_path / "project", + clock, + b"inline", + issue_reference=False, + ) + artifact_open = True + try: + _deliver(store, finalized) + record = store.get_record(_CAPTURE_ID) + assert record is not None + assert record.next_attempt_at <= clock.wall() + _close_artifact(artifact) + artifact_open = False + outcome = store.sweep(capture_reconcile.RUNNER_TAIL_BUDGET) + assert outcome.deleted == 1 + deleted = store.get_record(_CAPTURE_ID) + assert deleted is not None + assert deleted.state is CaptureState.DELETED + finally: + if artifact_open: + _close_artifact(artifact) + root.close() + anchor.close() + + +def test_delivered_reference_capture_reclaims_at_reference_expiry( + tmp_path: Path, +) -> None: + clock = _Clock() + anchor, root, store, artifact, finalized = _finalize( + tmp_path / "project", + clock, + b"referenced", + ) + reader = None + artifact_open = True + try: + published = store.publish_reference(finalized) + _deliver(store, published) + record = store.get_record(_CAPTURE_ID) + assert record is not None + assert record.manifest is not None + assert record.manifest.reference_expiry is not None + expiry = record.manifest.reference_expiry + _close_artifact(artifact) + artifact_open = False + reader = store.open_verified_capture(published.token) + + clock.advance(expiry - clock.wall() - 1) + before_expiry = store.sweep(capture_reconcile.RUNNER_TAIL_BUDGET) + assert before_expiry.deleted == 0 + retained = store.get_record(_CAPTURE_ID) + assert retained is not None + assert retained.state is CaptureState.FINALIZED + + clock.advance(2) + at_expiry = store.sweep(capture_reconcile.RUNNER_TAIL_BUDGET) + assert at_expiry.carrier_lease_live == 1 + retained = store.get_record(_CAPTURE_ID) + assert retained is not None + assert retained.state is CaptureState.FINALIZED + reader.close() + reader = None + + after_reader = store.sweep(capture_reconcile.RUNNER_TAIL_BUDGET) + assert after_reader.deleted == 1 + deleted = store.get_record(_CAPTURE_ID) + assert deleted is not None + assert deleted.state is CaptureState.DELETED + finally: + if reader is not None: + reader.close() + if artifact_open: + _close_artifact(artifact) + root.close() + anchor.close() + + +def test_interrupted_delivery_keeps_full_hold(tmp_path: Path) -> None: + clock = _Clock() + project = tmp_path / "project" + anchor, root, store, artifact, finalized = _finalize( + project, + clock, + b"interrupted", + issue_reference=False, + ) + try: + store.transition_delivery( + finalized, + expected=CaptureDeliveryStatus.NOT_ATTEMPTED, + target=CaptureDeliveryStatus.ATTEMPTING, + ) + finally: + _close_artifact(artifact) + root.close() + anchor.close() + + anchor, root, store = _open_store(project, clock) + try: + record = store.get_record(_CAPTURE_ID) + assert record is not None + assert record.delivery_status is CaptureDeliveryStatus.UNKNOWN + clock.advance(3599) + assert store.sweep(capture_reconcile.RUNNER_TAIL_BUDGET).deleted == 0 + clock.advance(2) + assert store.sweep(capture_reconcile.RUNNER_TAIL_BUDGET).deleted == 1 + finally: + root.close() + anchor.close() + + +def test_priority_capture_outside_ordinary_inspection_window_is_attempted_first( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _Clock() + resources = [] + capture_ids = ( + "0000000000000001", + "0000000000000002", + "0000000000000003", + ) + priority_capture_id = capture_ids[-1] + try: + for capture_id in capture_ids: + anchor, root, store, artifact, finalized = _finalize( + tmp_path / "project", + clock, + b"inline", + issue_reference=False, + capture_id=capture_id, + ) + resources.append((root, anchor)) + try: + _deliver(store, finalized) + finally: + _close_artifact(artifact) + + attempted = [] + real_sweep_one = store._sweep_one + + def observe_attempt(capture_id: str): + attempted.append(capture_id) + return real_sweep_one(capture_id) + + monkeypatch.setattr(store, "_sweep_one", observe_attempt) + budget = capture_lifecycle.SweepBudgetSpec( + max_records_inspected=2, + max_attempts=1, + max_duration_seconds=1.0, + ) + outcome = store.sweep(budget, priority_capture_id=priority_capture_id) + + assert attempted[0] == priority_capture_id + assert outcome.examined == 1 + assert outcome.records_inspected <= budget.max_records_inspected + assert outcome.deleted == 1 + finally: + for root, anchor in resources: + root.close() + anchor.close() diff --git a/tests/hooks/test_shell_capture_conformance.py b/tests/hooks/test_shell_capture_conformance.py index f164c7509a..2c780ad1b4 100644 --- a/tests/hooks/test_shell_capture_conformance.py +++ b/tests/hooks/test_shell_capture_conformance.py @@ -199,9 +199,6 @@ def _assert_expectation( expected = raw.stdout assert raw.stderr is None assert runner.stderr == b"" - artifacts = _artifact_files(runner_project) - assert len(artifacts) == 1 - assert artifacts[0].read_bytes() == expected if len(expected) <= _INLINE_BYTES: assert runner.stdout == expected else: @@ -218,7 +215,6 @@ def _assert_expectation( assert b"late" in raw.stdout assert runner.stdout == raw.stdout assert runner.stderr == b"" - assert _artifact_files(runner_project)[0].read_bytes() == raw.stdout return if expectation is ConformanceExpectation.LATE_PIPE_BYTES_SETTLED: @@ -450,7 +446,11 @@ def test_command_conformance( runner = _run_runner(case.command, projects[mode.value], mode=mode.value) if mode is ConformanceMode.CAPTURE: - assert len(_artifact_files(projects[mode.value])) == 1 + published = any( + line.startswith(b"[AutoSkillit shell capture v2:") + for line in runner.stdout.splitlines() + ) + assert len(_artifact_files(projects[mode.value])) == int(published) else: assert not _artifact_files(projects[mode.value]) _assert_expectation(case, mode, raw, runner, projects) @@ -603,19 +603,8 @@ def test_runner_keeps_project_and_execution_authorities_distinct( assert not (project / "execution-sentinel").exists() assert not _artifact_files(execution_dir) - project_artifacts = _artifact_files(project) - assert bool(project_artifacts) is expect_capture + assert not _artifact_files(project) if expect_capture: - assert len(project_artifacts) == 1 - capture_id = project_artifacts[0].stem.removeprefix("shell_") - with open_capture_lifecycle( - str(project), - create=False, - lock_wait=HOT_PATH_LOCK_WAIT, - ) as lifecycle: - record = lifecycle.get_record(capture_id) - assert record is not None - assert record.state is CaptureState.FINALIZED assert (_capture_dir(project) / ".capture-lifecycle.ledger").is_file() assert not _capture_dir(execution_dir).exists() else: @@ -680,9 +669,7 @@ def test_retained_pipe_waits_for_actual_eof_and_includes_late_bytes( assert process.returncode == 0 assert stdout == expected assert stderr == b"" - artifacts = _artifact_files(tmp_path) - assert len(artifacts) == 1 - assert artifacts[0].read_bytes() == expected + assert not _artifact_files(tmp_path) with open_capture_lifecycle( str(tmp_path), create=False, @@ -690,7 +677,7 @@ def test_retained_pipe_waits_for_actual_eof_and_includes_late_bytes( ) as lifecycle: finalized = lifecycle.get_record(capture_id) assert finalized is not None and finalized.manifest is not None - assert finalized.state is CaptureState.FINALIZED + assert finalized.state is CaptureState.DELETED assert finalized.manifest.total_bytes == len(expected) assert finalized.manifest.sha256 == hashlib.sha256(expected).hexdigest() assert finalized.manifest.inline_length == len(expected) @@ -741,7 +728,8 @@ def test_detached_child_with_closed_pipe_does_not_delay_finalization( ) as lifecycle: finalized = lifecycle.get_record(capture_id) assert finalized is not None and finalized.manifest is not None - assert finalized.state is CaptureState.FINALIZED + assert finalized.state is CaptureState.DELETED + assert not _artifact_files(tmp_path) assert finalized.manifest.total_bytes == len(stdout) assert finalized.manifest.sha256 == hashlib.sha256(stdout).hexdigest() finally: @@ -761,14 +749,10 @@ def test_interleaved_stdout_stderr_ordering(tmp_path: Path) -> None: assert wrapped.returncode == raw_merged.returncode == 0 - artifacts = _artifact_files(tmp_path) - if artifacts: - actual = artifacts[0].read_bytes() - else: - actual = wrapped.stdout - - assert actual == raw_merged.stdout, ( - f"Interleaved ordering mismatch.\n raw_merged={raw_merged.stdout!r}\n actual={actual!r}" + assert not _artifact_files(tmp_path) + assert wrapped.stdout == raw_merged.stdout, ( + f"Interleaved ordering mismatch.\n raw_merged={raw_merged.stdout!r}" + f"\n actual={wrapped.stdout!r}" ) @@ -861,7 +845,7 @@ def test_main_generated_wrapper_accepts_symlinked_cwd( assert completed.returncode == 0 assert completed.stdout == b"anchored" - assert len(_artifact_files(project)) == 1 + assert not _artifact_files(project) @pytest.mark.parametrize("collision", ["symlink", "hardlink", "regular"]) From 84afa1e2e58f529bd97c223f4cc3465f3e6c0943 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 16:44:56 -0700 Subject: [PATCH 02/21] fix: account for complete capture capacity envelope --- docs/decisions/0006-output-containment.md | 54 ++- .../hooks/_capture/_artifact_setup.py | 5 +- src/autoskillit/hooks/_capture/_capacity.py | 10 +- .../hooks/_capture/_ledger_view.py | 2 + src/autoskillit/hooks/_capture/_types.py | 49 +- .../test_subpackage_isolation_singleton_io.py | 5 +- tests/hooks/test_capture_capacity_model.py | 236 ++++++++++ tests/hooks/test_capture_frame_bound.py | 199 ++++++++ tests/hooks/test_capture_lifecycle.py | 428 +++++++++++++++++- 9 files changed, 941 insertions(+), 47 deletions(-) create mode 100644 tests/hooks/test_capture_capacity_model.py create mode 100644 tests/hooks/test_capture_frame_bound.py diff --git a/docs/decisions/0006-output-containment.md b/docs/decisions/0006-output-containment.md index 5f452b09f3..3546c09536 100644 --- a/docs/decisions/0006-output-containment.md +++ b/docs/decisions/0006-output-containment.md @@ -185,6 +185,9 @@ hook can be retired in favor of that mechanism. rejected. Physical path strings are display hints, not filesystem authority. 5. General retrieval, publication/privacy policy, quota accounting, and upstream live visibility remain downstream work identified by ADR-0008. +6. Undelivered and otherwise unsettled-delivery anomalies retain the one-hour + hold and sit outside the declared load; their encoded frames still count + against actual capacity. ## Resolved @@ -226,10 +229,53 @@ classification applied to a reason that should be `DISCARD_OUTPUT`). The same enforcement surface bounds admission by measured reclamation debt: assist and stall are admission-only capacity reasons, and import-time totality requires their gate, failure, disposition, detail, and blocker classifications. -The production compaction floor is derived from the code-owned sustained -admission envelope, the canonical one-hour sweep grace, and the incident's -measured bytes per record. Runtime policy overrides below that derived floor -are rejected in favor of the safe production default. + +The capacity model derives the production byte floor and the operational, +retained, and evidence record caps from one required record count. +`capacity_envelope_violations(spec)` supplies both the import-time check for the +production default and runtime override validation. The import-time assertion +checks only the static default; live admission and transition gates use actual +projected bytes. The success-path hold uses `REFERENCE_LIFETIME_SECONDS`: at the +declared peak of 0.5333 admissions per +second and a 1,800-second reference lifetime, `ceil(0.5333… * 1,800) + 16` +producers yields 976 held records. The floor adds the due debt that can accrue +before reclamation assist engages, `reclamation_debt_assist_records - 1`; this +produces 1,231 required records at the default 256-record assist threshold. +The same count is required by each record cap. + +`PLANNED_RECORD_FRAME_BYTES` is 2,608 bytes: the largest measured encoding of +real `FINALIZED`, `DELETED`, and `FAILED` record shapes under the declared +numeric profile, rounded up to a multiple of eight. The measurement sets +non-schema counters and identities to `2**63 - 1`, uses the same compaction +epoch, valid finite timestamps, and bounded command outcomes. Adding the +default 256 tombstones gives `(1,231 + 256) * 2,608 = 3,878,096` bytes, below +the default `compaction_low_bytes` of 3,932,160. The declared allowance models +that load and profile; `CompactedFrameSizer` counts every encoded byte in the +retained projection, including opaque frames, for both admission and compaction. +A remeasured allowance above 2,644 bytes makes the default +import-time check fail and requires re-deriving the floor. Overrides that break +the byte or shared record-count envelope are rejected by the same function. + +`compaction_low_bytes` is a soft limit for state-advancing work. Admission is +gated separately; transitions that advance a record through stage, publish, +and finalize may be refused above it. Same-state bookkeeping, including +delivery and reference updates, open-time normalization, and sweep expiry, +remains subject to the hard bound. Recovery transitions retain their reserved +headroom. `_with_capacity_rescue` first attempts the operation; for its existing +rescuable refusal reasons, it performs one bounded sweep and retries from a +fresh locked load before final refusal. + +The retained-record envelope is a compacted projection of latest record +frames, allowed tombstones, and opaque bytes. Superseded transition frames +remain in the physical ledger until compaction, so they are temporary physical +amplification and are not multiplied into the retained-record floor. +`_append_locked` checks prospective physical size and compacts instead of +appending when the append trigger would be crossed. Compaction output may itself +exceed that trigger while remaining within the hard `MAX_LEDGER_BYTES` bound; +the trigger is not a universal ceiling. A projected-capacity refusal is a +`CaptureCapacityError`; an existing physical file above `MAX_LEDGER_BYTES` is +rejected on load as `CaptureLedgerError` and does not enter the capacity-only +degraded path. ## Consequences diff --git a/src/autoskillit/hooks/_capture/_artifact_setup.py b/src/autoskillit/hooks/_capture/_artifact_setup.py index 4fce9e394c..a34535e43f 100644 --- a/src/autoskillit/hooks/_capture/_artifact_setup.py +++ b/src/autoskillit/hooks/_capture/_artifact_setup.py @@ -254,9 +254,8 @@ def _policy_capacity(value: object) -> CaptureCapacitySpec | None: candidate = replace(CaptureCapacitySpec(), **validated) except (TypeError, ValueError): return None - if ( - candidate.compaction_low_bytes < _types.REQUIRED_RETENTION_BYTES - or candidate.hard_ledger_bytes > _capture_lifecycle.MAX_LEDGER_BYTES + if _types.capacity_envelope_violations(candidate) or ( + candidate.hard_ledger_bytes > _capture_lifecycle.MAX_LEDGER_BYTES ): return None return candidate diff --git a/src/autoskillit/hooks/_capture/_capacity.py b/src/autoskillit/hooks/_capture/_capacity.py index a2d01eb0ec..3bcb44ada0 100644 --- a/src/autoskillit/hooks/_capture/_capacity.py +++ b/src/autoskillit/hooks/_capture/_capacity.py @@ -68,10 +68,14 @@ class CompactedFrameSizer: def __init__(self) -> None: self._frame_sizes: dict[str, tuple[Record, int, int]] = {} + self._opaque_bytes = 0 def reset(self) -> None: self._frame_sizes.clear() + def set_opaque_frames(self, frames: tuple[bytes, ...]) -> None: + self._opaque_bytes = sum(map(len, frames)) + def retain_records(self, records: Mapping[str, Record]) -> None: for capture_id in tuple(self._frame_sizes): if capture_id not in records: @@ -114,7 +118,7 @@ def total_bytes( ) self._frame_sizes[record.capture_id] = (record, compaction_epoch, size) total += size - return total + return total + self._opaque_bytes _REASON_DETAILS = { @@ -251,8 +255,10 @@ def transition_reason( projected = _projected(records, candidate) encoded = sizer.total_bytes(projected, compaction_epoch, spec) recovery_state = transition_compaction_bound(candidate, spec) == spec.hard_ledger_bytes + previous = records.get(candidate.capture_id) + state_changes = previous is None or previous.state is not candidate.state if encoded > transition_compaction_bound(candidate, spec): return CaptureCapacityReason.HARD_LEDGER_CAPACITY - if not recovery_state and encoded > spec.compaction_low_bytes: + if state_changes and not recovery_state and encoded > spec.compaction_low_bytes: return CaptureCapacityReason.PROJECTED_COMPACTED_BYTES return None diff --git a/src/autoskillit/hooks/_capture/_ledger_view.py b/src/autoskillit/hooks/_capture/_ledger_view.py index e129c886e1..48f93e81b2 100644 --- a/src/autoskillit/hooks/_capture/_ledger_view.py +++ b/src/autoskillit/hooks/_capture/_ledger_view.py @@ -163,6 +163,7 @@ def install( self.records = records self.opaque_frames = opaque_frames self.opaque_capture_ids = opaque_capture_ids + self._sizer.set_opaque_frames(opaque_frames) self.incarnation = LedgerIncarnation( value.st_dev, value.st_ino, @@ -246,6 +247,7 @@ def _load_full( if decoded.saw_legacy: self.opaque_frames = decoded.opaque_frames self.opaque_capture_ids = decoded.opaque_capture_ids + self._sizer.set_opaque_frames(self.opaque_frames) compact_legacy( decoded.records, decoded.compaction_epoch + 1, diff --git a/src/autoskillit/hooks/_capture/_types.py b/src/autoskillit/hooks/_capture/_types.py index 8395f288a5..e9e2370e49 100644 --- a/src/autoskillit/hooks/_capture/_types.py +++ b/src/autoskillit/hooks/_capture/_types.py @@ -30,12 +30,12 @@ "DEBT_ASSIST_BUDGET", "DEBT_ASSIST_MAX_TRANSITIONS", "HOT_PATH_LOCK_WAIT", - "MEASURED_BYTES_PER_RECORD", - "REQUIRED_RETENTION_BYTES", + "PLANNED_RECORD_FRAME_BYTES", "SUPPORTED_ADMISSIONS_PER_PRODUCER_SECOND", "SUPPORTED_PEAK_ADMISSIONS_PER_SECOND", "SUPPORTED_PEAK_PRODUCERS", "classify_cleanup_outcome", + "capacity_envelope_violations", "lock_wait_for_budget", ] @@ -383,29 +383,42 @@ class Tampered(Exception): max_directory_entries_scanned=0, ) -# Static sustained-admission envelope for the shared project store. Issue -# #4511 observed 13-16 concurrent producers and found that roughly one hour of -# fleet admissions filled the 1,962-record window. One admission per producer -# every 30 seconds declares the sustained envelope represented by that event; -# short bursts above it consume remaining byte headroom, while sustained excess -# is governed by debt assist and stall admission. +# Static success-path envelope for the shared project store. It covers records +# held for the reference lifetime, debt accumulated before assistance starts, +# and retained tombstones. Unsettled captures keep their one-hour forensic hold +# outside the declared load; the byte projection accounts for their cost. SUPPORTED_PEAK_PRODUCERS = 16 SUPPORTED_ADMISSIONS_PER_PRODUCER_SECOND = 1.0 / 30.0 SUPPORTED_PEAK_ADMISSIONS_PER_SECOND = ( SUPPORTED_PEAK_PRODUCERS * SUPPORTED_ADMISSIONS_PER_PRODUCER_SECOND ) -MEASURED_BYTES_PER_RECORD = 2004 -REQUIRED_RETENTION_BYTES = math.ceil( - SUPPORTED_PEAK_ADMISSIONS_PER_SECOND - * _lifecycle_policy.SWEEP_GRACE_SECONDS - * MEASURED_BYTES_PER_RECORD -) +PLANNED_RECORD_FRAME_BYTES = 2608 + -if CaptureCapacitySpec().compaction_low_bytes < REQUIRED_RETENTION_BYTES: - raise AssertionError( - "production capture capacity must cover the supported admission envelope " - "for the complete sweep-grace interval" +def capacity_envelope_violations(spec: CaptureCapacitySpec) -> tuple[str, ...]: + """Return production envelope limits that a capacity specification misses.""" + held = ( + math.ceil( + SUPPORTED_PEAK_ADMISSIONS_PER_SECOND * _lifecycle_policy.REFERENCE_LIFETIME_SECONDS + ) + + SUPPORTED_PEAK_PRODUCERS ) + records = held + spec.reclamation_debt_assist_records - 1 + requirements = ( + ( + "compaction_low_bytes", + PLANNED_RECORD_FRAME_BYTES * (records + spec.max_tombstones), + spec.compaction_low_bytes, + ), + ("max_operational_records", records, spec.max_operational_records), + ("max_retained_records", records, spec.max_retained_records), + ("max_evidence_records", records, spec.max_evidence_records), + ) + return tuple(field for field, required, actual in requirements if actual < required) + + +if capacity_envelope_violations(CaptureCapacitySpec()): + raise AssertionError("production capture capacity must cover the declared retention envelope") DEBT_ASSIST_MAX_TRANSITIONS = 16 DEBT_ASSIST_BUDGET = SweepBudgetSpec( diff --git a/tests/arch/test_subpackage_isolation_singleton_io.py b/tests/arch/test_subpackage_isolation_singleton_io.py index 915ab661d1..4d0a0c4815 100644 --- a/tests/arch/test_subpackage_isolation_singleton_io.py +++ b/tests/arch/test_subpackage_isolation_singleton_io.py @@ -108,8 +108,7 @@ def _get_call_func_name(node: ast.Call) -> str | None: "methodology_venue_appendix", # recipe/methodology_venue_appendix.py: _ML_SUB_AREA_CACHE "rules_blocks", # recipe/rules/rules_blocks.py: _BUDGETS_CACHE = YamlFileCache() "rules_phoropter_adjacency", # recipe/rules/rules_phoropter_adjacency.py: _PREFIXES_CACHE - # _RETENTION_SECONDS resolved once at import time from the single-source-of-truth - # STATE_RECLAIMABILITY sweep grace (_lifecycle_policy.SWEEP_GRACE_SECONDS). + # Lifecycle constants resolved once at import time from _lifecycle_policy. "_capture_lifecycle", # join ledger alphabet/filename constants resolved once at import time. "_join_ledger", # hooks/_join_ledger.py: _BATCH_ID_ALPHABET, LEDGER_FILENAME @@ -162,7 +161,6 @@ def _get_call_func_name(node: ast.Call) -> str | None: ("src/autoskillit/core/types/foundation/_type_dimensions.py", "ASCII_YAML_POLICY"), ("src/autoskillit/hooks/_capture/_types.py", "DEBT_ASSIST_BUDGET"), ("src/autoskillit/hooks/_capture/_types.py", "HOT_PATH_LOCK_WAIT"), - ("src/autoskillit/hooks/_capture/_types.py", "REQUIRED_RETENTION_BYTES"), ("src/autoskillit/hooks/_capture/_types.py", "TRANSITION_RESCUE_BUDGET"), ("src/autoskillit/pipeline/_context_admission_ledger/_codec.py", "_EVENT_TYPES"), ("src/autoskillit/pipeline/_context_admission_ledger/_codec.py", "_EFFECT_TYPES"), @@ -318,7 +316,6 @@ def test_capture_types_singleton_is_path_and_assignment_scoped(tmp_path: Path) - } == { "DEBT_ASSIST_BUDGET", "HOT_PATH_LOCK_WAIT", - "REQUIRED_RETENTION_BYTES", "TRANSITION_RESCUE_BUDGET", } unrelated = tmp_path / "_types.py" diff --git a/tests/hooks/test_capture_capacity_model.py b/tests/hooks/test_capture_capacity_model.py new file mode 100644 index 0000000000..260494dbb3 --- /dev/null +++ b/tests/hooks/test_capture_capacity_model.py @@ -0,0 +1,236 @@ +"""Independent boundary checks for the declared capture capacity envelope.""" + +from __future__ import annotations + +import os +from dataclasses import replace +from pathlib import Path + +import pytest + +import autoskillit.hooks._capture._artifact_setup as capture_artifact_setup +import autoskillit.hooks._capture._capacity as capture_capacity +import autoskillit.hooks._capture._lifecycle_policy as capture_lifecycle_policy +import autoskillit.hooks._capture._lifecycle_record as capture_lifecycle_record +import autoskillit.hooks._capture._reconcile as capture_reconcile +import autoskillit.hooks._capture._snapshot as capture_snapshot +import autoskillit.hooks._capture._types as capture_types +import autoskillit.hooks._capture_lifecycle as capture_lifecycle +from autoskillit.hooks._capture_artifacts import create_capture_artifact +from tests.hooks.test_capture_lifecycle import _Clock, _open_store, _verified_snapshot + +pytestmark = [pytest.mark.layer("hooks"), pytest.mark.small] + +_PROFILE_RECORDS = 976 +_PROFILE_LOW_BYTES = 3_878_096 +_PRE_ASSIST_RECORDS = 1231 + + +def _capacity_spec(**changes: int) -> capture_types.CaptureCapacitySpec: + return replace( + capture_types.CaptureCapacitySpec(), + max_operational_records=_PRE_ASSIST_RECORDS, + max_retained_records=_PRE_ASSIST_RECORDS, + max_evidence_records=_PRE_ASSIST_RECORDS, + compaction_low_bytes=_PROFILE_LOW_BYTES, + **changes, + ) + + +def test_envelope_counts_tombstones_and_pre_assist_debt() -> None: + spec = _capacity_spec() + assert capture_types.PLANNED_RECORD_FRAME_BYTES == 2608 + assert ( + _PROFILE_RECORDS + spec.reclamation_debt_assist_records - 1 + spec.max_tombstones + ) * capture_types.PLANNED_RECORD_FRAME_BYTES == _PROFILE_LOW_BYTES + assert capture_types.capacity_envelope_violations(spec) == () + assert capture_types.capacity_envelope_violations( + replace(spec, compaction_low_bytes=_PROFILE_LOW_BYTES - 1) + ) == ("compaction_low_bytes",) + + added_tombstones = 3 + shifted_requirement = _PROFILE_LOW_BYTES + ( + added_tombstones * capture_types.PLANNED_RECORD_FRAME_BYTES + ) + more_tombstones = replace(spec, max_tombstones=spec.max_tombstones + added_tombstones) + assert capture_types.capacity_envelope_violations( + replace(more_tombstones, compaction_low_bytes=shifted_requirement - 1) + ) == ("compaction_low_bytes",) + assert ( + capture_types.capacity_envelope_violations( + replace(more_tombstones, compaction_low_bytes=shifted_requirement) + ) + == () + ) + + higher_assist = replace(spec, reclamation_debt_assist_records=257) + assert set(capture_types.capacity_envelope_violations(higher_assist)) == { + "compaction_low_bytes", + "max_operational_records", + "max_retained_records", + "max_evidence_records", + } + + +@pytest.mark.parametrize( + "field", + ("max_operational_records", "max_retained_records", "max_evidence_records"), +) +def test_record_caps_follow_the_same_envelope(field: str) -> None: + changes = {field: _PRE_ASSIST_RECORDS - 1} + if field == "max_operational_records": + changes["max_retained_records"] = _PRE_ASSIST_RECORDS - 1 + assert field in capture_types.capacity_envelope_violations(_capacity_spec(**changes)) + + +def test_one_hour_success_path_hold_violates_the_envelope( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(capture_lifecycle_policy, "REFERENCE_LIFETIME_SECONDS", 3600.0) + assert capture_types.capacity_envelope_violations(capture_types.CaptureCapacitySpec()) + + +def test_override_rejected_when_it_breaks_the_envelope() -> None: + assert capture_artifact_setup._policy_capacity({"max_tombstones": 2000}) is None + assert capture_artifact_setup._policy_capacity({"compaction_low_bytes": 1_000_000}) is None + assert capture_artifact_setup._policy_capacity({"max_tombstones": 128}) is not None + + +def _close_artifact(artifact) -> None: + try: + artifact.close_artifact_fd() + finally: + artifact.release_lease() + + +def _finalize_template(store, root, clock: _Clock, capture_id: str, *, delivered: bool): + artifact = create_capture_artifact(root, capture_id, store) + try: + data = b"capacity template" + os.write(artifact.fd, data) + finalized = store.commit_verified_snapshot( + _verified_snapshot(store, artifact, data, clock), + issue_reference=False, + ) + if delivered: + store.transition_delivery( + finalized, + expected=capture_lifecycle.CaptureDeliveryStatus.NOT_ATTEMPTED, + target=capture_lifecycle.CaptureDeliveryStatus.ATTEMPTING, + ) + return store.transition_delivery( + finalized, + expected=capture_lifecycle.CaptureDeliveryStatus.ATTEMPTING, + target=capture_lifecycle.CaptureDeliveryStatus.DELIVERED, + ) + return store.get_record(capture_id) + finally: + _close_artifact(artifact) + + +def _replicate_record( + record: capture_lifecycle.CaptureLifecycleRecord, + capture_id: str, +) -> capture_lifecycle.CaptureLifecycleRecord: + assert record.manifest is not None + incarnation = f"{int(capture_id, 16):032x}" + wire = capture_snapshot.decode_capture_manifest_wire(record.manifest_bytes) + manifest = capture_snapshot._restore_capture_final_manifest( + replace( + wire, + capture_id=capture_id, + incarnation=incarnation, + carrier_name=f"shell_{capture_id}.log", + ) + ) + candidate = replace( + record, + capture_id=capture_id, + staging_name=f".capture-staging-{capture_id}-{incarnation[:16]}", + public_name=f"shell_{capture_id}.log", + incarnation=incarnation, + manifest=manifest, + manifest_bytes=capture_snapshot.encode_capture_final_manifest(manifest), + ) + capture_lifecycle_record.validate_record(candidate) + return candidate + + +def _admission_decision( + records: dict[str, capture_lifecycle.CaptureLifecycleRecord], + candidate: capture_lifecycle.CaptureLifecycleRecord, + spec: capture_types.CaptureCapacitySpec, + clock: _Clock, +): + return capture_capacity.admission_reason( + records, + candidate, + compaction_epoch=1, + spec=spec, + active_limit=min( + capture_lifecycle.MAX_ACTIVE_RECORDS, + spec.max_operational_records, + ), + sizer=capture_capacity.CompactedFrameSizer(), + now=clock.wall(), + sweep_active=False, + ) + + +def test_pre_assist_debt_is_refused_by_assist_not_bytes(tmp_path: Path) -> None: + clock = _Clock() + anchor, root, store = _open_store(tmp_path / "project", clock) + try: + held_template = _finalize_template( + store, + root, + clock, + "aaaaaaaaaaaaaaa1", + delivered=False, + ) + due_template = _finalize_template( + store, + root, + clock, + "aaaaaaaaaaaaaaa2", + delivered=True, + ) + assert held_template is not None + assert due_template is not None + assert due_template.next_attempt_at <= clock.wall() + assert store.sweep(capture_reconcile.RUNNER_TAIL_BUDGET).deleted == 1 + tombstone_template = store.get_record("aaaaaaaaaaaaaaa2") + assert tombstone_template is not None + assert tombstone_template.state is capture_lifecycle.CaptureState.DELETED + reserved = store.reserve_capture("ffffffffffffffff") + finally: + root.close() + anchor.close() + + tombstones = { + capture_id: _replicate_record(tombstone_template, capture_id) + for capture_id in (f"{index:016x}" for index in range(20_000, 20_256)) + } + held = { + capture_id: _replicate_record(held_template, capture_id) + for capture_id in (f"{index:016x}" for index in range(1, 976)) + } + due = { + capture_id: _replicate_record(due_template, capture_id) + for capture_id in (f"{index:016x}" for index in range(1_000, 1_255)) + } + spec = _capacity_spec() + baseline = tombstones | held | due + assert len(baseline) == 1486 + assert _admission_decision(baseline, reserved, spec, clock).reason is None + + one_more_due = dict(baseline) + new_due_id = "00000000000004e7" + one_more_due[new_due_id] = _replicate_record(due_template, new_due_id) + # Replace one held entry so the live-record count stays at the envelope cap. + del one_more_due["0000000000000001"] + assert len(one_more_due) == len(baseline) + decision = _admission_decision(one_more_due, reserved, spec, clock) + assert decision.reason is capture_lifecycle.CaptureCapacityReason.RECLAMATION_DEBT_ASSIST + assert decision.reason is not capture_lifecycle.CaptureCapacityReason.PROJECTED_COMPACTED_BYTES + assert decision.assist_transition_limit == 1 diff --git a/tests/hooks/test_capture_frame_bound.py b/tests/hooks/test_capture_frame_bound.py new file mode 100644 index 0000000000..7a503027ce --- /dev/null +++ b/tests/hooks/test_capture_frame_bound.py @@ -0,0 +1,199 @@ +"""Real-encoder profile that pins the capture frame planning allowance.""" + +from __future__ import annotations + +import os +from dataclasses import replace + +import pytest + +import autoskillit.hooks._capture._capacity as capture_capacity +import autoskillit.hooks._capture._failure_policy as capture_failure_policy +import autoskillit.hooks._capture._lifecycle_record as capture_lifecycle_record +import autoskillit.hooks._capture._reconcile as capture_reconcile +import autoskillit.hooks._capture._snapshot as capture_snapshot +import autoskillit.hooks._capture._types as capture_types +import autoskillit.hooks._capture_lifecycle as capture_lifecycle +from autoskillit.hooks._capture_artifacts import create_capture_artifact +from tests.hooks.test_capture_lifecycle import _Clock, _open_store, _verified_snapshot + +pytestmark = [pytest.mark.layer("hooks"), pytest.mark.medium] + +_MAX_NUMERIC = 2**63 - 1 +_COMPACTION_EPOCH = _MAX_NUMERIC +_REFERENCE_CAPTURE_ID = "0000000000000001" +_FAILURE_CAPTURE_ID = "0000000000000002" + + +def _close_artifact(artifact) -> None: + try: + artifact.close_artifact_fd() + finally: + artifact.release_lease() + + +def _profile_record( + record: capture_lifecycle.CaptureLifecycleRecord, +) -> capture_lifecycle.CaptureLifecycleRecord: + identity = (_MAX_NUMERIC, _MAX_NUMERIC) + manifest = None + manifest_bytes = b"" + finalized_at_revision = record.finalized_at_revision + if record.manifest is not None: + wire = capture_snapshot.decode_capture_manifest_wire(record.manifest_bytes) + manifest = capture_snapshot._restore_capture_final_manifest( + replace( + wire, + finalized_at_revision=_MAX_NUMERIC, + project_identity=identity, + root_identity=identity, + carrier_identity=identity, + total_bytes=_MAX_NUMERIC, + inline_length=_MAX_NUMERIC, + head_length=_MAX_NUMERIC, + tail_length=_MAX_NUMERIC, + ) + ) + manifest_bytes = capture_snapshot.encode_capture_final_manifest(manifest) + finalized_at_revision = _MAX_NUMERIC + candidate = replace( + record, + project_identity=identity, + root_identity=identity, + artifact_identity=identity if record.artifact_identity is not None else None, + observed_size=_MAX_NUMERIC, + revision=_MAX_NUMERIC, + finalized_at_revision=finalized_at_revision, + compaction_epoch=_MAX_NUMERIC, + retry_count=_MAX_NUMERIC, + manifest=manifest, + manifest_bytes=manifest_bytes, + ) + capture_lifecycle_record.validate_record(candidate) + return candidate + + +@pytest.fixture(scope="module") +def profiled_records( + tmp_path_factory: pytest.TempPathFactory, +) -> tuple[capture_lifecycle.CaptureLifecycleRecord, ...]: + clock = _Clock() + anchor, root, store = _open_store(tmp_path_factory.mktemp("capture-frame-bound"), clock) + try: + reference_artifact = create_capture_artifact( + root, + _REFERENCE_CAPTURE_ID, + store, + ) + try: + data = b"published frame" + os.write(reference_artifact.fd, data) + verified = _verified_snapshot(store, reference_artifact, data, clock) + finalized = store.commit_verified_snapshot(verified, issue_reference=True) + published = store.publish_reference(finalized) + store.transition_delivery( + published, + expected=capture_lifecycle.CaptureDeliveryStatus.NOT_ATTEMPTED, + target=capture_lifecycle.CaptureDeliveryStatus.ATTEMPTING, + ) + store.transition_delivery( + published, + expected=capture_lifecycle.CaptureDeliveryStatus.ATTEMPTING, + target=capture_lifecycle.CaptureDeliveryStatus.DELIVERED, + ) + finally: + _close_artifact(reference_artifact) + + finalized_record = store.get_record(_REFERENCE_CAPTURE_ID) + assert finalized_record is not None + assert finalized_record.state is capture_lifecycle.CaptureState.FINALIZED + profiled_finalized = _profile_record(finalized_record) + + assert finalized_record.manifest is not None + assert finalized_record.manifest.reference_expiry is not None + clock.advance(finalized_record.manifest.reference_expiry - clock.wall() + 1) + assert store.sweep(capture_reconcile.RUNNER_TAIL_BUDGET).deleted == 1 + deleted_record = store.get_record(_REFERENCE_CAPTURE_ID) + assert deleted_record is not None + assert deleted_record.state is capture_lifecycle.CaptureState.DELETED + profiled_deleted = _profile_record(deleted_record) + + failure_artifact = create_capture_artifact(root, _FAILURE_CAPTURE_ID, store) + try: + detail = capture_failure_policy.normalize_failure_detail("\x01" * 240) + assert detail == "\x01" * 240 + failed_record = store.commit_capture_failure( + failure_artifact.authority, + capture_lifecycle.CaptureFailureEvidence( + stage="frame_bound", + detail=detail, + ), + observed_size=0, + ) + finally: + _close_artifact(failure_artifact) + assert failed_record.state is capture_lifecycle.CaptureState.FAILED + profiled_failed = _profile_record(failed_record) + return profiled_finalized, profiled_deleted, profiled_failed + finally: + root.close() + anchor.close() + + +def _encoded_size(record: capture_lifecycle.CaptureLifecycleRecord) -> int: + return len( + capture_lifecycle._capture_ledger.encode_frame( + capture_lifecycle._capture_lifecycle_record.record_to_dict(record), + compaction_epoch=_COMPACTION_EPOCH, + ) + ) + + +def test_real_frame_shapes_fit_the_planning_allowance( + profiled_records: tuple[capture_lifecycle.CaptureLifecycleRecord, ...], +) -> None: + sizes = tuple(_encoded_size(record) for record in profiled_records) + measured_max = max(sizes) + violations = capture_types.capacity_envelope_violations(capture_types.CaptureCapacitySpec()) + diagnostic = ( + f"measured_max={measured_max}, planned={capture_types.PLANNED_RECORD_FRAME_BYTES}, " + f"default_low={capture_types.CaptureCapacitySpec().compaction_low_bytes}, " + f"default_violations={violations}" + ) + assert measured_max <= capture_types.PLANNED_RECORD_FRAME_BYTES <= measured_max + 64, ( + diagnostic + ) + + +def test_larger_valid_frame_is_charged_at_its_actual_size( + profiled_records: tuple[capture_lifecycle.CaptureLifecycleRecord, ...], +) -> None: + oversized = replace( + profiled_records[0], + revision=10**1000, + next_attempt_at=profiled_records[0].next_attempt_at + 3600, + ) + capture_lifecycle_record.validate_record(oversized) + encoded_size = _encoded_size(oversized) + assert encoded_size > capture_types.PLANNED_RECORD_FRAME_BYTES + + base = capture_types.CaptureCapacitySpec() + low_bytes = encoded_size - 1 + high_bytes = encoded_size + 1024 + spec = replace( + base, + compaction_low_bytes=low_bytes, + compaction_high_bytes=high_bytes, + hard_ledger_bytes=high_bytes + base.recovery_headroom_bytes, + ) + decision = capture_capacity.admission_reason( + {}, + oversized, + compaction_epoch=_COMPACTION_EPOCH, + spec=spec, + active_limit=min(capture_lifecycle.MAX_ACTIVE_RECORDS, spec.max_operational_records), + sizer=capture_capacity.CompactedFrameSizer(), + now=profiled_records[0].created_at, + sweep_active=False, + ) + assert decision.reason is capture_lifecycle.CaptureCapacityReason.PROJECTED_COMPACTED_BYTES diff --git a/tests/hooks/test_capture_lifecycle.py b/tests/hooks/test_capture_lifecycle.py index 2c57d715b5..d3f15056b2 100644 --- a/tests/hooks/test_capture_lifecycle.py +++ b/tests/hooks/test_capture_lifecycle.py @@ -53,11 +53,6 @@ DEBT_ASSIST_BUDGET, DEBT_ASSIST_MAX_TRANSITIONS, HOT_PATH_LOCK_WAIT, - MEASURED_BYTES_PER_RECORD, - REQUIRED_RETENTION_BYTES, - SUPPORTED_ADMISSIONS_PER_PRODUCER_SECOND, - SUPPORTED_PEAK_ADMISSIONS_PER_SECOND, - SUPPORTED_PEAK_PRODUCERS, TRANSITION_RESCUE_BUDGET, CaptureCapacitySpec, CaptureCleanupOutcome, @@ -944,6 +939,52 @@ def _frame_from_payload(payload: bytes) -> bytes: ) +def _opaque_frame_pair(record: CaptureLifecycleRecord) -> tuple[bytes, bytes]: + epoch = record.compaction_epoch + future = capture_lifecycle._capture_lifecycle_record.record_to_dict( + replace(record, revision=record.revision + 1) + ) + future["state"] = "future-state" + opaque = capture_lifecycle._capture_ledger.encode_frame( + future, + compaction_epoch=epoch, + ) + later = capture_lifecycle._capture_ledger.encode_frame( + capture_lifecycle._capture_lifecycle_record.record_to_dict( + replace(record, revision=record.revision + 2) + ), + compaction_epoch=epoch, + ) + return opaque, later + + +def _append_opaque_frames(store: CaptureLifecycleStore, frames: tuple[bytes, ...]) -> None: + with store._locked(): + fd = store._open_ledger() + try: + os.lseek(fd, 0, os.SEEK_END) + capture_lifecycle._capture_ledger.write_all(fd, b"".join(frames)) + os.fsync(fd) + finally: + os.close(fd) + + +def _encoded_compacted_bytes( + records: dict[str, CaptureLifecycleRecord], + compaction_epoch: int, + capacity: CaptureCapacitySpec, +) -> int: + return sum( + len( + capture_lifecycle._capture_ledger.encode_frame( + capture_lifecycle._capture_lifecycle_record.record_to_dict(record), + compaction_epoch=compaction_epoch, + ) + ) + for record in capture_capacity.compacted_records(records, capacity) + ) + + def _legacy_frame(record: dict[str, object], *, generation: int = 1) -> bytes: payload = capture_lifecycle._capture_lifecycle_record.canonical_json( { @@ -1708,6 +1749,211 @@ def test_delivery_transition_rejects_illegal_predecessor_target_pair( anchor.close() +@pytest.mark.parametrize( + ("delivery_status", "expected_reference_status", "expected_delivery_status"), + [ + pytest.param( + CaptureDeliveryStatus.ATTEMPTING, + CaptureReferenceStatus.PUBLISHED, + CaptureDeliveryStatus.UNKNOWN, + id="attempting", + ), + pytest.param( + CaptureDeliveryStatus.NOT_ATTEMPTED, + CaptureReferenceStatus.UNAVAILABLE, + CaptureDeliveryStatus.NOT_ATTEMPTED, + id="published-not-attempted", + ), + ], +) +def test_store_open_normalization_succeeds_above_soft_limit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + delivery_status: CaptureDeliveryStatus, + expected_reference_status: CaptureReferenceStatus, + expected_delivery_status: CaptureDeliveryStatus, +) -> None: + project = tmp_path / "project" + clock = _Clock(1_000.0) + due_capture_id = "1111111111111111" + with ExitStack() as cleanup: + anchor, root, store = _open_store(project, clock) + cleanup.callback(anchor.close) + cleanup.callback(root.close) + + due_artifact = create_capture_artifact(root, due_capture_id, store) + cleanup.callback(due_artifact.release_lease) + cleanup.callback(due_artifact.close_artifact_fd) + os.write(due_artifact.fd, b"due") + _commit_verified(store, due_artifact, b"due", clock) + + clock.advance(100_000.0) + crash_artifact = create_capture_artifact(root, _CAPTURE_ID, store) + cleanup.callback(crash_artifact.release_lease) + cleanup.callback(crash_artifact.close_artifact_fd) + os.write(crash_artifact.fd, b"crash-left") + finalized = store.commit_verified_snapshot( + _verified_snapshot(store, crash_artifact, b"crash-left", clock), + issue_reference=True, + ) + published = store.publish_reference(finalized) + if delivery_status is CaptureDeliveryStatus.ATTEMPTING: + store.transition_delivery( + published, + expected=CaptureDeliveryStatus.NOT_ATTEMPTED, + target=CaptureDeliveryStatus.ATTEMPTING, + ) + crash_record = store.get_record(_CAPTURE_ID) + due_record = store.get_record(due_capture_id) + assert crash_record is not None + assert due_record is not None + assert crash_record.reference_status is CaptureReferenceStatus.PUBLISHED + assert crash_record.delivery_status is delivery_status + assert due_record.next_attempt_at <= clock.wall() < crash_record.next_attempt_at + + with store._locked(): + records, compaction_epoch, _size = store._load_locked() + projected = store._ledger_view.sizer.total_bytes( + records, + compaction_epoch, + store._capacity, + ) + base_capacity = store._capacity + scaled_capacity = replace( + base_capacity, + compaction_low_bytes=projected - 1, + compaction_high_bytes=projected, + hard_ledger_bytes=projected + base_capacity.recovery_headroom_bytes + 64, + ) + assert scaled_capacity.compaction_low_bytes < projected + assert ( + scaled_capacity.hard_ledger_bytes - scaled_capacity.recovery_headroom_bytes + >= projected + 64 + ) + cleanup.close() + + @contextmanager + def open_scaled_store( + requested_cwd: str, + *, + create: bool = False, + lock_wait: LockWaitSpec, + ) -> Iterator[CaptureLifecycleStore]: + opened_anchor = open_project_anchor(requested_cwd) + opened_root: CaptureRoot | None = None + try: + opened_root = open_capture_root(opened_anchor, create=create) + yield CaptureLifecycleStore.from_open_authorities( + opened_anchor, + opened_root, + wall_clock=clock.wall, + monotonic=clock.monotonic, + lock_wait=lock_wait, + capacity=scaled_capacity, + ) + finally: + try: + if opened_root is not None: + opened_root.close() + finally: + opened_anchor.close() + + monkeypatch.setattr( + capture_reconcile._authority, + "open_capture_lifecycle", + open_scaled_store, + ) + outcome = capture_reconcile.reconcile_capture_store( + str(project), + capture_reconcile.RUNNER_TAIL_BUDGET, + ) + + assert outcome.deleted == 1 + assert outcome.progress is CleanupProgress.RETIRED + with open_scaled_store( + str(project), + create=False, + lock_wait=HOT_PATH_LOCK_WAIT, + ) as reopened: + normalized = reopened.get_record(_CAPTURE_ID) + due = reopened.get_record(due_capture_id) + + assert normalized is not None + assert normalized.state is CaptureState.FINALIZED + assert normalized.reference_status is expected_reference_status + assert normalized.delivery_status is expected_delivery_status + assert normalized.next_attempt_at > clock.wall() + assert due is not None and due.state is CaptureState.DELETED + + +def test_state_advancing_transition_above_soft_limit_is_refused( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _Clock() + anchor, root, store = _open_store(tmp_path / "project", clock) + artifact = create_capture_artifact(root, _CAPTURE_ID, store) + try: + os.write(artifact.fd, b"captured") + snapshot = _verified_snapshot(store, artifact, b"captured", clock) + writing = store.get_record(_CAPTURE_ID) + assert writing is not None + assert writing.state is CaptureState.PUBLISHED_WRITING + scaled_capacity = replace( + store._capacity, + compaction_low_bytes=1, + compaction_high_bytes=2, + ) + monkeypatch.setattr(store, "_capacity", scaled_capacity) + + with pytest.raises(CaptureCapacityError) as failure: + store.commit_verified_snapshot(snapshot, issue_reference=False) + + assert failure.value.reason is CaptureCapacityReason.PROJECTED_COMPACTED_BYTES + current = store.get_record(_CAPTURE_ID) + assert current is not None and current.state is CaptureState.PUBLISHED_WRITING + finally: + artifact.close_artifact_fd() + artifact.release_lease() + root.close() + anchor.close() + + +def test_recovery_transitions_pass_between_low_and_hard( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _Clock() + project = tmp_path / "project" + anchor, root, store = _open_store(project, clock) + artifact = create_capture_artifact(root, _CAPTURE_ID, store) + try: + os.write(artifact.fd, b"captured") + _commit_verified(store, artifact, b"captured", clock) + artifact.close_artifact_fd() + artifact.release_lease() + clock.advance(SWEEP_GRACE_SECONDS + 1.0) + scaled_capacity = replace( + store._capacity, + compaction_low_bytes=1, + compaction_high_bytes=2, + ) + monkeypatch.setattr(store, "_capacity", scaled_capacity) + + outcome = store.sweep(capture_reconcile.RUNNER_TAIL_BUDGET) + + record = store.get_record(_CAPTURE_ID) + assert outcome.deleted == 1 + assert outcome.progress is CleanupProgress.RETIRED + assert record is not None and record.state is CaptureState.DELETED + assert not (project.joinpath(*CAPTURE_PATH_COMPONENTS) / artifact.name).exists() + finally: + artifact.close_artifact_fd() + artifact.release_lease() + root.close() + anchor.close() + + def test_delivered_reference_cannot_be_reclassified_as_unavailable( tmp_path: Path, ) -> None: @@ -4852,6 +5098,167 @@ def test_unknown_lifecycle_enum_frames_survive_incremental_load_and_compaction( anchor.close() +def test_admission_and_compaction_agree_on_opaque_frames( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + project = tmp_path / "project" + clock = _Clock() + anchor, root, seed_store = _open_store(project, clock) + try: + seed_store.reserve_capture(_CAPTURE_ID) + source_a = seed_store.reserve_capture("1" * 16) + seed_store.reserve_capture("2" * 16) + first_pair = _opaque_frame_pair(source_a) + _append_opaque_frames(seed_store, first_pair) + + base_store = CaptureLifecycleStore.from_open_authorities( + anchor, + root, + wall_clock=clock.wall, + monotonic=clock.monotonic, + lock_wait=HOT_PATH_LOCK_WAIT, + ) + with base_store._locked(): + initial_records, initial_epoch, _size = base_store._load_locked() + assert base_store._ledger_view.opaque_frames == first_pair + initial_opaque_bytes = sum(map(len, first_pair)) + assert ( + base_store._ledger_view.sizer.total_bytes( + initial_records, + initial_epoch, + base_store._capacity, + ) + == _encoded_compacted_bytes( + initial_records, + initial_epoch, + base_store._capacity, + ) + + initial_opaque_bytes + ) + + other_store = CaptureLifecycleStore.from_open_authorities( + anchor, + root, + wall_clock=clock.wall, + monotonic=clock.monotonic, + lock_wait=HOT_PATH_LOCK_WAIT, + ) + source_b = other_store.get_record("2" * 16) + assert source_b is not None + second_pair = _opaque_frame_pair(source_b) + _append_opaque_frames(other_store, second_pair) + expected_opaque_frames = first_pair + second_pair + expected_opaque_bytes = sum(map(len, expected_opaque_frames)) + + full_store = CaptureLifecycleStore.from_open_authorities( + anchor, + root, + wall_clock=clock.wall, + monotonic=clock.monotonic, + lock_wait=HOT_PATH_LOCK_WAIT, + ) + with full_store._locked(): + full_records, full_epoch, _size = full_store._load_locked() + assert full_store._ledger_view.opaque_frames == expected_opaque_frames + assert ( + full_store._ledger_view.sizer.total_bytes( + full_records, + full_epoch, + full_store._capacity, + ) + == _encoded_compacted_bytes( + full_records, + full_epoch, + full_store._capacity, + ) + + expected_opaque_bytes + ) + + full_reload_count = 0 + real_decode_full = capture_ledger_view._decode_full + + def count_full_reload(*args, **kwargs): + nonlocal full_reload_count + full_reload_count += 1 + return real_decode_full(*args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(capture_ledger_view, "_decode_full", count_full_reload) + with base_store._locked(): + records, compaction_epoch, _size = base_store._load_locked() + + assert full_reload_count == 0 + assert base_store._ledger_view.opaque_frames == expected_opaque_frames + assert ( + base_store._ledger_view.sizer.total_bytes( + records, + compaction_epoch, + base_store._capacity, + ) + == _encoded_compacted_bytes( + records, + compaction_epoch, + base_store._capacity, + ) + + expected_opaque_bytes + ) + + candidate_id = "f" * 16 + candidate = CaptureLifecycleRecord( + capture_id=candidate_id, + state=CaptureState.RESERVED, + staging_name=f".capture-staging-{candidate_id}-{'0' * 16}", + public_name=f"shell_{candidate_id}.log", + project_identity=base_store._project_identity, + root_identity=base_store._root_identity, + created_at=clock.wall(), + next_attempt_at=clock.wall() + SWEEP_GRACE_SECONDS, + incarnation="e" * 32, + revision=1, + ) + projected_records = {**records, candidate_id: candidate} + visible_projection = _encoded_compacted_bytes( + projected_records, + compaction_epoch, + base_store._capacity, + ) + projection_with_opaque = base_store._ledger_view.sizer.total_bytes( + projected_records, + compaction_epoch, + base_store._capacity, + ) + low = visible_projection + expected_opaque_bytes // 2 + assert visible_projection < low < projection_with_opaque + scaled_capacity = replace( + base_store._capacity, + compaction_low_bytes=low, + compaction_high_bytes=low + 1, + ) + monkeypatch.setattr(base_store, "_capacity", scaled_capacity) + + with pytest.raises(CaptureCapacityError) as failure: + base_store.reserve_capture(candidate_id) + + assert failure.value.reason is CaptureCapacityReason.PROJECTED_COMPACTED_BYTES + assert base_store.get_record(candidate_id) is None + with base_store._locked(): + records, compaction_epoch, _size = base_store._load_locked() + base_store._compact_locked(records, compaction_epoch + 1) + records, compaction_epoch, _size = base_store._load_locked() + assert base_store._ledger_view.opaque_frames == expected_opaque_frames + assert base_store._ledger_view.sizer.total_bytes( + records, + compaction_epoch, + scaled_capacity, + ) == _encoded_compacted_bytes(records, compaction_epoch, scaled_capacity) + ( + expected_opaque_bytes + ) + finally: + root.close() + anchor.close() + + def test_future_ledger_format_reports_observed_and_current_versions( tmp_path: Path, ) -> None: @@ -5509,17 +5916,6 @@ def observe_capacity_rescue(*args, **kwargs): anchor.close() -def test_capacity_window_covers_the_supported_retention_envelope() -> None: - expected_admission_rate = SUPPORTED_PEAK_PRODUCERS * SUPPORTED_ADMISSIONS_PER_PRODUCER_SECOND - expected_retention_bytes = math.ceil( - expected_admission_rate * SWEEP_GRACE_SECONDS * MEASURED_BYTES_PER_RECORD - ) - - assert SUPPORTED_PEAK_ADMISSIONS_PER_SECOND == expected_admission_rate - assert REQUIRED_RETENTION_BYTES == expected_retention_bytes - assert CaptureCapacitySpec().compaction_low_bytes >= REQUIRED_RETENTION_BYTES - - def test_installed_reclamation_owners_include_a_directory_scan_budget() -> None: installed_owner_budgets = ( capture_reconcile.RUNNER_TAIL_BUDGET, From 8a3a018ce3f8a16aa71b533f07f23147ff55b126 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 16:49:08 -0700 Subject: [PATCH 03/21] test: correct capacity boundary and frame profile fixtures --- tests/hooks/test_capture_capacity_model.py | 4 ++-- tests/hooks/test_capture_frame_bound.py | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/hooks/test_capture_capacity_model.py b/tests/hooks/test_capture_capacity_model.py index 260494dbb3..8ac6d319fa 100644 --- a/tests/hooks/test_capture_capacity_model.py +++ b/tests/hooks/test_capture_capacity_model.py @@ -27,14 +27,14 @@ def _capacity_spec(**changes: int) -> capture_types.CaptureCapacitySpec: - return replace( + base = replace( capture_types.CaptureCapacitySpec(), max_operational_records=_PRE_ASSIST_RECORDS, max_retained_records=_PRE_ASSIST_RECORDS, max_evidence_records=_PRE_ASSIST_RECORDS, compaction_low_bytes=_PROFILE_LOW_BYTES, - **changes, ) + return replace(base, **changes) def test_envelope_counts_tombstones_and_pre_assist_debt() -> None: diff --git a/tests/hooks/test_capture_frame_bound.py b/tests/hooks/test_capture_frame_bound.py index 7a503027ce..1ff4ba5d40 100644 --- a/tests/hooks/test_capture_frame_bound.py +++ b/tests/hooks/test_capture_frame_bound.py @@ -52,20 +52,36 @@ def _profile_record( inline_length=_MAX_NUMERIC, head_length=_MAX_NUMERIC, tail_length=_MAX_NUMERIC, + command_outcome_value=( + 255 + if wire.command_outcome_kind is capture_snapshot.CommandOutcomeKind.EXITED + else 127 + ), ) ) manifest_bytes = capture_snapshot.encode_capture_final_manifest(manifest) finalized_at_revision = _MAX_NUMERIC + failure = record.failure + if failure is not None: + failure = replace( + failure, + stage="a" + "b" * (capture_failure_policy.FAILURE_STAGE_MAX_BYTES - 1), + settlement_returncode=255, + failure_reason=( + capture_failure_policy.CaptureFailureReason.PROJECTED_COMPACTED_BYTES_EXHAUSTED + ).value, + ) candidate = replace( record, project_identity=identity, root_identity=identity, - artifact_identity=identity if record.artifact_identity is not None else None, + artifact_identity=identity, observed_size=_MAX_NUMERIC, revision=_MAX_NUMERIC, finalized_at_revision=finalized_at_revision, compaction_epoch=_MAX_NUMERIC, retry_count=_MAX_NUMERIC, + failure=failure, manifest=manifest, manifest_bytes=manifest_bytes, ) From 51d5b8ca6e11e6f48cae1136116f5c15606b2217 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 17:11:11 -0700 Subject: [PATCH 04/21] feat: report capture capacity headroom and refusal measurements --- docs/safety/hooks.md | 29 +++- .../cli/doctor/_doctor_capture_store.py | 20 ++- src/autoskillit/cli/ops/_capture_store.py | 15 +- .../hooks/_capture/_artifact_setup.py | 1 + src/autoskillit/hooks/_capture/_authority.py | 11 +- src/autoskillit/hooks/_capture/_capacity.py | 52 +++++- src/autoskillit/hooks/_capture/_reconcile.py | 23 ++- src/autoskillit/hooks/_capture/_runner.py | 7 +- .../hooks/_capture_lifecycle/_errors.py | 9 +- .../hooks/_capture_lifecycle/_transactions.py | 1 + tests/cli/test_capture_store.py | 156 +++++++++++++++++- tests/hooks/test_capture_artifacts.py | 35 ++++ 12 files changed, 336 insertions(+), 23 deletions(-) diff --git a/docs/safety/hooks.md b/docs/safety/hooks.md index 0bb0807af6..6aa19114ac 100644 --- a/docs/safety/hooks.md +++ b/docs/safety/hooks.md @@ -382,11 +382,30 @@ the kernel to block indefinitely. `hooks._capture._reconcile.capture_store_stats()` is the single read-only adapter both the doctor battery's capture-store check and `autoskillit capture-store` (without `--reclaim`) call, so neither surface can drift from -what a real reconciliation pass would find. `autoskillit capture-store ---reclaim` loops `reconcile_capture_store` with a generous one-time -`RECLAIM_BUDGET` until a clean pass — no due records, no adoptable orphans — -or a hard iteration cap. It exists for bulk pre-existing backlog; the -SessionStart scan phase above keeps new debris from ever accumulating again. +what a real reconciliation pass would find. Its capacity line appends +`projected_compacted_bytes=… low_bytes=… utilization_pct=…` and the label +`(default capacity limits; runtime overrides are runner-local)`. Doctor and the +CLI emit a WARNING when the soft limit is positive and projected compacted +bytes reach 80% of it: `projected_compacted_bytes / compaction_low_bytes >= 0.80`. + +`ledger_bytes` is the physical ledger size and includes superseded transition +frames until compaction. `projected_compacted_bytes` is the admission quantity +and the numerator for the warning; it measures the compacted projection of +latest records, tombstones, and opaque frames. These values answer different +questions. An existing physical ledger above `MAX_LEDGER_BYTES` fails store +load as `CaptureLedgerError`; a projected compacted-capacity refusal is a +`CaptureCapacityError` with the capacity handling described in +[ADR-0006](../decisions/0006-output-containment.md). + +The temporary output-budget overlay workaround is removed on every +`close_kitchen`. To disable the guard durably, set +`output_budget: {guard_enabled: false}` in `.autoskillit/config.yaml`. + +`autoskillit capture-store --reclaim` loops `reconcile_capture_store` with a +generous one-time `RECLAIM_BUDGET` until a clean pass — no due records, no +adoptable orphans — or a hard iteration cap. It exists for bulk pre-existing +backlog; the SessionStart scan phase above keeps new debris from ever +accumulating again. ### `generated_file_write_guard.py` **Guarded tools:** `Write`, `Edit` diff --git a/src/autoskillit/cli/doctor/_doctor_capture_store.py b/src/autoskillit/cli/doctor/_doctor_capture_store.py index d93ca821a1..1c16ee5a19 100644 --- a/src/autoskillit/cli/doctor/_doctor_capture_store.py +++ b/src/autoskillit/cli/doctor/_doctor_capture_store.py @@ -5,7 +5,7 @@ from pathlib import Path from autoskillit.core import Severity -from autoskillit.hooks import CleanupBlocker, capture_store_stats +from autoskillit.hooks import CaptureStoreStats, CleanupBlocker, capture_store_stats from ._doctor_types import DoctorResult @@ -19,6 +19,19 @@ _RECLAMATION_DEBT_WARNING_THRESHOLD = 256 +def _capacity_line(stats: CaptureStoreStats) -> tuple[bool, str]: + utilization_pct = ( + f"{stats.projected_compacted_bytes / stats.compaction_low_bytes:.1%}" + if stats.compaction_low_bytes > 0 + else "n/a" + ) + return stats.capacity_warning, ( + f"capacity: projected_compacted_bytes={stats.projected_compacted_bytes} " + f"low_bytes={stats.compaction_low_bytes} utilization_pct={utilization_pct} " + "(default capacity limits; runtime overrides are runner-local)" + ) + + def _check_capture_store_stats(project_dir: Path | None = None) -> DoctorResult: """Report capture-store ledger/directory statistics — read-only, never reclaims. @@ -45,8 +58,11 @@ def _check_capture_store_stats(project_dir: Path | None = None) -> DoctorResult: f"unledgered_aged={stats.unledgered_aged_files} " f"unledgered_aged_bytes={stats.unledgered_aged_bytes}" ) + capacity_warning, capacity_message = _capacity_line(stats) + message = f"{message}; {capacity_message}" if ( - stats.unledgered_aged_files >= _UNLEDGERED_WARNING_THRESHOLD + capacity_warning + or stats.unledgered_aged_files >= _UNLEDGERED_WARNING_THRESHOLD or stats.due_records >= _RECLAMATION_DEBT_WARNING_THRESHOLD ): return DoctorResult( diff --git a/src/autoskillit/cli/ops/_capture_store.py b/src/autoskillit/cli/ops/_capture_store.py index 378508d161..f4b9d326c1 100644 --- a/src/autoskillit/cli/ops/_capture_store.py +++ b/src/autoskillit/cli/ops/_capture_store.py @@ -56,13 +56,26 @@ def _print_stats(stats: CaptureStoreStats) -> None: "capture-store ledger: " f"live={stats.live_records} due_records={stats.due_records} " f"eligible_abandoned={stats.eligible_records} " - f"deleting={stats.deleting_records} ledger_bytes={stats.ledger_bytes}" + f"deleting={stats.deleting_records} physical_ledger_bytes={stats.ledger_bytes}" ) print( "capture-store directory: " f"files={stats.directory_files} unledgered_aged={stats.unledgered_aged_files} " f"unledgered_aged_bytes={stats.unledgered_aged_bytes}" ) + utilization_pct = ( + f"{stats.projected_compacted_bytes / stats.compaction_low_bytes:.1%}" + if stats.compaction_low_bytes > 0 + else "n/a" + ) + print( + "capture-store capacity: " + f"projected_compacted_bytes={stats.projected_compacted_bytes} " + f"low_bytes={stats.compaction_low_bytes} utilization_pct={utilization_pct} " + "(default capacity limits; runtime overrides are runner-local)" + ) + if stats.capacity_warning: + print("WARNING: compacted capture-store bytes have reached 80% of the soft limit") def run_capture_store(*, reclaim: bool = False) -> None: diff --git a/src/autoskillit/hooks/_capture/_artifact_setup.py b/src/autoskillit/hooks/_capture/_artifact_setup.py index a34535e43f..cb0a153999 100644 --- a/src/autoskillit/hooks/_capture/_artifact_setup.py +++ b/src/autoskillit/hooks/_capture/_artifact_setup.py @@ -168,6 +168,7 @@ def create_capture_artifact( raise CaptureSetupError( exc.failure_reason, "capture capacity is exhausted", + public_detail=str(exc) if exc.decision is not None else None, ) from exc except (CaptureLifecycleError, CaptureTransitionCommittedError) as exc: raise CaptureSetupError( diff --git a/src/autoskillit/hooks/_capture/_authority.py b/src/autoskillit/hooks/_capture/_authority.py index a77a9dce14..b419a5769d 100644 --- a/src/autoskillit/hooks/_capture/_authority.py +++ b/src/autoskillit/hooks/_capture/_authority.py @@ -79,11 +79,20 @@ class CaptureSetupError(RuntimeError): """Raised when the descriptor-anchored capture authority cannot be established.""" - def __init__(self, reason: CaptureFailureReason, detail: str) -> None: + def __init__( + self, + reason: CaptureFailureReason, + detail: str, + *, + public_detail: str | None = None, + ) -> None: if type(reason) is not CaptureFailureReason: raise TypeError("capture setup reason must be closed") self.reason = reason self.detail = normalize_failure_detail(detail) + self.public_detail = ( + normalize_failure_detail(public_detail) if public_detail is not None else None + ) super().__init__(self.detail) @classmethod diff --git a/src/autoskillit/hooks/_capture/_capacity.py b/src/autoskillit/hooks/_capture/_capacity.py index 3bcb44ada0..7ef1a2d630 100644 --- a/src/autoskillit/hooks/_capture/_capacity.py +++ b/src/autoskillit/hooks/_capture/_capacity.py @@ -7,7 +7,11 @@ from enum import StrEnum from . import _ledger, _sweep_cursor -from ._failure_policy import CAPACITY_FAILURE_REASONS, CaptureFailureReason +from ._failure_policy import ( + CAPACITY_FAILURE_REASONS, + CaptureFailureReason, + normalize_failure_detail, +) from ._module_identity import register_module_aliases from ._types import DEBT_ASSIST_MAX_TRANSITIONS, CaptureCapacityReason, CaptureCapacitySpec @@ -61,6 +65,9 @@ class AdmissionDecision: reason: CaptureCapacityReason | None assist_transition_limit: int | None = None + projected_bytes: int | None = None + limit_bytes: int | None = None + due_records: int | None = None class CompactedFrameSizer: @@ -156,6 +163,26 @@ def reason_detail(reason: CaptureCapacityReason) -> str: return _REASON_DETAILS[reason] +def measurement_detail(decision: AdmissionDecision) -> str: + if decision.reason is None: + raise ValueError("capacity measurement requires a refusal decision") + measurements = " ".join( + value + for value in ( + f"projected={decision.projected_bytes}" + if decision.projected_bytes is not None + else None, + f"limit={decision.limit_bytes}" if decision.limit_bytes is not None else None, + f"due={decision.due_records}" if decision.due_records is not None else None, + ) + if value is not None + ) + detail = reason_detail(decision.reason) + if measurements: + detail = f"{detail}: {measurements}" + return normalize_failure_detail(detail) + + def failure_reason(reason: CaptureCapacityReason) -> CaptureFailureReason: return _FAILURE_REASONS[reason] @@ -221,18 +248,28 @@ def admission_reason( forensic += is_forensic due += _sweep_cursor.is_due_record(record, now, terminal_states) if operational > active_limit: - return AdmissionDecision(CaptureCapacityReason.ACTIVE_CAPACITY) + return AdmissionDecision(CaptureCapacityReason.ACTIVE_CAPACITY, due_records=due) if retained > spec.max_retained_records: - return AdmissionDecision(CaptureCapacityReason.RETENTION_CAPACITY) + return AdmissionDecision(CaptureCapacityReason.RETENTION_CAPACITY, due_records=due) if operational + forensic > spec.max_evidence_records: - return AdmissionDecision(CaptureCapacityReason.EVIDENCE_CAPACITY) + return AdmissionDecision(CaptureCapacityReason.EVIDENCE_CAPACITY, due_records=due) encoded = sizer.total_bytes(projected, compaction_epoch, spec) if encoded + spec.recovery_headroom_bytes > spec.hard_ledger_bytes: - return AdmissionDecision(CaptureCapacityReason.HARD_LEDGER_CAPACITY) + return AdmissionDecision( + CaptureCapacityReason.HARD_LEDGER_CAPACITY, + projected_bytes=encoded, + limit_bytes=spec.hard_ledger_bytes - spec.recovery_headroom_bytes, + due_records=due, + ) if encoded > spec.compaction_low_bytes: - return AdmissionDecision(CaptureCapacityReason.PROJECTED_COMPACTED_BYTES) + return AdmissionDecision( + CaptureCapacityReason.PROJECTED_COMPACTED_BYTES, + projected_bytes=encoded, + limit_bytes=spec.compaction_low_bytes, + due_records=due, + ) if due >= spec.reclamation_debt_stall_records: - return AdmissionDecision(CaptureCapacityReason.RECLAMATION_DEBT_STALL) + return AdmissionDecision(CaptureCapacityReason.RECLAMATION_DEBT_STALL, due_records=due) if due >= spec.reclamation_debt_assist_records and not sweep_active: return AdmissionDecision( CaptureCapacityReason.RECLAMATION_DEBT_ASSIST, @@ -240,6 +277,7 @@ def admission_reason( DEBT_ASSIST_MAX_TRANSITIONS, due - spec.reclamation_debt_assist_records + 1, ), + due_records=due, ) return AdmissionDecision(None) diff --git a/src/autoskillit/hooks/_capture/_reconcile.py b/src/autoskillit/hooks/_capture/_reconcile.py index b19649d930..dfa2434309 100644 --- a/src/autoskillit/hooks/_capture/_reconcile.py +++ b/src/autoskillit/hooks/_capture/_reconcile.py @@ -15,6 +15,7 @@ from ._module_identity import register_module_aliases from ._syntax import PUBLIC_NAME_RE from ._types import ( + CaptureCapacitySpec, CaptureCleanupOutcome, CleanupBlocker, CleanupProgress, @@ -49,6 +50,9 @@ # Single shared byte bound for every capture-cleanup diagnostic, regardless of # owner. DIAGNOSTIC_MAX_BYTES = 400 +# Warn as compacted bytes approach the default soft limit; runtime capacity +# overrides remain runner-local. +_CAPACITY_WARNING_RATIO = 0.80 _HOOK_VERSION = 1 _POLICY_EVENT_NAME = "capture_cleanup" @@ -164,6 +168,17 @@ class CaptureStoreStats: directory_files: int = 0 unledgered_aged_files: int = 0 unledgered_aged_bytes: int = 0 + projected_compacted_bytes: int = 0 + compaction_low_bytes: int = CaptureCapacitySpec().compaction_low_bytes + + @property + def capacity_warning(self) -> bool: + """Whether the compacted projection reaches the soft-limit warning.""" + return ( + self.compaction_low_bytes > 0 + and self.projected_compacted_bytes + >= _CAPACITY_WARNING_RATIO * self.compaction_low_bytes + ) def capture_store_stats(project_cwd: str) -> CaptureStoreStats: @@ -186,7 +201,11 @@ def capture_store_stats(project_cwd: str) -> CaptureStoreStats: lock_wait=lock_wait_for_budget(RUNNER_TAIL_BUDGET), ) as store: with store._locked(): - records, _compaction_epoch, ledger_bytes = store._load_locked() + records, compaction_epoch, ledger_bytes = store._load_locked() + projected_compacted_bytes = store._ledger_view.sizer.total_bytes( + records, compaction_epoch, store._capacity + ) + compaction_low_bytes = store._capacity.compaction_low_bytes tracked = frozenset( record.public_name for record in records.values() @@ -235,6 +254,8 @@ def capture_store_stats(project_cwd: str) -> CaptureStoreStats: directory_files=directory_files, unledgered_aged_files=unledgered_aged_files, unledgered_aged_bytes=unledgered_aged_bytes, + projected_compacted_bytes=projected_compacted_bytes, + compaction_low_bytes=compaction_low_bytes, ) except _authority.CaptureStoreAbsentError: return CaptureStoreStats(CleanupBlocker.STORE_ABSENT) diff --git a/src/autoskillit/hooks/_capture/_runner.py b/src/autoskillit/hooks/_capture/_runner.py index 9579081676..3e09d079f9 100644 --- a/src/autoskillit/hooks/_capture/_runner.py +++ b/src/autoskillit/hooks/_capture/_runner.py @@ -247,6 +247,11 @@ def run_capture( artifact_writer_fd = _duplicate_artifact_writer(artifact) except BaseException as exc: reason = _capture_failure_policy.runtime_failure_reason(exc) + detail = ( + exc.public_detail + if isinstance(exc, CaptureSetupError) and exc.public_detail + else f"{failure_stage} failed" + ) failure = _capture_replay.failure_transport( reason=reason, stage=( @@ -254,7 +259,7 @@ def run_capture( if reason in _capture_failure_policy.CAPACITY_FAILURE_REASONS else failure_stage ), - detail=f"{failure_stage} failed", + detail=detail, shell_returncode=None, settlement=None, ) diff --git a/src/autoskillit/hooks/_capture_lifecycle/_errors.py b/src/autoskillit/hooks/_capture_lifecycle/_errors.py index 0b8dc5a898..50c9bb9db3 100644 --- a/src/autoskillit/hooks/_capture_lifecycle/_errors.py +++ b/src/autoskillit/hooks/_capture_lifecycle/_errors.py @@ -52,8 +52,15 @@ def __init__( self, reason: CaptureCapacityReason, assist_transition_limit: int | None = None, + decision: _capture_capacity.AdmissionDecision | None = None, ) -> None: self.reason = reason self.assist_transition_limit = assist_transition_limit + self.decision = decision self.failure_reason = _capture_capacity.failure_reason(reason) - super().__init__(_capture_capacity.reason_detail(reason)) + detail = ( + _capture_capacity.measurement_detail(decision) + if decision is not None + else _capture_capacity.reason_detail(reason) + ) + super().__init__(detail) diff --git a/src/autoskillit/hooks/_capture_lifecycle/_transactions.py b/src/autoskillit/hooks/_capture_lifecycle/_transactions.py index 331c5d4c8d..f0d6fbdeba 100644 --- a/src/autoskillit/hooks/_capture_lifecycle/_transactions.py +++ b/src/autoskillit/hooks/_capture_lifecycle/_transactions.py @@ -385,6 +385,7 @@ def _attempt() -> CaptureLifecycleRecord: raise CaptureCapacityError( decision.reason, decision.assist_transition_limit, + decision=decision, ) store._append_locked(record, records, compaction_epoch, size) return record diff --git a/tests/cli/test_capture_store.py b/tests/cli/test_capture_store.py index 8bf0fa6f2d..056a3dd6dd 100644 --- a/tests/cli/test_capture_store.py +++ b/tests/cli/test_capture_store.py @@ -35,9 +35,17 @@ CommandOutcome, verify_capture_snapshot, ) -from autoskillit.hooks._capture._types import HOT_PATH_LOCK_WAIT, CleanupBlocker +from autoskillit.hooks._capture._types import ( + HOT_PATH_LOCK_WAIT, + CaptureCapacitySpec, + CleanupBlocker, +) from autoskillit.hooks._capture_artifacts import CaptureArtifact, create_capture_artifact -from autoskillit.hooks._capture_lifecycle import LOCK_NAME, CaptureLifecycleStore +from autoskillit.hooks._capture_lifecycle import ( + LOCK_NAME, + CaptureDeliveryStatus, + CaptureLifecycleStore, +) from tests.conftest import production_interpreter_env pytestmark = [pytest.mark.layer("cli"), pytest.mark.medium] @@ -142,7 +150,17 @@ def test_capture_store_stats_reports_due_finalized_record_to_operators( finalized_at=finalized_at, retention_deadline=finalized_at + SWEEP_GRACE_SECONDS, ) - lifecycle.commit_verified_snapshot(verified, issue_reference=False) + finalized = lifecycle.commit_verified_snapshot(verified, issue_reference=False) + lifecycle.transition_delivery( + finalized, + expected=CaptureDeliveryStatus.NOT_ATTEMPTED, + target=CaptureDeliveryStatus.ATTEMPTING, + ) + lifecycle.transition_delivery( + finalized, + expected=CaptureDeliveryStatus.ATTEMPTING, + target=CaptureDeliveryStatus.FAILED, + ) finally: if artifact is not None: artifact.close_artifact_fd() @@ -152,12 +170,48 @@ def test_capture_store_stats_reports_due_finalized_record_to_operators( stats = capture_store_stats(str(project)) + projection_anchor = open_project_anchor(str(project)) + projection_root = open_capture_root(projection_anchor, create=False) + try: + projection_store = CaptureLifecycleStore.from_open_authorities( + projection_anchor, + projection_root, + lock_wait=HOT_PATH_LOCK_WAIT, + ) + with projection_store._locked(): + records, epoch, _physical_size = projection_store._load_locked() + expected_projection = projection_store._ledger_view.sizer.total_bytes( + records, + epoch, + projection_store._capacity, + ) + finally: + projection_root.close() + projection_anchor.close() + assert stats.due_records == 1 assert stats.eligible_records == 0 + assert stats.projected_compacted_bytes == expected_projection + assert stats.compaction_low_bytes == CaptureCapacitySpec().compaction_low_bytes + assert stats.ledger_bytes > stats.projected_compacted_bytes assert "due_records=1" in _check_capture_store_stats(project).message monkeypatch.chdir(project) run_capture_store(reclaim=False) - assert "due_records=1" in capsys.readouterr().out + command_output = capsys.readouterr().out + assert "due_records=1" in command_output + assert "projected_compacted_bytes=" in command_output + assert "low_bytes=" in command_output + assert "physical_ledger_bytes=" in command_output + assert "default capacity limits" in command_output + assert "runtime overrides are runner-local" in command_output + + +def test_blocker_only_stats_keep_safe_defaults() -> None: + stats = CaptureStoreStats(CleanupBlocker.STORE_ABSENT) + + assert stats.projected_compacted_bytes == 0 + assert stats.compaction_low_bytes == CaptureCapacitySpec().compaction_low_bytes + assert not stats.capacity_warning def test_doctor_capture_store_reports_absent_store( @@ -242,6 +296,57 @@ def test_doctor_capture_store_warning_threshold( assert ("--reclaim" in result.message) is expects_remediation +@pytest.mark.parametrize( + ("projection_fraction", "expected_severity"), + [(0.81, Severity.WARNING), (0.79, Severity.OK)], +) +def test_doctor_warns_when_projection_approaches_the_soft_limit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + projection_fraction: float, + expected_severity: Severity, +) -> None: + low_bytes = CaptureCapacitySpec().compaction_low_bytes + projected_bytes = int(low_bytes * projection_fraction) + ledger_bytes = int(low_bytes * 0.85) + stats = CaptureStoreStats( + blocker=CleanupBlocker.NONE, + ledger_bytes=ledger_bytes, + projected_compacted_bytes=projected_bytes, + compaction_low_bytes=low_bytes, + ) + monkeypatch.setattr(doctor_capture_store, "capture_store_stats", lambda _project: stats) + + result = doctor_capture_store._check_capture_store_stats(tmp_path) + + assert result.severity is expected_severity + assert "projected_compacted_bytes=" in result.message + assert "low_bytes=" in result.message + assert str(round(projection_fraction * 100)) in result.message + assert "%" in result.message + if projection_fraction == 0.79: + assert ledger_bytes > int(low_bytes * 0.80) + + +def test_doctor_ok_for_existing_blocker_free_stub( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + doctor_capture_store, + "capture_store_stats", + lambda _project: CaptureStoreStats( + blocker=CleanupBlocker.NONE, + due_records=99, + ), + ) + + result = doctor_capture_store._check_capture_store_stats(tmp_path) + + assert result.severity is Severity.OK + assert "due_records=99" in result.message + + def test_capture_store_command_without_reclaim_does_not_mutate( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: @@ -401,3 +506,46 @@ def stat(*, follow_symlinks: bool): stats = capture_reconcile.capture_store_stats(str(project)) assert stats.blocker is CleanupBlocker.PERMISSION_DENIED + + +@pytest.mark.parametrize( + ("projection_fraction", "expects_capacity_warning"), + [(0.81, True), (0.79, False)], +) +def test_capture_store_command_reports_projected_capacity( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + projection_fraction: float, + expects_capacity_warning: bool, +) -> None: + project = tmp_path / "project" + project.mkdir() + low_bytes = CaptureCapacitySpec().compaction_low_bytes + projected_bytes = int(low_bytes * projection_fraction) + ledger_bytes = int(low_bytes * 0.85) + monkeypatch.setattr( + capture_store_command, + "capture_store_stats", + lambda _project: CaptureStoreStats( + blocker=CleanupBlocker.NONE, + ledger_bytes=ledger_bytes, + projected_compacted_bytes=projected_bytes, + compaction_low_bytes=low_bytes, + ), + ) + monkeypatch.chdir(project) + + run_capture_store(reclaim=False) + + command_output = capsys.readouterr().out + assert f"projected_compacted_bytes={projected_bytes}" in command_output + assert f"low_bytes={low_bytes}" in command_output + assert f"physical_ledger_bytes={ledger_bytes}" in command_output + assert "default capacity limits" in command_output + assert "runtime overrides are runner-local" in command_output + assert str(round(projection_fraction * 100)) in command_output + assert "%" in command_output + assert ("WARNING" in command_output) is expects_capacity_warning + if projection_fraction == 0.79: + assert ledger_bytes > int(low_bytes * 0.80) diff --git a/tests/hooks/test_capture_artifacts.py b/tests/hooks/test_capture_artifacts.py index 3d3d0343d7..4dc9dac2aa 100644 --- a/tests/hooks/test_capture_artifacts.py +++ b/tests/hooks/test_capture_artifacts.py @@ -7,6 +7,7 @@ import errno import json import os +import re import select import shlex import stat @@ -21,6 +22,7 @@ import autoskillit.hooks._capture._artifact_setup as capture_setup import autoskillit.hooks._capture._authority as capture_authority +import autoskillit.hooks._capture._capacity as capture_capacity import autoskillit.hooks._capture._failure_policy as capture_failure_policy import autoskillit.hooks._capture._publication as capture_publication import autoskillit.hooks._capture._reconcile as capture_reconcile @@ -1243,6 +1245,39 @@ def fail_create(*_args, **_kwargs): assert not sentinel.exists() +def test_reserve_capacity_refusal_detail_carries_measurement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], +) -> None: + project = tmp_path / "project" + project.mkdir() + reason = CaptureCapacityReason.PROJECTED_COMPACTED_BYTES + decision = capture_capacity.AdmissionDecision( + reason, + projected_bytes=18_422, + limit_bytes=18_421, + due_records=7, + ) + + def refuse_capacity(_store: CaptureLifecycleStore, _capture_id: str) -> CaptureLifecycleRecord: + raise CaptureCapacityError(reason, decision=decision) + + monkeypatch.setattr(CaptureLifecycleStore, "reserve_capture", refuse_capacity) + + assert capture_runner._main(_runner_args(cwd=str(project))) == 1 + + failure = _single_failure_marker(capfd.readouterr().err) + expected_detail = "compacted lifecycle capacity reached: projected=18422 limit=18421 due=7" + assert failure.reason is CaptureFailureReason.PROJECTED_COMPACTED_BYTES_EXHAUSTED + assert failure.detail == expected_detail + assert re.fullmatch( + r"^compacted lifecycle capacity reached: projected=\d+ limit=\d+ due=\d+$", + failure.detail, + ) + assert len(failure.detail.encode("utf-8")) <= 240 + + def test_recovery_contention_is_classified_at_the_runner_boundary( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 78a707b7b40761a4ea61aa06dabf039a1ab6137a Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 17:13:49 -0700 Subject: [PATCH 05/21] refactor: extract capture setup failure transport --- src/autoskillit/hooks/_capture/_runner.py | 41 +++++++++++++---------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/src/autoskillit/hooks/_capture/_runner.py b/src/autoskillit/hooks/_capture/_runner.py index 3e09d079f9..4199ecb47b 100644 --- a/src/autoskillit/hooks/_capture/_runner.py +++ b/src/autoskillit/hooks/_capture/_runner.py @@ -166,6 +166,29 @@ def _run_direct_command(bash_path: str, command: str) -> int: pass +def _handle_setup_failure( + exc: BaseException, + failure_stage: str, +) -> _capture_replay.CaptureFailureV3: + reason = _capture_failure_policy.runtime_failure_reason(exc) + detail = ( + exc.public_detail + if isinstance(exc, CaptureSetupError) and exc.public_detail + else f"{failure_stage} failed" + ) + return _capture_replay.failure_transport( + reason=reason, + stage=( + "capture setup" + if reason in _capture_failure_policy.CAPACITY_FAILURE_REASONS + else failure_stage + ), + detail=detail, + shell_returncode=None, + settlement=None, + ) + + def run_capture( command: str, cwd: str, @@ -246,23 +269,7 @@ def run_capture( failure_stage = "capture writer duplication" artifact_writer_fd = _duplicate_artifact_writer(artifact) except BaseException as exc: - reason = _capture_failure_policy.runtime_failure_reason(exc) - detail = ( - exc.public_detail - if isinstance(exc, CaptureSetupError) and exc.public_detail - else f"{failure_stage} failed" - ) - failure = _capture_replay.failure_transport( - reason=reason, - stage=( - "capture setup" - if reason in _capture_failure_policy.CAPACITY_FAILURE_REASONS - else failure_stage - ), - detail=detail, - shell_returncode=None, - settlement=None, - ) + failure = _handle_setup_failure(exc, failure_stage) if not isinstance(exc, _CAPTURE_RUNTIME_ERRORS): _capture_replay.capture_failure_return(failure) raise From 335f643c1d22d704c7d3c00318e353758339d7b7 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 17:47:28 -0700 Subject: [PATCH 06/21] fix: run capacity-refused commands with bounded output --- docs/decisions/0006-output-containment.md | 3 + ...09-verified-output-delivery-disposition.md | 21 +- docs/decisions/README.md | 2 +- docs/safety/hooks.md | 5 + src/autoskillit/hooks/_capture/_replay.py | 13 + src/autoskillit/hooks/_capture/_runner.py | 91 ++++- ...est_codex_capture_lifecycle_integration.py | 101 ++++-- tests/hooks/test_capture_artifacts.py | 327 ++++++++++++++++-- tests/hooks/test_capture_lifecycle.py | 2 +- 9 files changed, 500 insertions(+), 65 deletions(-) diff --git a/docs/decisions/0006-output-containment.md b/docs/decisions/0006-output-containment.md index 3546c09536..3e8ebeedb1 100644 --- a/docs/decisions/0006-output-containment.md +++ b/docs/decisions/0006-output-containment.md @@ -226,6 +226,9 @@ bookkeeping failure discards or misreports a verified primary result outside of what the import-time assertion alone can catch (e.g. a `PRESERVE_OUTPUT` classification applied to a reason that should be `DISCARD_OUTPUT`). +Containment remains in force under setup-stage capacity degradation because +agent-visible output stays bounded by `shell_max_inline_bytes`. + The same enforcement surface bounds admission by measured reclamation debt: assist and stall are admission-only capacity reasons, and import-time totality requires their gate, failure, disposition, detail, and blocker classifications. diff --git a/docs/decisions/0009-verified-output-delivery-disposition.md b/docs/decisions/0009-verified-output-delivery-disposition.md index c661736e38..07eec2b965 100644 --- a/docs/decisions/0009-verified-output-delivery-disposition.md +++ b/docs/decisions/0009-verified-output-delivery-disposition.md @@ -67,11 +67,18 @@ delivery — the guard falls through to the ordinary fail-closed handler in that case, matching the pre-existing behavior for every failure that occurs before verification completes. -The reclamation-debt stall occurs before the child starts. Its -`PRESERVE_OUTPUT` classification therefore supplies no output or delivery -authority: the runner emits the ordinary structured setup failure and never -enters degraded delivery because neither a verified snapshot nor a completed -child exists. +When a capacity refusal occurs before the child starts, it supplies no +captured-output authority. Capacity is retention bookkeeping, not an output +integrity fault, so the runner executes the command uncaptured through the +existing spawn and drain path. Output remains bounded by `shell_max_inline_bytes`: +small output is inline, and oversized output is reduced to bounded head and tail +slices with a fixed elision line. No artifact or reference is issued. A +`capture_degraded` marker at stage `capture_setup` carries the capacity reason +and admission measurement. Other setup failures still fail closed. + +The marker signals command-level degradation; lineage observations continue to +describe the launch's capture/direct routing. The marker is not an aggregate +degradation rate or alarm. ### 3. Exit-code parity under degraded delivery @@ -129,10 +136,12 @@ reclaimability and gate-reachability trustworthy. 2. This ADR governs only the AutoSkillit Codex native-shell capture runner established by ADR-0008. It does not extend disposition semantics to Claude Code native shell, MCP `run_cmd`, or any other artifact contract. -3. Degraded delivery inherits ADR-0008's bounded-transport limits +3. Verified-snapshot degraded delivery inherits ADR-0008's bounded-transport limits (`render_degraded_capture()` reuses the same inline-byte cap and V2 reference framing as the success path) — it does not add a new output size allowance. +4. Setup-capacity degradation loses output beyond the inline budget and creates + no full-output artifact. ## Consequences diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 2b2f946476..aa5bde4d33 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -8,7 +8,7 @@ - [0006-output-containment.md](0006-output-containment.md) — Retire pre-execution command-shape classification in favor of per-backend output-boundary bounding on measured bytes - [0007-context-admission.md](0007-context-admission.md) — Freeze the versioned cumulative context-admission boundary, authority contract, producer coverage, and privacy rules - [0008-shell-capture-snapshot-authority.md](0008-shell-capture-snapshot-authority.md) — Make verified pipe-EOF snapshots, opaque V2 references, and checked delivery the sole Codex shell-capture authority -- [0009-verified-output-delivery-disposition.md](0009-verified-output-delivery-disposition.md) — A checksum-verified capture is delivered or the failure explicitly says why not; a bookkeeping failure never discards it +- [0009-verified-output-delivery-disposition.md](0009-verified-output-delivery-disposition.md) — A checksum-verified capture is delivered or the failure explains why; eligible post-verification bookkeeping faults preserve it, while setup-stage capacity refusals may run uncaptured within the inline bound with a degradation marker; other setup failures fail closed - [0010-systemd-scope-defense-in-depth.md](0010-systemd-scope-defense-in-depth.md) — systemd scope wrapping is a best-effort kernel backstop on top of the tether sweep, never the ceiling of record - [0015-write-containment.md](0015-write-containment.md) — typed skill write scopes composed by union across interactive and headless sessions - [0016-owner-scope.md](0016-owner-scope.md) — a fleet dispatch settles every funnel-spawned descendant of its L2, selected by owner-scope registration, before it persists or returns diff --git a/docs/safety/hooks.md b/docs/safety/hooks.md index 6aa19114ac..44246ef255 100644 --- a/docs/safety/hooks.md +++ b/docs/safety/hooks.md @@ -199,10 +199,15 @@ is implemented. | Ownership | The producer transfers the retained carrier while preserving exclusive ownership through initial publication and delivery | | Cleanup — installed runner | Every valid run/reject invocation performs one bounded sweep after producer resources release and prioritizes its just-delivered inline capture before the cursor-rotated backlog within the existing `RUNNER_TAIL_BUDGET` | | Cleanup — session lifecycle | `capture_lifecycle_hook.py` performs the same bounded sweep at `SessionStart` in both interactive and headless sessions | +| Setup capacity fallback | A capacity refusal runs the command uncaptured within the `shell_max_inline_bytes` head/tail bound; oversized output gets a fixed elision line, with no artifact or reference. The `capture_degraded` marker at `capture_setup` carries the capacity reason and measurement. Other setup failures remain fail closed | | Naming contract | `shell_[0-9a-f]{16}.log` — files not matching this pattern are never deleted | | Safety | Capture components reject symlinks; artifacts reject collisions, hardlinks, unsafe modes, and identity changes; a hostile same-UID process that ignores advisory locks is excluded | | Failure mode | Pre-FINAL failure has no manifest/reference/success marker; post-FINAL failure preserves FINAL and changes only reference or delivery state; cleanup remains bounded and fail open | +The `capture_degraded` marker signals degradation for that command; lineage +observations continue to describe the launch's capture/direct routing. It is +not an aggregate degradation rate or alarm. + The durable lifecycle is `RESERVED` → `STAGED` → `PUBLISHED_WRITING` → `FINALIZED` or `FAILED`. An unlocked active record becomes `ABANDONED`. Eligible terminal or abandoned records move through `DELETING` to `DELETED`, diff --git a/src/autoskillit/hooks/_capture/_replay.py b/src/autoskillit/hooks/_capture/_replay.py index 7ed175ee60..49875fc90d 100644 --- a/src/autoskillit/hooks/_capture/_replay.py +++ b/src/autoskillit/hooks/_capture/_replay.py @@ -15,6 +15,7 @@ if TYPE_CHECKING: from autoskillit.hooks._capture._snapshot import ( + CaptureMeasurement, FinalizedCapture, PublishedCaptureReference, UnavailableCaptureReference, @@ -29,6 +30,7 @@ ) else: from ._snapshot import ( + CaptureMeasurement, FinalizedCapture, PublishedCaptureReference, UnavailableCaptureReference, @@ -189,6 +191,17 @@ def render_inline_capture(finalized: FinalizedCapture) -> bytes: return snapshot.measurement.inline +def render_uncaptured_bounded(measurement: CaptureMeasurement) -> bytes: + if measurement.total_bytes <= measurement.inline_bytes: + return measurement.inline + omitted = measurement.total_bytes - len(measurement.head) - len(measurement.tail) + elision = ( + b"[AutoSkillit shell capture elided: %d of %d bytes omitted; no capture artifact]" + % (omitted, measurement.total_bytes) + ) + return measurement.head + b"\n" + elision + b"\n" + measurement.tail + + def render_oversized_capture( value: PublishedCaptureReference | UnavailableCaptureReference, ) -> bytes: diff --git a/src/autoskillit/hooks/_capture/_runner.py b/src/autoskillit/hooks/_capture/_runner.py index 4199ecb47b..cb281d3668 100644 --- a/src/autoskillit/hooks/_capture/_runner.py +++ b/src/autoskillit/hooks/_capture/_runner.py @@ -8,6 +8,8 @@ import os import subprocess import sys +from collections.abc import Callable +from dataclasses import replace from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -123,16 +125,22 @@ def _drain_capture( process: subprocess.Popen[bytes] | OwnedProcessGroup, artifact_writer_fd: int, inline_bytes: int, + *, + write_all: Callable[[int, bytes], None] | None = None, ) -> PipeEofEvidence: return _drain_owned_capture( process, artifact_writer_fd, inline_bytes, digest_factory=hashlib.sha256, - write_all=_write_all, + write_all=_write_all if write_all is None else write_all, ) +def _discard_write(_fd: int, _data: bytes) -> None: + pass + + def _resolve_bash() -> str: return _resolve_trusted_bash(_TRUSTED_BASH_CANDIDATES) @@ -169,14 +177,18 @@ def _run_direct_command(bash_path: str, command: str) -> int: def _handle_setup_failure( exc: BaseException, failure_stage: str, -) -> _capture_replay.CaptureFailureV3: + *, + bash_path: str, + command: str, + policy: CapturePolicy, +) -> int: reason = _capture_failure_policy.runtime_failure_reason(exc) detail = ( exc.public_detail if isinstance(exc, CaptureSetupError) and exc.public_detail else f"{failure_stage} failed" ) - return _capture_replay.failure_transport( + failure = _capture_replay.failure_transport( reason=reason, stage=( "capture setup" @@ -187,6 +199,68 @@ def _handle_setup_failure( shell_returncode=None, settlement=None, ) + if isinstance(exc, _CAPTURE_RUNTIME_ERRORS) and ( + reason in _capture_failure_policy.CAPACITY_FAILURE_REASONS + ): + return _run_bounded_uncaptured( + bash_path, + command, + policy.inline_bytes, + replace(failure, stage="capture_setup"), + ) + return _capture_replay.capture_failure_return(failure) + + +def _run_bounded_uncaptured( + bash_path: str, + command: str, + inline_bytes: int, + failure: _capture_replay.CaptureFailureV3, +) -> int: + process: subprocess.Popen[bytes] | OwnedProcessGroup | None = None + try: + process = _spawn_bash(bash_path, command, capture_output=True) + drained = _drain_capture( + process, + -1, + inline_bytes, + write_all=_discard_write, + ) + outcome = CommandOutcome.from_wait_result(process.wait()) + except BaseException as exc: + settlement = _settle_failed_capture(process) if process is not None else None + result = _capture_replay.capture_failure_return( + _capture_replay.failure_transport( + reason=_capture_failure_policy.runtime_failure_reason(exc), + stage="capture_setup", + detail="uncaptured command execution failed", + shell_returncode=None, + settlement=settlement, + ) + ) + if not isinstance(exc, _CAPTURE_RUNTIME_ERRORS): + raise + return result + finally: + if process is not None and process.stdout is not None: + try: + process.stdout.close() + except _CAPTURE_RUNTIME_ERRORS: + pass + + try: + _capture_replay.write_and_flush_hook_stdout( + _capture_replay.render_uncaptured_bounded(drained.measurement) + ) + except OSError: + return _capture_replay.capture_failure_return( + replace(failure, shell_returncode=outcome.shell_returncode) + ) + try: + _capture_replay._emit_degraded(replace(failure, shell_returncode=outcome.shell_returncode)) + except _CAPTURE_RUNTIME_ERRORS: + pass + return _capture_replay.degraded_delivery_return(outcome.shell_returncode) def run_capture( @@ -269,11 +343,16 @@ def run_capture( failure_stage = "capture writer duplication" artifact_writer_fd = _duplicate_artifact_writer(artifact) except BaseException as exc: - failure = _handle_setup_failure(exc, failure_stage) + result = _handle_setup_failure( + exc, + failure_stage, + bash_path=bash_path, + command=command, + policy=policy, + ) if not isinstance(exc, _CAPTURE_RUNTIME_ERRORS): - _capture_replay.capture_failure_return(failure) raise - return _capture_replay.capture_failure_return(failure) + return result command_outcome: CommandOutcome | None = None command_returncode: int | None = None settlement: _capture_replay.RunnerSettlementEvidence | None = None diff --git a/tests/execution/backends/test_codex_capture_lifecycle_integration.py b/tests/execution/backends/test_codex_capture_lifecycle_integration.py index 3998f917a0..290c23103e 100644 --- a/tests/execution/backends/test_codex_capture_lifecycle_integration.py +++ b/tests/execution/backends/test_codex_capture_lifecycle_integration.py @@ -20,6 +20,7 @@ from autoskillit.execution.backends.codex import CodexBackend from autoskillit.execution.session import DefaultManagedHeadlessSessionLineageStore from autoskillit.hooks._capture import _ledger as capture_ledger +from autoskillit.hooks._capture._artifact_setup import CapturePolicy from autoskillit.hooks._capture._reconcile import ( RUNNER_TAIL_BUDGET, reconcile_capture_store, @@ -45,9 +46,11 @@ NATIVE_SHELL_CAPTURE_MODE_ENV_VAR, PROTECTED_CAPTURE_ENV_VARS, CaptureFailureReason, + CaptureFailureV3, CaptureLineageRef, canonical_json_bytes, decode_capture_request, + parse_capture_degraded_v3, parse_capture_failure_v3, ) from autoskillit.hooks._capture_lifecycle import ( @@ -395,6 +398,22 @@ def _rewritten_runner( return json.loads(rewritten.stdout)["hookSpecificOutput"]["updatedInput"]["command"] +def _degraded_markers(stderr: str) -> list[CaptureFailureV3]: + return [ + parse_capture_degraded_v3(line.encode()) + for line in stderr.splitlines() + if line.startswith("[AutoSkillit shell capture degraded v3:") + ] + + +def _failure_markers(stderr: str) -> list[CaptureFailureV3]: + return [ + parse_capture_failure_v3(line.encode()) + for line in stderr.splitlines() + if line.startswith("[AutoSkillit shell capture failure v3:") + ] + + @pytest.mark.parametrize("headless", [False, True]) def test_snapshotted_session_start_owner_reclaims_from_payload_project( tmp_path: Path, @@ -454,7 +473,7 @@ def test_saturated_installed_store_recovers_across_both_cleanup_owners( project=project, decoy_cwd=decoy_cwd, turn_id="saturated-process-a", - user_command=(f"printf should-not-run > {shlex.quote(str(process_a_sentinel))}"), + user_command=(f"printf process-a > {shlex.quote(str(process_a_sentinel))}"), ) completed_a = subprocess.run( ["bash", "-c", runner_a], @@ -464,16 +483,16 @@ def test_saturated_installed_store_recovers_across_both_cleanup_owners( timeout=_TIMEOUT_SECONDS, check=False, ) - failures = [ - parse_capture_failure_v3(line.encode()) - for line in completed_a.stderr.splitlines() - if line.startswith("[AutoSkillit shell capture failure v3:") - ] - - assert completed_a.returncode == 1 - assert len(failures) == 1 - assert failures[0].reason is CaptureFailureReason.ACTIVE_CAPACITY_EXHAUSTED - assert not process_a_sentinel.exists() + degraded_markers = _degraded_markers(completed_a.stderr) + failure_markers = _failure_markers(completed_a.stderr) + + assert completed_a.returncode == 0 + assert process_a_sentinel.read_text() == "process-a" + assert len(degraded_markers) == 1 + assert not failure_markers + assert degraded_markers[0].reason is CaptureFailureReason.ACTIVE_CAPACITY_EXHAUSTED + assert degraded_markers[0].stage == "capture_setup" + assert degraded_markers[0].shell_returncode == 0 assert not first_due.exists() assert session_due.exists() validation_anchor = open_project_anchor(str(project)) @@ -593,12 +612,20 @@ def test_installed_hook_reports_projected_compacted_capacity_at_capture_setup( _seed_projected_compacted_store(project) pre_tool_command = _installed_command(config, "PreToolUse", "shell_capture_hook") sentinel = project / "projected-compacted-command-ran" + inline_bytes = CapturePolicy().inline_bytes + output_bytes = bytes(65 + index % 26 for index in range(inline_bytes * 3 + 17)) + python_command = ( + f"import os; os.write(1, bytes(65 + index % 26 for index in range({len(output_bytes)})))" + ) runner = _rewritten_runner( pre_tool_command, project=project, decoy_cwd=decoy_cwd, turn_id="projected-compacted-capacity", - user_command=f"printf should-not-run > {shlex.quote(str(sentinel))}", + user_command=( + f"printf projected > {shlex.quote(str(sentinel))}; " + f"python3 -c {shlex.quote(python_command)}; exit 7" + ), ) completed = subprocess.run( @@ -609,17 +636,45 @@ def test_installed_hook_reports_projected_compacted_capacity_at_capture_setup( timeout=_TIMEOUT_SECONDS, check=False, ) - failures = [ - parse_capture_failure_v3(line.encode()) - for line in completed.stderr.splitlines() - if line.startswith("[AutoSkillit shell capture failure v3:") - ] - - assert completed.returncode == 1 - assert len(failures) == 1 - assert failures[0].reason is CaptureFailureReason.PROJECTED_COMPACTED_BYTES_EXHAUSTED - assert failures[0].stage == "capture_setup" - assert not sentinel.exists() + degraded_markers = _degraded_markers(completed.stderr) + failure_markers = _failure_markers(completed.stderr) + + assert completed.returncode == 7 + assert sentinel.read_text() == "projected" + assert len(degraded_markers) == 1 + assert not failure_markers + assert degraded_markers[0].reason is CaptureFailureReason.PROJECTED_COMPACTED_BYTES_EXHAUSTED + assert degraded_markers[0].stage == "capture_setup" + assert degraded_markers[0].shell_returncode == 7 + + measurement = CaptureMeasurement.from_bytes(output_bytes, inline_bytes=inline_bytes) + omitted = measurement.total_bytes - len(measurement.head) - len(measurement.tail) + elision = ( + b"[AutoSkillit shell capture elided: %d of %d bytes omitted; no capture artifact]" + % (omitted, measurement.total_bytes) + ) + expected_stdout = measurement.head + b"\n" + elision + b"\n" + measurement.tail + assert completed.stdout.encode() == expected_stdout + assert len(measurement.head) + len(measurement.tail) == inline_bytes + assert omitted == len(output_bytes) - inline_bytes + + request = decode_capture_request(shlex.split(runner.splitlines()[-1])[-1]) + capture_dir = project.joinpath(*CAPTURE_PATH_COMPONENTS) + assert list(capture_dir.glob("shell_*.log")) == [] + validation_anchor = open_project_anchor(str(project)) + try: + validation_root = open_capture_root(validation_anchor, create=False) + try: + validation_store = CaptureLifecycleStore.from_open_authorities( + validation_anchor, + validation_root, + lock_wait=HOT_PATH_LOCK_WAIT, + ) + assert validation_store.get_record(request.capture_id) is None + finally: + validation_root.close() + finally: + validation_anchor.close() @pytest.mark.parametrize("headless", [False, True]) diff --git a/tests/hooks/test_capture_artifacts.py b/tests/hooks/test_capture_artifacts.py index 4dc9dac2aa..e75f1fd3e9 100644 --- a/tests/hooks/test_capture_artifacts.py +++ b/tests/hooks/test_capture_artifacts.py @@ -215,6 +215,38 @@ def _runner_args(**changes: object) -> list[str]: return [encode_capture_request(_runner_request(**changes))] # type: ignore[arg-type] +def _install_reserve_capacity_refusal( + monkeypatch: pytest.MonkeyPatch, + reason: CaptureCapacityReason, +) -> capture_capacity.AdmissionDecision: + decision = capture_capacity.AdmissionDecision( + reason, + projected_bytes=18_422, + limit_bytes=18_421, + due_records=7, + ) + + def refuse(_store: CaptureLifecycleStore, _capture_id: str) -> CaptureLifecycleRecord: + raise CaptureCapacityError(reason, decision=decision) + + monkeypatch.setattr(CaptureLifecycleStore, "reserve_capture", refuse) + return decision + + +def _install_fake_capture_process( + monkeypatch: pytest.MonkeyPatch, + process: _FakeCaptureProcess, +) -> list[dict[str, object]]: + spawn_calls: list[dict[str, object]] = [] + + def spawn(*_args, **kwargs): + spawn_calls.append(kwargs) + return process + + monkeypatch.setattr(capture_runner, "_spawn_bash", spawn) + return spawn_calls + + def _open_authority(project: Path): anchor = open_project_anchor(str(project)) try: @@ -345,8 +377,9 @@ def close(self) -> None: class _FakeCaptureProcess: - def __init__(self, value: bytes) -> None: + def __init__(self, value: bytes, returncode: int = 0) -> None: self.stdout = _ReadableStream(value) + self.returncode = returncode self.terminated = False self.killed = False self.wait_calls = 0 @@ -363,7 +396,7 @@ def kill(self) -> None: def wait(self, timeout: float | None = None) -> int: del timeout self.wait_calls += 1 - return 0 + return self.returncode def _record_artifact_fds(monkeypatch: pytest.MonkeyPatch) -> list[int]: @@ -1022,7 +1055,7 @@ def test_setup_failure_prevents_user_command_and_emits_failure_marker( assert not (project / "command_ran").exists() -def test_reclamation_debt_stall_emits_structured_failure_without_starting_command( +def test_reclamation_debt_stall_runs_command_bounded_uncaptured( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str], @@ -1084,16 +1117,14 @@ def due_store( capture_runner._main( _runner_args(command=f"printf ran > {shlex.quote(str(sentinel))}", cwd=str(project)) ) - == 1 + == 0 ) - failure = _single_failure_marker(capfd.readouterr().err) - assert failure.reason is CaptureFailureReason.RECLAMATION_DEBT_STALL - assert ( - capture_failure_policy.FAILURE_DISPOSITIONS[failure.reason].disposition - is capture_failure_policy.CaptureFailureDisposition.PRESERVE_OUTPUT - ) - assert not sentinel.exists() + degraded = _single_degraded_marker(capfd.readouterr().err) + assert degraded.reason is CaptureFailureReason.RECLAMATION_DEBT_STALL + assert degraded.stage == "capture_setup" + assert degraded.shell_returncode == 0 + assert sentinel.exists() @pytest.mark.parametrize("root_shape", ["blocking_file", "symlink"]) @@ -1215,7 +1246,14 @@ def wait(self, timeout: float | None = None) -> int: assert process.stdout.closed -@pytest.mark.parametrize("reason", tuple(CaptureFailureReason)) +@pytest.mark.parametrize( + "reason", + tuple( + reason + for reason in CaptureFailureReason + if reason not in capture_failure_policy.CAPACITY_FAILURE_REASONS + ), +) def test_setup_failure_reason_survives_runner_transport_without_sensitive_detail( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -1253,29 +1291,262 @@ def test_reserve_capacity_refusal_detail_carries_measurement( project = tmp_path / "project" project.mkdir() reason = CaptureCapacityReason.PROJECTED_COMPACTED_BYTES - decision = capture_capacity.AdmissionDecision( - reason, - projected_bytes=18_422, - limit_bytes=18_421, - due_records=7, + _install_reserve_capacity_refusal(monkeypatch, reason) + process = _FakeCaptureProcess(b"measured-refusal-output") + spawn_calls = _install_fake_capture_process(monkeypatch, process) + + assert capture_runner._main(_runner_args(cwd=str(project))) == 0 + + captured = capfd.readouterr() + degraded = _single_degraded_marker(captured.err) + expected_detail = "compacted lifecycle capacity reached: projected=18422 limit=18421 due=7" + assert captured.out == "measured-refusal-output" + assert degraded.reason is CaptureFailureReason.PROJECTED_COMPACTED_BYTES_EXHAUSTED + assert degraded.stage == "capture_setup" + assert degraded.shell_returncode == 0 + assert degraded.detail == expected_detail + assert re.fullmatch( + r"^compacted lifecycle capacity reached: projected=\d+ limit=\d+ due=\d+$", + degraded.detail, ) + assert len(degraded.detail.encode("utf-8")) <= 240 + assert spawn_calls == [{"capture_output": True}] + assert process.wait_calls == 1 - def refuse_capacity(_store: CaptureLifecycleStore, _capture_id: str) -> CaptureLifecycleRecord: - raise CaptureCapacityError(reason, decision=decision) - monkeypatch.setattr(CaptureLifecycleStore, "reserve_capture", refuse_capacity) +@pytest.mark.parametrize("reason", tuple(CaptureCapacityReason)) +@pytest.mark.parametrize("child_returncode", [0, 7], ids=("zero", "nonzero")) +def test_capacity_setup_refusal_runs_command_bounded_uncaptured( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], + reason: CaptureCapacityReason, + child_returncode: int, +) -> None: + project = tmp_path / "project" + project.mkdir() + expected = b"uncaptured child output" + decision = _install_reserve_capacity_refusal(monkeypatch, reason) + process = _FakeCaptureProcess(expected, returncode=child_returncode) + spawn_calls = _install_fake_capture_process(monkeypatch, process) + + assert capture_runner._main(_runner_args(cwd=str(project))) == child_returncode + + captured = capfd.readouterr() + degraded = _single_degraded_marker(captured.err) + assert captured.out == expected.decode("utf-8") + assert spawn_calls == [{"capture_output": True}] + assert process.wait_calls == 1 + assert degraded.stage == "capture_setup" + assert degraded.reason is capture_capacity.failure_reason(reason) + assert degraded.detail == capture_capacity.measurement_detail(decision) + assert degraded.shell_returncode == child_returncode + assert "shell capture failure v3:" not in captured.err + assert not list(_capture_dir(project).glob("shell_*.log")) + with open_capture_lifecycle( + str(project), + create=False, + lock_wait=capture_types.HOT_PATH_LOCK_WAIT, + ) as lifecycle: + assert lifecycle.get_record(_CAPTURE_ID) is None + + +def test_capacity_setup_refusal_bounds_large_output( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], +) -> None: + project = tmp_path / "project" + project.mkdir() + inline_bytes = 12 + expected = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + assert len(expected) == 3 * inline_bytes + decision = _install_reserve_capacity_refusal( + monkeypatch, + CaptureCapacityReason.PROJECTED_COMPACTED_BYTES, + ) + process = _FakeCaptureProcess(expected) + _install_fake_capture_process(monkeypatch, process) + monkeypatch.setattr( + capture_runner, + "read_capture_policy", + lambda _anchor: CapturePolicy(inline_bytes=inline_bytes), + ) + + assert capture_runner._main(_runner_args(cwd=str(project))) == 0 + + captured = capfd.readouterr() + measurement = CaptureMeasurement.from_bytes(expected, inline_bytes=inline_bytes) + omitted = len(expected) - len(measurement.head) - len(measurement.tail) + elision = ( + b"[AutoSkillit shell capture elided: %d of %d bytes omitted; no capture artifact]" + % (omitted, len(expected)) + ) + expected_output = measurement.head + b"\n" + elision + b"\n" + measurement.tail + assert captured.out.encode("utf-8") == expected_output + assert len(measurement.head) + len(measurement.tail) == inline_bytes + assert b"shell capture v2:" not in expected_output + degraded = _single_degraded_marker(captured.err) + assert degraded.detail == capture_capacity.measurement_detail(decision) + + +@pytest.mark.parametrize("child_returncode", [0, 7], ids=("zero", "nonzero")) +def test_degraded_stdout_failure_emits_single_failure_envelope( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], + child_returncode: int, +) -> None: + project = tmp_path / "project" + project.mkdir() + _install_reserve_capacity_refusal( + monkeypatch, + CaptureCapacityReason.PROJECTED_COMPACTED_BYTES, + ) + process = _FakeCaptureProcess(b"uncaptured child output", returncode=child_returncode) + _install_fake_capture_process(monkeypatch, process) + stdout_attempts: list[bool] = [] + + def fail_stdout(_payload: bytes) -> None: + stdout_attempts.append(True) + raise OSError("fault injection") + + monkeypatch.setattr(capture_replay, "write_and_flush_hook_stdout", fail_stdout) + + assert capture_runner._main(_runner_args(cwd=str(project))) == (child_returncode or 1) + + captured = capfd.readouterr() + failure = _single_failure_marker(captured.err) + assert captured.out == "" + assert stdout_attempts == [True] + assert failure.shell_returncode == child_returncode + assert "shell capture degraded v3:" not in captured.err + + +def test_degraded_interrupt_settles_and_reraises( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], +) -> None: + project = tmp_path / "project" + project.mkdir() + _install_reserve_capacity_refusal( + monkeypatch, + CaptureCapacityReason.PROJECTED_COMPACTED_BYTES, + ) + process = _FakeCaptureProcess(b"output discarded after interruption") + _install_fake_capture_process(monkeypatch, process) + + def interrupt_drain(*_args, **_kwargs): + raise KeyboardInterrupt("fault injection") + + monkeypatch.setattr(capture_runner, "_drain_capture", interrupt_drain) + + with pytest.raises(KeyboardInterrupt, match="fault injection"): + capture_runner._main(_runner_args(cwd=str(project))) + + captured = capfd.readouterr() + _single_failure_marker(captured.err) + assert captured.out == "" + assert process.terminated + assert process.wait_calls > 0 + assert process.stdout.closed + assert "shell capture degraded v3:" not in captured.err + + +def test_degraded_spawn_failure_emits_single_failure_envelope( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], +) -> None: + project = tmp_path / "project" + project.mkdir() + _install_reserve_capacity_refusal( + monkeypatch, + CaptureCapacityReason.PROJECTED_COMPACTED_BYTES, + ) + spawn_calls: list[bool] = [] + settlement_attempts: list[object] = [] + + def fail_spawn(*_args, **_kwargs): + spawn_calls.append(True) + raise OSError("spawn fault injection") + + def observe_settlement(process: object): + settlement_attempts.append(process) + + monkeypatch.setattr(capture_runner, "_spawn_bash", fail_spawn) + monkeypatch.setattr(capture_runner, "_settle_failed_capture", observe_settlement) assert capture_runner._main(_runner_args(cwd=str(project))) == 1 - failure = _single_failure_marker(capfd.readouterr().err) - expected_detail = "compacted lifecycle capacity reached: projected=18422 limit=18421 due=7" - assert failure.reason is CaptureFailureReason.PROJECTED_COMPACTED_BYTES_EXHAUSTED - assert failure.detail == expected_detail - assert re.fullmatch( - r"^compacted lifecycle capacity reached: projected=\d+ limit=\d+ due=\d+$", - failure.detail, + captured = capfd.readouterr() + _single_failure_marker(captured.err) + assert captured.out == "" + assert spawn_calls == [True] + assert settlement_attempts == [] + assert "shell capture degraded v3:" not in captured.err + + +def test_degraded_stdout_close_error_preserves_result( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], +) -> None: + project = tmp_path / "project" + project.mkdir() + _install_reserve_capacity_refusal( + monkeypatch, + CaptureCapacityReason.PROJECTED_COMPACTED_BYTES, + ) + process = _FakeCaptureProcess(b"uncaptured child output", returncode=7) + _install_fake_capture_process(monkeypatch, process) + close_attempts: list[bool] = [] + + def fail_close() -> None: + close_attempts.append(True) + raise OSError("stdout close fault injection") + + monkeypatch.setattr(process.stdout, "close", fail_close) + + assert capture_runner._main(_runner_args(cwd=str(project))) == 7 + + captured = capfd.readouterr() + degraded = _single_degraded_marker(captured.err) + assert captured.out == "uncaptured child output" + assert close_attempts == [True] + assert degraded.shell_returncode == 7 + assert "shell capture failure v3:" not in captured.err + + +def test_degraded_settlement_failure_discards_output( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], +) -> None: + project = tmp_path / "project" + project.mkdir() + _install_reserve_capacity_refusal( + monkeypatch, + CaptureCapacityReason.PROJECTED_COMPACTED_BYTES, ) - assert len(failure.detail.encode("utf-8")) <= 240 + process = _FakeCaptureProcess(b"output discarded after settlement failure") + _install_fake_capture_process(monkeypatch, process) + + def fail_drain(*_args, **_kwargs): + raise CaptureSetupError(CaptureFailureReason.RUNNER_SETTLEMENT, "settlement failed") + + monkeypatch.setattr(capture_runner, "_drain_capture", fail_drain) + + assert capture_runner._main(_runner_args(cwd=str(project))) == 1 + + captured = capfd.readouterr() + failure = _single_failure_marker(captured.err) + assert captured.out == "" + assert failure.reason is CaptureFailureReason.RUNNER_SETTLEMENT + assert process.terminated + assert process.wait_calls > 0 + assert "shell capture degraded v3:" not in captured.err def test_recovery_contention_is_classified_at_the_runner_boundary( diff --git a/tests/hooks/test_capture_lifecycle.py b/tests/hooks/test_capture_lifecycle.py index d3f15056b2..5751035f81 100644 --- a/tests/hooks/test_capture_lifecycle.py +++ b/tests/hooks/test_capture_lifecycle.py @@ -1821,7 +1821,7 @@ def test_store_open_normalization_succeeds_above_soft_limit( base_capacity = store._capacity scaled_capacity = replace( base_capacity, - compaction_low_bytes=projected - 1, + compaction_low_bytes=projected - 64, compaction_high_bytes=projected, hard_ledger_bytes=projected + base_capacity.recovery_headroom_bytes + 64, ) From 2dd951308b5ce839c2ff824cb46bac188faecd65 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:25:37 -0700 Subject: [PATCH 07/21] fix(review): Reuse the lifecycle test helpers --- tests/hooks/test_capture_capacity_model.py | 14 ++-- tests/hooks/test_capture_frame_bound.py | 14 ++-- tests/hooks/test_capture_lifecycle.py | 7 ++ .../hooks/test_capture_sustained_admission.py | 64 ++----------------- 4 files changed, 25 insertions(+), 74 deletions(-) diff --git a/tests/hooks/test_capture_capacity_model.py b/tests/hooks/test_capture_capacity_model.py index 8ac6d319fa..41584c7cd0 100644 --- a/tests/hooks/test_capture_capacity_model.py +++ b/tests/hooks/test_capture_capacity_model.py @@ -17,7 +17,12 @@ import autoskillit.hooks._capture._types as capture_types import autoskillit.hooks._capture_lifecycle as capture_lifecycle from autoskillit.hooks._capture_artifacts import create_capture_artifact -from tests.hooks.test_capture_lifecycle import _Clock, _open_store, _verified_snapshot +from tests.hooks.test_capture_lifecycle import ( + _Clock, + _close_artifact, + _open_store, + _verified_snapshot, +) pytestmark = [pytest.mark.layer("hooks"), pytest.mark.small] @@ -96,13 +101,6 @@ def test_override_rejected_when_it_breaks_the_envelope() -> None: assert capture_artifact_setup._policy_capacity({"max_tombstones": 128}) is not None -def _close_artifact(artifact) -> None: - try: - artifact.close_artifact_fd() - finally: - artifact.release_lease() - - def _finalize_template(store, root, clock: _Clock, capture_id: str, *, delivered: bool): artifact = create_capture_artifact(root, capture_id, store) try: diff --git a/tests/hooks/test_capture_frame_bound.py b/tests/hooks/test_capture_frame_bound.py index 1ff4ba5d40..f5e716e016 100644 --- a/tests/hooks/test_capture_frame_bound.py +++ b/tests/hooks/test_capture_frame_bound.py @@ -15,7 +15,12 @@ import autoskillit.hooks._capture._types as capture_types import autoskillit.hooks._capture_lifecycle as capture_lifecycle from autoskillit.hooks._capture_artifacts import create_capture_artifact -from tests.hooks.test_capture_lifecycle import _Clock, _open_store, _verified_snapshot +from tests.hooks.test_capture_lifecycle import ( + _Clock, + _close_artifact, + _open_store, + _verified_snapshot, +) pytestmark = [pytest.mark.layer("hooks"), pytest.mark.medium] @@ -25,13 +30,6 @@ _FAILURE_CAPTURE_ID = "0000000000000002" -def _close_artifact(artifact) -> None: - try: - artifact.close_artifact_fd() - finally: - artifact.release_lease() - - def _profile_record( record: capture_lifecycle.CaptureLifecycleRecord, ) -> capture_lifecycle.CaptureLifecycleRecord: diff --git a/tests/hooks/test_capture_lifecycle.py b/tests/hooks/test_capture_lifecycle.py index 5751035f81..937c3c9322 100644 --- a/tests/hooks/test_capture_lifecycle.py +++ b/tests/hooks/test_capture_lifecycle.py @@ -926,6 +926,13 @@ def _open_store( return anchor, root, store +def _close_artifact(artifact) -> None: + try: + artifact.close_artifact_fd() + finally: + artifact.release_lease() + + def _capture_dir(project: Path) -> Path: return project.joinpath(*CAPTURE_PATH_COMPONENTS) diff --git a/tests/hooks/test_capture_sustained_admission.py b/tests/hooks/test_capture_sustained_admission.py index 4b0d48092a..859a77956a 100644 --- a/tests/hooks/test_capture_sustained_admission.py +++ b/tests/hooks/test_capture_sustained_admission.py @@ -17,69 +17,17 @@ CommandOutcome, verify_capture_snapshot, ) -from autoskillit.hooks._capture._types import ( - HOT_PATH_LOCK_WAIT, - CaptureCapacitySpec, -) -from autoskillit.hooks._capture_artifacts import ( - create_capture_artifact, - open_capture_root, - open_project_anchor, +from autoskillit.hooks._capture._types import CaptureCapacitySpec +from autoskillit.hooks._capture_artifacts import create_capture_artifact +from tests.hooks.test_capture_lifecycle import ( + _Clock, + _close_artifact, + _open_store, ) pytestmark = [pytest.mark.layer("hooks"), pytest.mark.medium] -class _Clock: - def __init__(self, value: float = 1_000_000.0) -> None: - self.value = value - - def wall(self) -> float: - return self.value - - def monotonic(self) -> float: - return self.value - - def advance(self, seconds: float) -> None: - self.value += seconds - - -def _open_store( - project: Path, - clock: _Clock, - *, - capacity: CaptureCapacitySpec | None = None, -): - project.mkdir() - anchor = open_project_anchor(str(project)) - try: - root = open_capture_root(anchor, create=True) - except BaseException: - anchor.close() - raise - try: - store = capture_lifecycle.CaptureLifecycleStore.from_open_authorities( - anchor, - root, - wall_clock=clock.wall, - monotonic=clock.monotonic, - lock_wait=HOT_PATH_LOCK_WAIT, - capacity=capacity, - ) - except BaseException: - root.close() - anchor.close() - raise - return anchor, root, store - - -def _close_artifact(artifact) -> None: - try: - artifact.close_artifact_fd() - finally: - artifact.release_lease() - - def _begin_capture( store, anchor, From bb5268f8e161e403718241b2162bbc037e387582 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:27:48 -0700 Subject: [PATCH 08/21] fix(review): report actual admission measurements and verify refusal diagnostics --- src/autoskillit/hooks/_capture/_capacity.py | 27 ++++++- tests/hooks/test_capture_artifacts.py | 80 ++++++++++++++++++--- tests/hooks/test_capture_capacity_model.py | 64 +++++++++++++++++ 3 files changed, 157 insertions(+), 14 deletions(-) diff --git a/src/autoskillit/hooks/_capture/_capacity.py b/src/autoskillit/hooks/_capture/_capacity.py index 7ef1a2d630..1e840acfda 100644 --- a/src/autoskillit/hooks/_capture/_capacity.py +++ b/src/autoskillit/hooks/_capture/_capacity.py @@ -68,6 +68,8 @@ class AdmissionDecision: projected_bytes: int | None = None limit_bytes: int | None = None due_records: int | None = None + record_count: int | None = None + record_limit: int | None = None class CompactedFrameSizer: @@ -173,6 +175,10 @@ def measurement_detail(decision: AdmissionDecision) -> str: if decision.projected_bytes is not None else None, f"limit={decision.limit_bytes}" if decision.limit_bytes is not None else None, + f"records={decision.record_count}" if decision.record_count is not None else None, + f"limit_records={decision.record_limit}" + if decision.record_limit is not None + else None, f"due={decision.due_records}" if decision.due_records is not None else None, ) if value is not None @@ -248,11 +254,26 @@ def admission_reason( forensic += is_forensic due += _sweep_cursor.is_due_record(record, now, terminal_states) if operational > active_limit: - return AdmissionDecision(CaptureCapacityReason.ACTIVE_CAPACITY, due_records=due) + return AdmissionDecision( + CaptureCapacityReason.ACTIVE_CAPACITY, + due_records=due, + record_count=operational, + record_limit=active_limit, + ) if retained > spec.max_retained_records: - return AdmissionDecision(CaptureCapacityReason.RETENTION_CAPACITY, due_records=due) + return AdmissionDecision( + CaptureCapacityReason.RETENTION_CAPACITY, + due_records=due, + record_count=retained, + record_limit=spec.max_retained_records, + ) if operational + forensic > spec.max_evidence_records: - return AdmissionDecision(CaptureCapacityReason.EVIDENCE_CAPACITY, due_records=due) + return AdmissionDecision( + CaptureCapacityReason.EVIDENCE_CAPACITY, + due_records=due, + record_count=operational + forensic, + record_limit=spec.max_evidence_records, + ) encoded = sizer.total_bytes(projected, compaction_epoch, spec) if encoded + spec.recovery_headroom_bytes > spec.hard_ledger_bytes: return AdmissionDecision( diff --git a/tests/hooks/test_capture_artifacts.py b/tests/hooks/test_capture_artifacts.py index e75f1fd3e9..99b8b1a764 100644 --- a/tests/hooks/test_capture_artifacts.py +++ b/tests/hooks/test_capture_artifacts.py @@ -219,12 +219,35 @@ def _install_reserve_capacity_refusal( monkeypatch: pytest.MonkeyPatch, reason: CaptureCapacityReason, ) -> capture_capacity.AdmissionDecision: - decision = capture_capacity.AdmissionDecision( - reason, - projected_bytes=18_422, - limit_bytes=18_421, - due_records=7, - ) + if reason in ( + CaptureCapacityReason.ACTIVE_CAPACITY, + CaptureCapacityReason.RETENTION_CAPACITY, + CaptureCapacityReason.EVIDENCE_CAPACITY, + ): + decision = capture_capacity.AdmissionDecision( + reason, + record_count=3, + record_limit=2, + due_records=7, + ) + elif reason in ( + CaptureCapacityReason.HARD_LEDGER_CAPACITY, + CaptureCapacityReason.PROJECTED_COMPACTED_BYTES, + ): + decision = capture_capacity.AdmissionDecision( + reason, + projected_bytes=18_422, + limit_bytes=18_421, + due_records=7, + ) + elif reason is CaptureCapacityReason.RECLAMATION_DEBT_ASSIST: + decision = capture_capacity.AdmissionDecision( + reason, + assist_transition_limit=1, + due_records=7, + ) + else: + decision = capture_capacity.AdmissionDecision(reason, due_records=7) def refuse(_store: CaptureLifecycleStore, _capture_id: str) -> CaptureLifecycleRecord: raise CaptureCapacityError(reason, decision=decision) @@ -1314,19 +1337,52 @@ def test_reserve_capacity_refusal_detail_carries_measurement( assert process.wait_calls == 1 -@pytest.mark.parametrize("reason", tuple(CaptureCapacityReason)) +@pytest.mark.parametrize( + ("reason", "expected_detail"), + [ + ( + CaptureCapacityReason.ACTIVE_CAPACITY, + "active lifecycle record bound reached: records=3 limit_records=2 due=7", + ), + ( + CaptureCapacityReason.RETENTION_CAPACITY, + "retention capacity reached: records=3 limit_records=2 due=7", + ), + ( + CaptureCapacityReason.EVIDENCE_CAPACITY, + "evidence record capacity reached: records=3 limit_records=2 due=7", + ), + ( + CaptureCapacityReason.PROJECTED_COMPACTED_BYTES, + "compacted lifecycle capacity reached: projected=18422 limit=18421 due=7", + ), + ( + CaptureCapacityReason.HARD_LEDGER_CAPACITY, + "hard lifecycle ledger capacity reached: projected=18422 limit=18421 due=7", + ), + ( + CaptureCapacityReason.RECLAMATION_DEBT_ASSIST, + "reclamation debt requires bounded assist: due=7", + ), + ( + CaptureCapacityReason.RECLAMATION_DEBT_STALL, + "reclamation debt admission ceiling reached: due=7", + ), + ], +) @pytest.mark.parametrize("child_returncode", [0, 7], ids=("zero", "nonzero")) def test_capacity_setup_refusal_runs_command_bounded_uncaptured( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str], reason: CaptureCapacityReason, + expected_detail: str, child_returncode: int, ) -> None: project = tmp_path / "project" project.mkdir() expected = b"uncaptured child output" - decision = _install_reserve_capacity_refusal(monkeypatch, reason) + _install_reserve_capacity_refusal(monkeypatch, reason) process = _FakeCaptureProcess(expected, returncode=child_returncode) spawn_calls = _install_fake_capture_process(monkeypatch, process) @@ -1339,7 +1395,7 @@ def test_capacity_setup_refusal_runs_command_bounded_uncaptured( assert process.wait_calls == 1 assert degraded.stage == "capture_setup" assert degraded.reason is capture_capacity.failure_reason(reason) - assert degraded.detail == capture_capacity.measurement_detail(decision) + assert degraded.detail == expected_detail assert degraded.shell_returncode == child_returncode assert "shell capture failure v3:" not in captured.err assert not list(_capture_dir(project).glob("shell_*.log")) @@ -1361,7 +1417,7 @@ def test_capacity_setup_refusal_bounds_large_output( inline_bytes = 12 expected = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" assert len(expected) == 3 * inline_bytes - decision = _install_reserve_capacity_refusal( + _install_reserve_capacity_refusal( monkeypatch, CaptureCapacityReason.PROJECTED_COMPACTED_BYTES, ) @@ -1387,7 +1443,9 @@ def test_capacity_setup_refusal_bounds_large_output( assert len(measurement.head) + len(measurement.tail) == inline_bytes assert b"shell capture v2:" not in expected_output degraded = _single_degraded_marker(captured.err) - assert degraded.detail == capture_capacity.measurement_detail(decision) + assert degraded.detail == ( + "compacted lifecycle capacity reached: projected=18422 limit=18421 due=7" + ) @pytest.mark.parametrize("child_returncode", [0, 7], ids=("zero", "nonzero")) diff --git a/tests/hooks/test_capture_capacity_model.py b/tests/hooks/test_capture_capacity_model.py index 41584c7cd0..dfe42b4086 100644 --- a/tests/hooks/test_capture_capacity_model.py +++ b/tests/hooks/test_capture_capacity_model.py @@ -175,6 +175,70 @@ def _admission_decision( ) +@pytest.mark.parametrize( + ("reason", "limit_field", "existing_count"), + [ + ( + capture_lifecycle.CaptureCapacityReason.ACTIVE_CAPACITY, + "max_operational_records", + 3, + ), + ( + capture_lifecycle.CaptureCapacityReason.RETENTION_CAPACITY, + "max_retained_records", + 1, + ), + ( + capture_lifecycle.CaptureCapacityReason.EVIDENCE_CAPACITY, + "max_evidence_records", + 2, + ), + ], + ids=("active", "retention", "evidence"), +) +def test_admission_reason_reports_record_count_and_limit( + tmp_path: Path, + reason: capture_lifecycle.CaptureCapacityReason, + limit_field: str, + existing_count: int, +) -> None: + spec = _capacity_spec( + max_operational_records=4, + max_retained_records=4, + max_evidence_records=4, + reclamation_debt_assist_records=1, + reclamation_debt_stall_records=2, + ) + clock = _Clock() + anchor, root, store = _open_store(tmp_path / "project", clock) + try: + candidate = _finalize_template( + store, + root, + clock, + "aaaaaaaaaaaaaaa1", + delivered=False, + ) + assert candidate is not None + records = { + f"{index:016x}": _replicate_record(candidate, f"{index:016x}") + for index in range(1, existing_count + 1) + } + decision = _admission_decision( + records, + candidate, + replace(spec, **{limit_field: existing_count}), + clock, + ) + finally: + root.close() + anchor.close() + + assert decision.reason is reason + assert decision.record_count == existing_count + 1 + assert decision.record_limit == existing_count + + def test_pre_assist_debt_is_refused_by_assist_not_bytes(tmp_path: Path) -> None: clock = _Clock() anchor, root, store = _open_store(tmp_path / "project", clock) From e5e528284b4796be3f46f4189d4a63cf350609a7 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:29:55 -0700 Subject: [PATCH 09/21] fix(review): List violated envelope fields in the import-time assertion --- src/autoskillit/hooks/_capture/_types.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/autoskillit/hooks/_capture/_types.py b/src/autoskillit/hooks/_capture/_types.py index e9e2370e49..00b3c82776 100644 --- a/src/autoskillit/hooks/_capture/_types.py +++ b/src/autoskillit/hooks/_capture/_types.py @@ -417,8 +417,12 @@ def capacity_envelope_violations(spec: CaptureCapacitySpec) -> tuple[str, ...]: return tuple(field for field, required, actual in requirements if actual < required) -if capacity_envelope_violations(CaptureCapacitySpec()): - raise AssertionError("production capture capacity must cover the declared retention envelope") +_envelope_violations = capacity_envelope_violations(CaptureCapacitySpec()) +if _envelope_violations: + raise AssertionError( + "production capture capacity must cover the declared retention envelope: " + + ", ".join(_envelope_violations) + ) DEBT_ASSIST_MAX_TRANSITIONS = 16 DEBT_ASSIST_BUDGET = SweepBudgetSpec( From 32967cd5cd7be7ec1b6933cbc79e1584a0c99fc6 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:31:12 -0700 Subject: [PATCH 10/21] fix(review): Document the encoder measurement source and rounding for the frame allowance --- src/autoskillit/hooks/_capture/_types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/autoskillit/hooks/_capture/_types.py b/src/autoskillit/hooks/_capture/_types.py index 00b3c82776..63f5ea1876 100644 --- a/src/autoskillit/hooks/_capture/_types.py +++ b/src/autoskillit/hooks/_capture/_types.py @@ -392,6 +392,7 @@ class Tampered(Exception): SUPPORTED_PEAK_ADMISSIONS_PER_SECOND = ( SUPPORTED_PEAK_PRODUCERS * SUPPORTED_ADMISSIONS_PER_PRODUCER_SECOND ) +# ADR-0006: maximum measured encoder frame, rounded up to an eight-byte boundary. PLANNED_RECORD_FRAME_BYTES = 2608 From c00af5af30e78f3ba65708edf1af6978cca49ceb Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:32:25 -0700 Subject: [PATCH 11/21] fix(review): Use the required predecessor directly in transition capacity checks --- src/autoskillit/hooks/_capture/_capacity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/autoskillit/hooks/_capture/_capacity.py b/src/autoskillit/hooks/_capture/_capacity.py index 1e840acfda..440b8f1395 100644 --- a/src/autoskillit/hooks/_capture/_capacity.py +++ b/src/autoskillit/hooks/_capture/_capacity.py @@ -314,8 +314,8 @@ def transition_reason( projected = _projected(records, candidate) encoded = sizer.total_bytes(projected, compaction_epoch, spec) recovery_state = transition_compaction_bound(candidate, spec) == spec.hard_ledger_bytes - previous = records.get(candidate.capture_id) - state_changes = previous is None or previous.state is not candidate.state + previous = records[candidate.capture_id] + state_changes = previous.state is not candidate.state if encoded > transition_compaction_bound(candidate, spec): return CaptureCapacityReason.HARD_LEDGER_CAPACITY if state_changes and not recovery_state and encoded > spec.compaction_low_bytes: From 0b5e7779c86ddf3ab553baaaebe26b7bfdd6115c Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:33:39 -0700 Subject: [PATCH 12/21] fix(review): expose degraded capture emitter --- src/autoskillit/hooks/_capture/_replay.py | 2 +- src/autoskillit/hooks/_capture/_runner.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/autoskillit/hooks/_capture/_replay.py b/src/autoskillit/hooks/_capture/_replay.py index 49875fc90d..aed4c3819d 100644 --- a/src/autoskillit/hooks/_capture/_replay.py +++ b/src/autoskillit/hooks/_capture/_replay.py @@ -254,7 +254,7 @@ def degraded_delivery_return(shell_returncode: int | None) -> int: return shell_returncode -def _emit_degraded(failure: CaptureFailureV3) -> None: +def emit_degraded(failure: CaptureFailureV3) -> None: """Emit a degraded-delivery diagnostic marker to stderr.""" _write_and_flush_hook_stderr(render_capture_degraded_v3(failure) + b"\n") diff --git a/src/autoskillit/hooks/_capture/_runner.py b/src/autoskillit/hooks/_capture/_runner.py index cb281d3668..97d4763294 100644 --- a/src/autoskillit/hooks/_capture/_runner.py +++ b/src/autoskillit/hooks/_capture/_runner.py @@ -257,7 +257,7 @@ def _run_bounded_uncaptured( replace(failure, shell_returncode=outcome.shell_returncode) ) try: - _capture_replay._emit_degraded(replace(failure, shell_returncode=outcome.shell_returncode)) + _capture_replay.emit_degraded(replace(failure, shell_returncode=outcome.shell_returncode)) except _CAPTURE_RUNTIME_ERRORS: pass return _capture_replay.degraded_delivery_return(outcome.shell_returncode) @@ -461,7 +461,7 @@ def run_capture( settlement=None, ) try: - _capture_replay._emit_degraded(degraded_failure) + _capture_replay.emit_degraded(degraded_failure) except _CAPTURE_RUNTIME_ERRORS: pass # output already delivered; lost diagnostic return _capture_replay.degraded_delivery_return(command_returncode) From ce4b971ca06314de0e15469fe78ca61ed968f408 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:37:59 -0700 Subject: [PATCH 13/21] fix(review): Encapsulate capture-store projection stats --- src/autoskillit/hooks/_capture/_reconcile.py | 12 +++++------ .../hooks/_capture_lifecycle/_store.py | 20 +++++++++++++++++++ tests/cli/test_capture_store.py | 14 +++++++++---- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/autoskillit/hooks/_capture/_reconcile.py b/src/autoskillit/hooks/_capture/_reconcile.py index dfa2434309..b9f65148ce 100644 --- a/src/autoskillit/hooks/_capture/_reconcile.py +++ b/src/autoskillit/hooks/_capture/_reconcile.py @@ -200,12 +200,12 @@ def capture_store_stats(project_cwd: str) -> CaptureStoreStats: create=False, lock_wait=lock_wait_for_budget(RUNNER_TAIL_BUDGET), ) as store: - with store._locked(): - records, compaction_epoch, ledger_bytes = store._load_locked() - projected_compacted_bytes = store._ledger_view.sizer.total_bytes( - records, compaction_epoch, store._capacity - ) - compaction_low_bytes = store._capacity.compaction_low_bytes + ( + records, + ledger_bytes, + projected_compacted_bytes, + compaction_low_bytes, + ) = store.read_stats() tracked = frozenset( record.public_name for record in records.values() diff --git a/src/autoskillit/hooks/_capture_lifecycle/_store.py b/src/autoskillit/hooks/_capture_lifecycle/_store.py index cefb54c358..53350512dc 100644 --- a/src/autoskillit/hooks/_capture_lifecycle/_store.py +++ b/src/autoskillit/hooks/_capture_lifecycle/_store.py @@ -279,6 +279,26 @@ def _load_locked(self) -> tuple[dict[str, CaptureLifecycleRecord], int, int]: finally: os.close(fd) + def read_stats(self) -> tuple[dict[str, CaptureLifecycleRecord], int, int, int]: + """Read records and byte statistics for a capture-store report. + + The tuple contains records, physical ledger bytes, projected compacted + bytes, and the compaction low-water limit. + """ + with self._locked(): + records, compaction_epoch, ledger_bytes = self._load_locked() + projected_compacted_bytes = self._ledger_view.sizer.total_bytes( + records, + compaction_epoch, + self._capacity, + ) + return ( + records, + ledger_bytes, + projected_compacted_bytes, + self._capacity.compaction_low_bytes, + ) + def _append_locked( self, record: CaptureLifecycleRecord, diff --git a/tests/cli/test_capture_store.py b/tests/cli/test_capture_store.py index 056a3dd6dd..5603199028 100644 --- a/tests/cli/test_capture_store.py +++ b/tests/cli/test_capture_store.py @@ -22,6 +22,8 @@ import autoskillit.cli.doctor._doctor_capture_store as doctor_capture_store import autoskillit.cli.ops._capture_store as capture_store_command +import autoskillit.hooks._capture._capacity as capture_capacity +import autoskillit.hooks._capture_lifecycle as capture_lifecycle from autoskillit.cli.doctor._doctor_capture_store import _check_capture_store_stats from autoskillit.cli.ops import run_capture_store from autoskillit.core import Severity @@ -180,11 +182,15 @@ def test_capture_store_stats_reports_due_finalized_record_to_operators( ) with projection_store._locked(): records, epoch, _physical_size = projection_store._load_locked() - expected_projection = projection_store._ledger_view.sizer.total_bytes( - records, - epoch, - projection_store._capacity, + expected_projection = sum( + len( + capture_lifecycle._capture_ledger.encode_frame( + capture_lifecycle._capture_lifecycle_record.record_to_dict(record), + compaction_epoch=epoch, + ) ) + for record in capture_capacity.compacted_records(records, CaptureCapacitySpec()) + ) finally: projection_root.close() projection_anchor.close() From a5f5121f19c93692e5fdebeb11dac269f7485bd1 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:39:54 -0700 Subject: [PATCH 14/21] fix(review): Share capture-store capacity reporting --- .../cli/doctor/_doctor_capture_store.py | 20 +++-------------- src/autoskillit/cli/ops/_capture_store.py | 14 ++---------- src/autoskillit/hooks/_capture/_reconcile.py | 22 +++++++++++++++++++ 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/src/autoskillit/cli/doctor/_doctor_capture_store.py b/src/autoskillit/cli/doctor/_doctor_capture_store.py index 1c16ee5a19..3d6b12ea0c 100644 --- a/src/autoskillit/cli/doctor/_doctor_capture_store.py +++ b/src/autoskillit/cli/doctor/_doctor_capture_store.py @@ -5,7 +5,7 @@ from pathlib import Path from autoskillit.core import Severity -from autoskillit.hooks import CaptureStoreStats, CleanupBlocker, capture_store_stats +from autoskillit.hooks import CleanupBlocker, capture_store_stats from ._doctor_types import DoctorResult @@ -19,19 +19,6 @@ _RECLAMATION_DEBT_WARNING_THRESHOLD = 256 -def _capacity_line(stats: CaptureStoreStats) -> tuple[bool, str]: - utilization_pct = ( - f"{stats.projected_compacted_bytes / stats.compaction_low_bytes:.1%}" - if stats.compaction_low_bytes > 0 - else "n/a" - ) - return stats.capacity_warning, ( - f"capacity: projected_compacted_bytes={stats.projected_compacted_bytes} " - f"low_bytes={stats.compaction_low_bytes} utilization_pct={utilization_pct} " - "(default capacity limits; runtime overrides are runner-local)" - ) - - def _check_capture_store_stats(project_dir: Path | None = None) -> DoctorResult: """Report capture-store ledger/directory statistics — read-only, never reclaims. @@ -58,10 +45,9 @@ def _check_capture_store_stats(project_dir: Path | None = None) -> DoctorResult: f"unledgered_aged={stats.unledgered_aged_files} " f"unledgered_aged_bytes={stats.unledgered_aged_bytes}" ) - capacity_warning, capacity_message = _capacity_line(stats) - message = f"{message}; {capacity_message}" + message = f"{message}; {stats.capacity_line}" if ( - capacity_warning + stats.capacity_warning or stats.unledgered_aged_files >= _UNLEDGERED_WARNING_THRESHOLD or stats.due_records >= _RECLAMATION_DEBT_WARNING_THRESHOLD ): diff --git a/src/autoskillit/cli/ops/_capture_store.py b/src/autoskillit/cli/ops/_capture_store.py index f4b9d326c1..adc32cd3ca 100644 --- a/src/autoskillit/cli/ops/_capture_store.py +++ b/src/autoskillit/cli/ops/_capture_store.py @@ -63,19 +63,9 @@ def _print_stats(stats: CaptureStoreStats) -> None: f"files={stats.directory_files} unledgered_aged={stats.unledgered_aged_files} " f"unledgered_aged_bytes={stats.unledgered_aged_bytes}" ) - utilization_pct = ( - f"{stats.projected_compacted_bytes / stats.compaction_low_bytes:.1%}" - if stats.compaction_low_bytes > 0 - else "n/a" - ) - print( - "capture-store capacity: " - f"projected_compacted_bytes={stats.projected_compacted_bytes} " - f"low_bytes={stats.compaction_low_bytes} utilization_pct={utilization_pct} " - "(default capacity limits; runtime overrides are runner-local)" - ) + print(f"capture-store {stats.capacity_line}") if stats.capacity_warning: - print("WARNING: compacted capture-store bytes have reached 80% of the soft limit") + print(stats.capacity_warning_message) def run_capture_store(*, reclaim: bool = False) -> None: diff --git a/src/autoskillit/hooks/_capture/_reconcile.py b/src/autoskillit/hooks/_capture/_reconcile.py index b9f65148ce..c49922b779 100644 --- a/src/autoskillit/hooks/_capture/_reconcile.py +++ b/src/autoskillit/hooks/_capture/_reconcile.py @@ -180,6 +180,28 @@ def capacity_warning(self) -> bool: >= _CAPACITY_WARNING_RATIO * self.compaction_low_bytes ) + @property + def capacity_line(self) -> str: + """Shared operator-facing capacity line for doctor and CLI.""" + utilization_pct = ( + f"{self.projected_compacted_bytes / self.compaction_low_bytes:.1%}" + if self.compaction_low_bytes > 0 + else "n/a" + ) + return ( + f"capacity: projected_compacted_bytes={self.projected_compacted_bytes} " + f"low_bytes={self.compaction_low_bytes} utilization_pct={utilization_pct} " + "(default capacity limits; runtime overrides are runner-local)" + ) + + @property + def capacity_warning_message(self) -> str: + """Operator-facing warning using the configured soft-limit ratio.""" + return ( + "WARNING: compacted capture-store bytes have reached " + f"{_CAPACITY_WARNING_RATIO:.0%} of the soft limit" + ) + def capture_store_stats(project_cwd: str) -> CaptureStoreStats: """Report capture-store ledger and directory statistics without mutating. From d71defb403a8a20771cdf6d8dcd50592d041f626 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:40:45 -0700 Subject: [PATCH 15/21] fix(review): Move output-budget guidance beside its policy --- docs/safety/hooks.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/safety/hooks.md b/docs/safety/hooks.md index 44246ef255..228421f375 100644 --- a/docs/safety/hooks.md +++ b/docs/safety/hooks.md @@ -186,6 +186,10 @@ stdlib hook bridge as `output_budget_policy.disabled` (the inverse of that internal bridge shape after verified policy loading. No rendered-marker ceiling is implemented. +The temporary output-budget overlay workaround is removed on every +`close_kitchen`. To disable the guard durably, set +`output_budget: {guard_enabled: false}` in `.autoskillit/config.yaml`. + #### Capture Artifact Lifecycle | Phase | Behavior | @@ -402,10 +406,6 @@ load as `CaptureLedgerError`; a projected compacted-capacity refusal is a `CaptureCapacityError` with the capacity handling described in [ADR-0006](../decisions/0006-output-containment.md). -The temporary output-budget overlay workaround is removed on every -`close_kitchen`. To disable the guard durably, set -`output_budget: {guard_enabled: false}` in `.autoskillit/config.yaml`. - `autoskillit capture-store --reclaim` loops `reconcile_capture_store` with a generous one-time `RECLAIM_BUDGET` until a clean pass — no due records, no adoptable orphans — or a hard iteration cap. It exists for bulk pre-existing From 68903e673d510134b0e7d4d98fa72ac591e8a6ff Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:41:14 -0700 Subject: [PATCH 16/21] fix(review): Assert capture authority leaves a deleted record --- tests/hooks/test_shell_capture_conformance.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/hooks/test_shell_capture_conformance.py b/tests/hooks/test_shell_capture_conformance.py index 2c780ad1b4..1d0daa198d 100644 --- a/tests/hooks/test_shell_capture_conformance.py +++ b/tests/hooks/test_shell_capture_conformance.py @@ -370,6 +370,7 @@ def _runner_argv( project: Path, *, mode: str, + capture_id: str | None = None, ) -> list[str]: lineage_ref = _direct_lineage_reference(project) if mode == "direct" else None request = CaptureRequest( @@ -379,7 +380,7 @@ def _runner_argv( attempt_id=_DIRECT_ATTEMPT_ID if lineage_ref is not None else None, lineage_ref=lineage_ref, cwd=str(project), - capture_id=uuid4().hex[:16], + capture_id=capture_id if capture_id is not None else uuid4().hex[:16], command=command, ) return [ @@ -396,13 +397,14 @@ def _run_runner( *, mode: str, execution_dir: Path | None = None, + capture_id: str | None = None, ) -> subprocess.CompletedProcess[bytes]: if execution_dir is None: execution_dir = project else: assert not execution_dir.samefile(project) return subprocess.run( - _runner_argv(command, project, mode=mode), + _runner_argv(command, project, mode=mode, capture_id=capture_id), env=production_interpreter_env(), capture_output=True, cwd=execution_dir, @@ -590,11 +592,13 @@ def test_runner_keeps_project_and_execution_authorities_distinct( config.parent.mkdir(parents=True) config.write_text(json.dumps({"output_budget_policy": {"disabled": disabled}})) + capture_id = "d" * 16 completed = _run_runner( "printf 'pwd=%s\\n' \"$PWD\"; printf ran > execution-sentinel", project, mode=requested_mode, execution_dir=execution_dir, + capture_id=capture_id, ) assert completed.returncode == 0 @@ -606,6 +610,14 @@ def test_runner_keeps_project_and_execution_authorities_distinct( assert not _artifact_files(project) if expect_capture: assert (_capture_dir(project) / ".capture-lifecycle.ledger").is_file() + with open_capture_lifecycle( + str(project), + create=False, + lock_wait=HOT_PATH_LOCK_WAIT, + ) as lifecycle: + record = lifecycle.get_record(capture_id) + assert record is not None + assert record.state is CaptureState.DELETED assert not _capture_dir(execution_dir).exists() else: assert not _capture_dir(project).exists() From 854ae7a29cd6c71b1dfc18edc045bd794ee00f2a Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:42:53 -0700 Subject: [PATCH 17/21] fix(review): Assert fallback spawn receives the original command --- tests/hooks/test_capture_artifacts.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/hooks/test_capture_artifacts.py b/tests/hooks/test_capture_artifacts.py index 99b8b1a764..0753dc0028 100644 --- a/tests/hooks/test_capture_artifacts.py +++ b/tests/hooks/test_capture_artifacts.py @@ -259,11 +259,11 @@ def refuse(_store: CaptureLifecycleStore, _capture_id: str) -> CaptureLifecycleR def _install_fake_capture_process( monkeypatch: pytest.MonkeyPatch, process: _FakeCaptureProcess, -) -> list[dict[str, object]]: - spawn_calls: list[dict[str, object]] = [] +) -> list[tuple[tuple[object, ...], dict[str, object]]]: + spawn_calls: list[tuple[tuple[object, ...], dict[str, object]]] = [] - def spawn(*_args, **kwargs): - spawn_calls.append(kwargs) + def spawn(*args, **kwargs): + spawn_calls.append((args, kwargs)) return process monkeypatch.setattr(capture_runner, "_spawn_bash", spawn) @@ -1333,7 +1333,7 @@ def test_reserve_capacity_refusal_detail_carries_measurement( degraded.detail, ) assert len(degraded.detail.encode("utf-8")) <= 240 - assert spawn_calls == [{"capture_output": True}] + assert [kwargs for _, kwargs in spawn_calls] == [{"capture_output": True}] assert process.wait_calls == 1 @@ -1381,17 +1381,21 @@ def test_capacity_setup_refusal_runs_command_bounded_uncaptured( ) -> None: project = tmp_path / "project" project.mkdir() + command = "printf 'uncaptured child output'" expected = b"uncaptured child output" _install_reserve_capacity_refusal(monkeypatch, reason) process = _FakeCaptureProcess(expected, returncode=child_returncode) spawn_calls = _install_fake_capture_process(monkeypatch, process) - assert capture_runner._main(_runner_args(cwd=str(project))) == child_returncode + assert ( + capture_runner._main(_runner_args(command=command, cwd=str(project))) == child_returncode + ) captured = capfd.readouterr() degraded = _single_degraded_marker(captured.err) assert captured.out == expected.decode("utf-8") - assert spawn_calls == [{"capture_output": True}] + assert [kwargs for _, kwargs in spawn_calls] == [{"capture_output": True}] + assert spawn_calls[0][0][1] == command assert process.wait_calls == 1 assert degraded.stage == "capture_setup" assert degraded.reason is capture_capacity.failure_reason(reason) From 0083c5756170d37ef2bd2221e15ef2a213cf80ff Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 20:43:22 -0700 Subject: [PATCH 18/21] fix(review): Pin planned frame slack below eight bytes --- tests/hooks/test_capture_frame_bound.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/hooks/test_capture_frame_bound.py b/tests/hooks/test_capture_frame_bound.py index f5e716e016..c0af5a808a 100644 --- a/tests/hooks/test_capture_frame_bound.py +++ b/tests/hooks/test_capture_frame_bound.py @@ -169,13 +169,11 @@ def test_real_frame_shapes_fit_the_planning_allowance( sizes = tuple(_encoded_size(record) for record in profiled_records) measured_max = max(sizes) violations = capture_types.capacity_envelope_violations(capture_types.CaptureCapacitySpec()) - diagnostic = ( - f"measured_max={measured_max}, planned={capture_types.PLANNED_RECORD_FRAME_BYTES}, " - f"default_low={capture_types.CaptureCapacitySpec().compaction_low_bytes}, " - f"default_violations={violations}" - ) - assert measured_max <= capture_types.PLANNED_RECORD_FRAME_BYTES <= measured_max + 64, ( - diagnostic + assert violations == () + assert measured_max <= capture_types.PLANNED_RECORD_FRAME_BYTES < measured_max + 8, ( + f"measured_max={measured_max}, " + f"planned={capture_types.PLANNED_RECORD_FRAME_BYTES}, " + f"default_low={capture_types.CaptureCapacitySpec().compaction_low_bytes}" ) From 9ade5d6698ceb002023c9f0219832ab187d59905 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 21:15:25 -0700 Subject: [PATCH 19/21] fix(review): keep admission measurement fixtures within valid capacity limits --- tests/hooks/test_capture_capacity_model.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/hooks/test_capture_capacity_model.py b/tests/hooks/test_capture_capacity_model.py index dfe42b4086..d18bb20cf9 100644 --- a/tests/hooks/test_capture_capacity_model.py +++ b/tests/hooks/test_capture_capacity_model.py @@ -176,21 +176,21 @@ def _admission_decision( @pytest.mark.parametrize( - ("reason", "limit_field", "existing_count"), + ("reason", "limits", "existing_count"), [ ( capture_lifecycle.CaptureCapacityReason.ACTIVE_CAPACITY, - "max_operational_records", + {"max_operational_records": 3, "max_retained_records": 3}, 3, ), ( capture_lifecycle.CaptureCapacityReason.RETENTION_CAPACITY, - "max_retained_records", + {"max_retained_records": 1}, 1, ), ( capture_lifecycle.CaptureCapacityReason.EVIDENCE_CAPACITY, - "max_evidence_records", + {"max_evidence_records": 2}, 2, ), ], @@ -199,7 +199,7 @@ def _admission_decision( def test_admission_reason_reports_record_count_and_limit( tmp_path: Path, reason: capture_lifecycle.CaptureCapacityReason, - limit_field: str, + limits: dict[str, int], existing_count: int, ) -> None: spec = _capacity_spec( @@ -227,7 +227,7 @@ def test_admission_reason_reports_record_count_and_limit( decision = _admission_decision( records, candidate, - replace(spec, **{limit_field: existing_count}), + replace(spec, **limits), clock, ) finally: From 65c037a1af2f6f4f63bc6768b4fadf8dad60f5e8 Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 21:16:31 -0700 Subject: [PATCH 20/21] fix(review): render envelope diagnostics without module singleton state --- src/autoskillit/hooks/_capture/_types.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/autoskillit/hooks/_capture/_types.py b/src/autoskillit/hooks/_capture/_types.py index 63f5ea1876..ff618dcdaf 100644 --- a/src/autoskillit/hooks/_capture/_types.py +++ b/src/autoskillit/hooks/_capture/_types.py @@ -418,11 +418,10 @@ def capacity_envelope_violations(spec: CaptureCapacitySpec) -> tuple[str, ...]: return tuple(field for field, required, actual in requirements if actual < required) -_envelope_violations = capacity_envelope_violations(CaptureCapacitySpec()) -if _envelope_violations: +if capacity_envelope_violations(CaptureCapacitySpec()): raise AssertionError( "production capture capacity must cover the declared retention envelope: " - + ", ".join(_envelope_violations) + + ", ".join(capacity_envelope_violations(CaptureCapacitySpec())) ) DEBT_ASSIST_MAX_TRANSITIONS = 16 From 9620cd03f885616fd536612f79afb1026bbc544e Mon Sep 17 00:00:00 2001 From: Trecek Date: Sun, 25 Oct 2026 21:17:48 -0700 Subject: [PATCH 21/21] fix(review): profile finite timestamp widths in frame allowance test --- tests/hooks/test_capture_frame_bound.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/hooks/test_capture_frame_bound.py b/tests/hooks/test_capture_frame_bound.py index c0af5a808a..9adfdcf61b 100644 --- a/tests/hooks/test_capture_frame_bound.py +++ b/tests/hooks/test_capture_frame_bound.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +import sys from dataclasses import replace import pytest @@ -25,6 +26,7 @@ pytestmark = [pytest.mark.layer("hooks"), pytest.mark.medium] _MAX_NUMERIC = 2**63 - 1 +_MAX_TIMESTAMP = sys.float_info.max _COMPACTION_EPOCH = _MAX_NUMERIC _REFERENCE_CAPTURE_ID = "0000000000000001" _FAILURE_CAPTURE_ID = "0000000000000002" @@ -74,6 +76,9 @@ def _profile_record( project_identity=identity, root_identity=identity, artifact_identity=identity, + created_at=_MAX_TIMESTAMP, + next_attempt_at=_MAX_TIMESTAMP, + retention_at=_MAX_TIMESTAMP if record.retention_at is not None else None, observed_size=_MAX_NUMERIC, revision=_MAX_NUMERIC, finalized_at_revision=finalized_at_revision,