diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40e2280..76029eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,14 +43,16 @@ jobs: - name: Lint all targets run: cargo clippy --all-targets --all-features --locked -- -D warnings - - name: Test all targets and verify acceptance coverage + - name: Test all targets and verify acceptance and lifecycle coverage run: python3 tools/run_acceptance_scenarios.py --all-targets - - name: Retain normalized acceptance evidence + - name: Retain normalized acceptance and lifecycle evidence uses: actions/upload-artifact@v4 with: name: acceptance-scenarios-${{ matrix.toolchain }} - path: target/acceptance-scenarios.v1.report.json + path: | + target/acceptance-scenarios.v1.report.json + target/lifecycle-scenarios.v1.report.json if-no-files-found: error retention-days: 7 @@ -93,6 +95,9 @@ jobs: - name: Reject incomplete acceptance reports run: python3 tools/test_acceptance_report.py + - name: Reject incomplete lifecycle reports and verify test resource limits + run: python3 tools/test_lifecycle_report.py + - name: Check durable contract references and closed shapes run: python3 tools/test_durable_contracts.py diff --git a/README.md b/README.md index 75f52ab..411cb84 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ or a sandbox. - [Artifact binding contract](docs/integrations/artifact-bindings.md) - [Scenario fixture contract](docs/integrations/scenario-fixtures.md) - [Executable acceptance matrix](docs/integrations/acceptance-scenarios.md) +- [Durable lifecycle scenarios and recovery evidence](docs/integrations/lifecycle-scenarios.md) - [Hermetic provider kit](docs/integrations/hermetic-provider-kit.md) - [Versioned contracts](contracts/README.md) - [Roadmap](ROADMAP.md) diff --git a/ROADMAP.md b/ROADMAP.md index b05f07e..3ecb47d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -3,7 +3,7 @@ schema: aether.architecture-document/v1 id: flow-roadmap title: Flow Roadmap kind: architecture-document -version: 1.3.3 +version: 1.3.4 status: draft owners: - egohygiene @@ -43,9 +43,14 @@ It adds [durable prepared execution](docs/integrations/durable-state.md): versioned plans and state, atomic immutable snapshots, workspace locking, accepted checkpoints, fresh resume eligibility, and explicit recovery decisions. #31 is active through three dependency-ordered review checkpoints: -[#64](https://github.com/egohygiene/flow/issues/64) adds fresh graph assessment and -safe dependent execution; [#65](https://github.com/egohygiene/flow/issues/65) adds -the deterministic durable lifecycle corpus; [#66](https://github.com/egohygiene/flow/issues/66) +[#64](https://github.com/egohygiene/flow/issues/64) merged through +[PR #67](https://github.com/egohygiene/flow/pull/67) as +`83f1ea161aa5aba03da5de6b287005252000cd4b`, with green +[default-branch CI](https://github.com/egohygiene/flow/actions/runs/36226457853). +It adds fresh graph assessment and safe dependent execution. +[#65](https://github.com/egohygiene/flow/issues/65) adds the +[deterministic durable lifecycle corpus](docs/integrations/lifecycle-scenarios.md) +with 34 recipes executed twice, fresh-process restarts, and bounded receipts; [#66](https://github.com/egohygiene/flow/issues/66) proves authority, duplicate-effect prevention, and recovery residuals. Land one review PR and verify default-branch CI before starting the next checkpoint. FLO-Q03 remains active until real released-provider adapters satisfy its @@ -104,7 +109,8 @@ Renderflow #421 → #422–#430 → Flow #10 exhaustive comic workflow ``` The suite therefore has four useful parallel ready fronts: -Flow #30, Renderflow #415, Optiflow #88, and Aniflow #8. Keep provider domain +Flow #65 (then #66 after merge and green default-branch CI), Renderflow #415, +Optiflow #88, and Aniflow #8. Keep provider domain logic in the provider repositories and consume only immutable public contracts from Flow. diff --git a/docs/architecture/foundation/ARCHITECTURE.md b/docs/architecture/foundation/ARCHITECTURE.md index a4fe739..f040ee0 100644 --- a/docs/architecture/foundation/ARCHITECTURE.md +++ b/docs/architecture/foundation/ARCHITECTURE.md @@ -3,7 +3,7 @@ schema: aether.architecture-document/v1 id: flow-architecture title: Flow Architecture kind: architecture-document -version: 0.13.0 +version: 0.13.1 status: draft owners: - egohygiene @@ -133,6 +133,13 @@ the assessment, and single-step entry points refuse dependent steps. Automatic scheduling, retry, cross-plan migration, and provider-native checkpoints remain later orchestration work. +The test-owned lifecycle corpus exercises these same public durable APIs with +synthetic providers, immutable transition projections, fresh-process reopening, +and explicit recovery decisions. Its versioned receipts are conformance evidence, +not executable plans, authorization tokens, or a second runtime state model. +Portable reports contain allowlisted identities and typed outcomes; raw operational +evidence stays local. The lifecycle guide owns recipe coverage and resource limits. + ### External adapters Adapters isolate process invocation, version/capability probing, structured @@ -281,7 +288,9 @@ production graph scheduler. Issue #49 adds the versioned durable coordinator, immutable prepared plans, atomic snapshots, accepted checkpoints, fresh reuse assessment, and explicit recovery decisions described above. Issue #64 requires fresh prerequisite evidence for graph assessment and dependent execution without -adding a scheduler or rewriting accepted history. Flow does not yet supply +adding a scheduler or rewriting accepted history. Issue #65 adds the bounded +[durable lifecycle corpus](../../integrations/lifecycle-scenarios.md), including +fresh-process restart and deterministic recovery receipts. Flow does not yet supply the public CLI, real holon adapters, signature or transparency verification, provider-native artifact validation, an atomic filesystem snapshot, an operating-system sandbox or authenticated enforcement evidence, diff --git a/docs/integrations/acceptance-scenarios.md b/docs/integrations/acceptance-scenarios.md index 18cae11..0789f8e 100644 --- a/docs/integrations/acceptance-scenarios.md +++ b/docs/integrations/acceptance-scenarios.md @@ -44,7 +44,8 @@ python3 tools/run_acceptance_scenarios.py python3 tools/run_acceptance_scenarios.py --all-targets ``` -The driver uses `cargo test --locked --offline` and writes +The Linux driver uses `cargo test --locked --offline` with one Rust test worker +to isolate fork-inherited test locks (explicit contention tests still run), and writes `target/acceptance-scenarios.v1.report.json`. It removes an older report before starting, fails on any failed Rust test or missing/duplicate/unexpected receipt, and records the catalog digest, tested source-file digests, Cargo version, @@ -57,12 +58,15 @@ The PR catalog budgets 30 seconds per execution, two executions per case, 16 KiB per receipt, four immediate output entries, and 1 MiB of generated regular-file output. The fixed recipes create only files and empty directories; this output-count budget is not a general recursive filesystem quota. The -driver has a 600-second Cargo wait timeout and a 4 MiB post-capture test-log -acceptance limit; it is not a general descendant-process supervisor. Provider +Linux driver enforces a 600-second command deadline and a 4 MiB output limit +while capturing. A Cargo target runner limits each test/provider process to +4 GiB address space and each written file to 16 MiB; compilation is excluded +from these kernel limits. Failed or timed-out test commands have their process +group killed and reaped. This is test supervision, not production containment. Provider stdout/stderr and process deadlines remain enforced by the existing locked kit limits. Budget measurements exclude compilation from the per-case time, but -include compilation in the driver timeout. Memory, whole-filesystem limits, -and kernel network isolation remain explicit gaps. +include compilation in the driver timeout. Aggregate process-tree memory, +whole-filesystem quotas, and kernel network isolation remain explicit gaps. The test profile omits debug symbols because the exact provider executable is copied and hashed repeatedly. This keeps generated package size and PR time @@ -83,7 +87,9 @@ validator; the receipt identifies the validator outcome, not a second launch. The kit is synthetic. Real released providers, native format validators, cryptographic publisher authentication, OS sandboxing, atomic filesystem snapshots, descriptor-bound launch, and descendant containment are not proven. -Flow #49 owns durable state; #31 owns retry and resume. The scenario catalog +The [durable lifecycle corpus](lifecycle-scenarios.md) separately qualifies +#49/#64 state and recovery APIs for #31. `--all-targets` verifies both corpora +and retains both reports. This acceptance scenario catalog does not expand those checkpoints or claim that two synthetic fixtures are two real provider adapters. diff --git a/docs/integrations/durable-state.md b/docs/integrations/durable-state.md index e2677af..44bf646 100644 --- a/docs/integrations/durable-state.md +++ b/docs/integrations/durable-state.md @@ -4,7 +4,7 @@ Flow #49 adds a library API for persistent execution of fully prepared process steps. It builds on the exact subject, authority, transcript, and artifact gates. Flow #64 adds fresh graph eligibility and safe caller-selected dependent execution. There is no product CLI or graph scheduler. #31 continues through -#65 (the lifecycle corpus) and #66 (authority/effect and residual-state proofs); +#65 (the [lifecycle corpus](lifecycle-scenarios.md)) and #66 (authority/effect and residual-state proofs); #53 owns the supported CLI. ## Public entry points @@ -205,5 +205,7 @@ cover deterministic fresh-root/reopen reports, transitive and branch invalidatio all local identity boundaries, complete inventories, stale-report non-authority, the single-step bypass, blocked future inputs, and preserved history. The acceptance driver includes these test sources in its evidence identity and runs -them with `--all-targets`; the broader lifecycle receipt catalog belongs to #65. +them with `--all-targets`, along with the 34-row [lifecycle receipt corpus](lifecycle-scenarios.md) +from #65. The latter runs every recipe twice, includes fresh-process recovery, +and checks bounded portable reports. macOS/Windows CI runs the portable store suite; Linux runs the full provider matrix. diff --git a/docs/integrations/lifecycle-scenarios.md b/docs/integrations/lifecycle-scenarios.md new file mode 100644 index 0000000..44686e5 --- /dev/null +++ b/docs/integrations/lifecycle-scenarios.md @@ -0,0 +1,131 @@ +# Durable lifecycle scenarios + +Flow #65 qualifies durable recovery through the public APIs introduced by #49 +and #64. The test-owned corpus lives in `tests/lifecycle_matrix/`, with 34 fixed +recipes and expected outcomes in `tests/fixtures/lifecycle-scenarios.v1.json`. +It supplements the [acceptance matrix](acceptance-scenarios.md). It does not add +a scheduler, a runtime state model, or a new public contract. + +## What the evidence proves + +Every recipe runs twice in independent workspaces, with the exact compiled +synthetic provider package. Both executions must produce equal normalized +receipts. Expectations are checked in as reviewed assertions; tests never +regenerate expected outcomes from observed results. + +| Family | Scenarios | Required evidence | +| --- | --- | --- | +| Cancellation | Before launch, after intent, during provider execution, between steps | Ordered snapshots, attempt numbers, absent checkpoints, current graph eligibility | +| Provider failures | Timeout, nonzero exit, actual Unix signal, stdout/stderr overflow, partial output | Exact typed process or acceptance error; failed evidence retained without promotion | +| Fresh-process restart | Completed run, partial graph, host exit after intent, host exit after provider reaping | A new host process reopens the same workspace, reconstructs current public contexts, and reuses or continues only eligible work | +| Recovery | Retry cancelled or interrupted work, abandon invalid partial output, refuse unacknowledged or completed retries | Explicit recovery decisions, ordered attempt transitions, preserved uncertain output before retry | +| Staleness | Input, exact plan, configuration values, provider identity, executable bytes, validator implementation, output bytes | Local stale boundary, downstream invalidation, independent branch reuse, dependent launch refusal without state writes | +| Store refusal | Changed validation profile, corrupt checkpoint context, checksum mismatch, partial write, future schema, malformed JSON | Exact reopen refusal; no fallback to an older accepted snapshot | + +The graph fixture is A → B plus independent C, with separate source bindings. +It proves ordering dependencies and stale-ancestor propagation. The existing +provider-kit compositions separately prove artifact handoff; #64's graph tests +cover transitive chains, fan-in, and complete context inventories. + +## Trace and receipt contract + +The test-owned `flow.lifecycle-scenario-catalog/v1` names each recipe's provider +mode, graph shape, recovery classification, expected ordered state frames, and +expected ordered assessments. Frames project the existing `RunState` snapshots: +sequence, status, attempt, checkpoint presence, typed failure, and recorded +recovery actions. Assessments reuse `RunStepAssessment` fields directly. + +`flow.lifecycle-scenario-receipt/v1` adds the exact recipe digest, observed package, +executable, manifest, binding and source identities, immutable plan digest, +retained history digest, and candidate artifact digests. On corrupt-history +recipes, frames describe the valid history before fault injection; the final +history digest identifies the rejected snapshot bytes. No successful reopen or +new assessment is implied by those frames. + +The fixture preserves originals before deliberate byte corruption and retains +failed candidates. Interrupted output is moved to a separate retained file +before an explicitly approved retry. An ignored Rust helper is a subprocess +entry point, explicitly executed by the matrix; it is not a skipped scenario. +Host exit code 73 bypasses destructors and proves lock release and fresh reopening. +The two exit points are before provider launch and after the runner reaps its +direct child but before durable acceptance. They do not simulate host death +while an unconfined provider or its descendants remain alive. + +Recovery classifications describe each **test recipe's operator decision**. +Two provider-failure recipes revalidate the exact retained process transcript +through Flow: a provider-classified retryable failure and a terminal validation +failure. Both remain unaccepted; only an explicit operator decision moves them +to pending or abandoned. Provider retryability hints do not trigger execution. +Cancellation or interruption may be retried after acknowledgement and fresh +context checks; invalid partial output is abandoned in its terminal recipe. +Flow still exposes typed `RunFailure` and `ResumeEligibility`, not an automatic +retryability policy. Reopening, assessing, or deserializing a receipt grants no +authority and launches nothing. The [durable-state guide](durable-state.md) owns +production storage and recovery semantics. + +## Running and checking coverage + +Populate Cargo's locked cache, then run on Linux: + +```console +python3 tools/run_acceptance_scenarios.py --lifecycle-only +python3 tools/run_acceptance_scenarios.py --all-targets +python3 tools/test_lifecycle_report.py +``` + +The first command writes `target/lifecycle-scenarios.v1.report.json`. The second +runs all Rust targets and verifies both this corpus and all 81 acceptance rows. +CI runs it on Rust 1.85 and stable and uploads both normalized JSON reports. +The driver removes older reports before running, captures output with a bound, +and refuses any failed test, missing/duplicate/unknown receipt, changed expected +trace, unsupported shape/version, recipe digest mismatch, inconsistent provider +identity, or fixture source mismatch. It hashes the tested sources before and +after execution; changes during validation prevent a successful report. + +Reports carry source and catalog digests, the actual toolchain, checked budgets, +coverage gaps, and receipts. Rust compares both fresh-workspace executions; +Python independently checks completeness and each exact expected outcome. +Failure logs and raw process output remain local, outside CI uploads. The +portable allowlist excludes absolute roots, PIDs, timing samples, configuration +values, source bytes, provider messages, and raw streams. Canary assertions check +fixture paths and values. This is not a general redactor for arbitrary content. + +## Resource bounds and remaining work + +| Resource | Qualification bound | +| --- | --- | +| Rust test workers | One; explicit child-process contention tests still overlap their opens | +| Per recipe | Two executions; each completed execution must take at most 60 seconds | +| Entire Cargo command | 600 seconds, including compilation; timeout kills and reaps the test command's process group | +| Captured command output | 4 MiB enforced while reading, before any unbounded capture | +| Test/provider process address space | Linux `RLIMIT_AS`: 4 GiB per process, inherited by host children and providers | +| Single file written by a test/provider | Linux `RLIMIT_FSIZE`: 16 MiB; core dumps disabled | +| Receipt | 32 KiB | +| Durable history | At most 16 snapshots and 2 MiB per recipe workspace | +| Fixture artifacts | At most 16 recursive files and 1 MiB per node artifact root | +| All fixture files | 128 MiB across the recipe's roots, including copied provider binaries and retained evidence | +| Provider execution | Existing pinned kit deadline (5 seconds), 64 KiB stdout and stderr limits, cancellation grace | + +The driver serializes Rust test workers because concurrent fork/exec can briefly +inherit another worker's Unix `flock` descriptors and make an unrelated immediate +reopen report `Busy`. This is test isolation, matching the portable store suite; +no lock assertions are weakened and the deliberate contention tests still run. +Concurrent unrelated host spawning is not qualified by this serial tier. + +The Cargo target runner applies kernel limits to test executables, not compiler +or linker processes. Direct `cargo test` is useful for debugging but does not +establish the memory/file qualification. Per-recipe elapsed time and aggregate +filesystem totals are checked after execution; an otherwise hung helper is +bounded by the driver's command deadline. The address-space limit is not an +aggregate process-tree RSS quota. Killing the test process group on failure is +test supervision, not a production descendant-containment claim. + +This tier qualifies Linux synthetic trusted-unconfined providers. Existing +macOS/Windows durable-store jobs provide narrower portability evidence. Real +provider releases, sandbox enforcement, hardware power loss, every crash window, +provider-native checkpoints, migration, automatic scheduling/retry, and +exactly-once external effects remain outside this proof. **Flow #66** owns the +remaining authority transitions, effect counters, and residual cleanup +qualification. Parent **#31 stays open** until that checkpoint passes. Future +scenario visualization in #62 can consume these checked receipts while retaining +these coverage limits. diff --git a/tests/fixtures/hermetic-provider/main.rs b/tests/fixtures/hermetic-provider/main.rs index de330e4..7d44f21 100644 --- a/tests/fixtures/hermetic-provider/main.rs +++ b/tests/fixtures/hermetic-provider/main.rs @@ -47,6 +47,9 @@ enum Behavior { CorruptArtifactEvidence, ContradictoryArtifactEvidence, NonzeroAfterSuccess, + SignalTermination, + RetryableFailure, + TerminalFailure, AwaitInterruption, StdoutOverflow, StderrOverflow, @@ -67,6 +70,9 @@ impl Behavior { "corrupt-artifact-evidence" => Ok(Self::CorruptArtifactEvidence), "contradictory-artifact-evidence" => Ok(Self::ContradictoryArtifactEvidence), "nonzero-after-success" => Ok(Self::NonzeroAfterSuccess), + "signal-termination" => Ok(Self::SignalTermination), + "retryable-failure" => Ok(Self::RetryableFailure), + "terminal-failure" => Ok(Self::TerminalFailure), "await-interruption" => Ok(Self::AwaitInterruption), "stdout-overflow" => Ok(Self::StdoutOverflow), "stderr-overflow" => Ok(Self::StderrOverflow), @@ -140,6 +146,11 @@ fn run() -> ProviderResult<()> { } match behavior { + Behavior::SignalTermination => { + #[cfg(unix)] + nix::sys::signal::raise(nix::sys::signal::Signal::SIGTERM)?; + return Err(invalid_input("signal termination requires Unix signals").into()); + } Behavior::StdoutOverflow => { let stdout = io::stdout(); write_overflow(&mut stdout.lock(), invocation.limits.max_stdout_bytes)?; @@ -354,6 +365,33 @@ fn run() -> ProviderResult<()> { message: "The hermetic provider completed with synthetic warning evidence.".to_owned(), redacted: true, }), + Behavior::RetryableFailure | Behavior::TerminalFailure => { + result.outcome = Outcome::Failed; + result.failure = Failure { + classification: if behavior == Behavior::RetryableFailure { + FailureClassification::Provider + } else { + FailureClassification::Validation + }, + code: "hermetic.failure".to_owned(), + message: "FLOW_LIFECYCLE_FAILURE_PRIVATE_CANARY".to_owned(), + retryable: behavior == Behavior::RetryableFailure, + }; + events[2].kind = EventKind::PhaseFailed; + events[2].state = EventState::Failed; + // Retain the exact transcript as host-local failed evidence. The + // lifecycle test revalidates it through the public transcript API. + let mut transcript = Vec::new(); + for event in &events { + write_record(&mut transcript, event)?; + } + write_record(&mut transcript, &result)?; + write_new_output( + &root, + Path::new("outputs/failure-transcript.jsonl"), + &transcript, + )?; + } Behavior::PartialResult | Behavior::PartialOutput => result.partial_result = true, Behavior::CorruptArtifactEvidence => result.provenance[2].value.clear(), Behavior::ContradictoryArtifactEvidence => events[1].artifact_refs.clear(), @@ -367,7 +405,10 @@ fn run() -> ProviderResult<()> { | Behavior::ExtraOutput | Behavior::NonzeroAfterSuccess | Behavior::SuccessWithHostRejection => {} - Behavior::AwaitInterruption | Behavior::StdoutOverflow | Behavior::StderrOverflow => { + Behavior::AwaitInterruption + | Behavior::SignalTermination + | Behavior::StdoutOverflow + | Behavior::StderrOverflow => { unreachable!("non-artifact behaviors return before evidence construction") } } diff --git a/tests/fixtures/lifecycle-scenarios.v1.json b/tests/fixtures/lifecycle-scenarios.v1.json new file mode 100644 index 0000000..93a0844 --- /dev/null +++ b/tests/fixtures/lifecycle-scenarios.v1.json @@ -0,0 +1,3955 @@ +{ + "schema_version": "flow.lifecycle-scenario-catalog/v1", + "fixture_version": "1.0.0", + "tier": "pull-request-linux", + "budget": { + "repetitions": 2, + "timeout_ms": 60000, + "max_receipt_bytes": 32768, + "max_snapshots": 16, + "max_history_bytes": 2097152, + "max_artifacts": 16, + "max_artifact_bytes": 1048576, + "max_fixture_bytes": 134217728, + "max_process_address_space_bytes": 4294967296, + "max_process_file_bytes": 16777216, + "max_driver_output_bytes": 4194304, + "driver_timeout_seconds": 600, + "test_threads": 1 + }, + "known_gaps": [ + "Authority transition/effect counters and residual cleanup qualification remain Flow #66.", + "Synthetic trusted-unconfined providers only; no real holon, sandbox, or descendant-containment qualification.", + "Host exits occur after intent before launch, or after direct-provider reaping before acceptance; live-child host death and every instruction/commit crash window are not covered.", + "Linux corpus only; existing macOS/Windows durable-store jobs provide narrower portability evidence.", + "No migration, automatic retry/scheduling, provider-native checkpoints, exactly-once effects, or hardware/power-loss guarantee.", + "Address-space and file-size limits are per process, not an aggregate process-tree memory quota; fixture/history totals are checked after each recipe.", + "Rust workers run serially to isolate fork-inherited test locks; concurrent unrelated host spawning is not qualified, while explicit workspace contention tests still run." + ], + "scenarios": [ + { + "scenario_id": "scenario:lifecycle-cancel-before-launch", + "recipe": "cancel-before-launch", + "mode": "success", + "graph": false, + "recovery": "operator-decision-required", + "expected": { + "code": "cancelled-before-launch", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "cancelled", + "attempt": 0, + "failure": "cancelled", + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 1, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "cancelled", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-cancel-after-intent", + "recipe": "cancel-after-intent", + "mode": "success", + "graph": false, + "recovery": "operator-decision-required", + "expected": { + "code": "cancelled-before-launch", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "cancelled", + "attempt": 1, + "failure": "cancelled", + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "cancelled", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-cancel-during-provider", + "recipe": "cancel-during-provider", + "mode": "await-interruption", + "graph": false, + "recovery": "operator-decision-required", + "expected": { + "code": "cancelled", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "cancelled", + "attempt": 1, + "failure": "cancelled", + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "cancelled", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-timeout", + "recipe": "timeout", + "mode": "await-interruption", + "graph": false, + "recovery": "operator-decision-required", + "expected": { + "code": "timed-out", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "failed", + "attempt": 1, + "failure": "timed-out", + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "failed", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-nonzero-exit", + "recipe": "nonzero-exit", + "mode": "nonzero-after-success", + "graph": false, + "recovery": "operator-decision-required", + "expected": { + "code": "process-exit-7", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "failed", + "attempt": 1, + "failure": "process", + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "failed", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-signal-termination", + "recipe": "signal-termination", + "mode": "signal-termination", + "graph": false, + "recovery": "operator-decision-required", + "expected": { + "code": "process-signal", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "failed", + "attempt": 1, + "failure": "process", + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "failed", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-stdout-limit", + "recipe": "stdout-limit", + "mode": "stdout-overflow", + "graph": false, + "recovery": "operator-decision-required", + "expected": { + "code": "stdout-limit", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "failed", + "attempt": 1, + "failure": "process", + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "failed", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-stderr-limit", + "recipe": "stderr-limit", + "mode": "stderr-overflow", + "graph": false, + "recovery": "operator-decision-required", + "expected": { + "code": "stderr-limit", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "failed", + "attempt": 1, + "failure": "process", + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "failed", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-partial-output", + "recipe": "partial-output", + "mode": "partial-output", + "graph": false, + "recovery": "operator-decision-required", + "expected": { + "code": "partial-output-rejected", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "failed", + "attempt": 1, + "failure": "artifact-acceptance", + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "failed", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-cancel-between-steps", + "recipe": "cancel-between-steps", + "mode": "success", + "graph": true, + "recovery": "operator-decision-required", + "expected": { + "code": "cancelled-between-steps", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 3, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + }, + { + "status": "cancelled", + "attempt": 0, + "failure": "cancelled", + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "dependency-blocked", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "first-complete", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:c", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 3, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "cancelled", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:c", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-completed-restart", + "recipe": "completed-restart", + "mode": "success", + "graph": false, + "recovery": "reuse-accepted-work", + "expected": { + "code": "completed-reused", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-restart", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "reopened", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-partial-restart", + "recipe": "partial-restart", + "mode": "success", + "graph": true, + "recovery": "continue-ready-work", + "expected": { + "code": "partial-run-resumed", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 3, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + }, + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 4, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "dependency-blocked", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-restart", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:c", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "reopened", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:c", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:c", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-host-exit-after-intent", + "recipe": "host-exit-after-intent", + "mode": "success", + "graph": false, + "recovery": "operator-decision-required", + "expected": { + "code": "uncertain-intent-retained", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "reopened", + "sequence": 1, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "running", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 1, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "running", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-host-exit-after-provider", + "recipe": "host-exit-after-provider", + "mode": "success", + "graph": false, + "recovery": "operator-decision-required", + "expected": { + "code": "uncertain-intent-retained", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "reopened", + "sequence": 1, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "running", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 1, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "running", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-retry-interrupted", + "recipe": "retry-interrupted", + "mode": "success", + "graph": false, + "recovery": "retry-with-acknowledgement", + "expected": { + "code": "interrupted-retry-succeeded", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "pending", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [ + "retry" + ] + }, + { + "sequence": 3, + "steps": [ + { + "status": "running", + "attempt": 2, + "failure": null, + "checkpoint": false + } + ], + "recovery": [ + "retry" + ] + }, + { + "sequence": 4, + "steps": [ + { + "status": "succeeded", + "attempt": 2, + "failure": null, + "checkpoint": true + } + ], + "recovery": [ + "retry" + ] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "reopened", + "sequence": 1, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "running", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "retry-approved", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-retry-cancelled", + "recipe": "retry-cancelled", + "mode": "success", + "graph": false, + "recovery": "retry-with-acknowledgement", + "expected": { + "code": "explicit-retry-succeeded", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "cancelled", + "attempt": 0, + "failure": "cancelled", + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [ + "retry" + ] + }, + { + "sequence": 3, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [ + "retry" + ] + }, + { + "sequence": 4, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [ + "retry" + ] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "recovery-required", + "sequence": 1, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "cancelled", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "retry-approved", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-retry-without-ack", + "recipe": "retry-without-ack", + "mode": "success", + "graph": false, + "recovery": "refuse-retry", + "expected": { + "code": "unacknowledged-retry-refused", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "cancelled", + "attempt": 0, + "failure": "cancelled", + "checkpoint": false + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 1, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "cancelled", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-abandon-terminal", + "recipe": "abandon-terminal", + "mode": "partial-output", + "graph": false, + "recovery": "abandon-invalid-evidence", + "expected": { + "code": "explicit-abandon", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "failed", + "attempt": 1, + "failure": "artifact-acceptance", + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 3, + "steps": [ + { + "status": "abandoned", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [ + "abandon" + ] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "recovery-required", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "failed", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 3, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "abandoned", + "eligibility": "abandoned", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-retry-completed", + "recipe": "retry-completed", + "mode": "success", + "graph": false, + "recovery": "reuse-accepted-work", + "expected": { + "code": "completed-retry-refused", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-changed-input", + "recipe": "changed-input", + "mode": "success", + "graph": true, + "recovery": "rebuild-current-context", + "expected": { + "code": "stale-branch-invalidated", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + }, + { + "sequence": 3, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + }, + { + "sequence": 4, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "dependency-blocked", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-change", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:c", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "invalidated", + "stale_boundary": "inputs", + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "invalidated", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-changed-configuration", + "recipe": "changed-configuration", + "mode": "success", + "graph": true, + "recovery": "rebuild-current-context", + "expected": { + "code": "stale-branch-invalidated", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + }, + { + "sequence": 3, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + }, + { + "sequence": 4, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "dependency-blocked", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-change", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:c", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "invalidated", + "stale_boundary": "configuration", + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "invalidated", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-changed-provider", + "recipe": "changed-provider", + "mode": "success", + "graph": true, + "recovery": "rebuild-current-context", + "expected": { + "code": "stale-branch-invalidated", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + }, + { + "sequence": 3, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + }, + { + "sequence": 4, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "dependency-blocked", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-change", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:c", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "invalidated", + "stale_boundary": "provider", + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "invalidated", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-changed-implementation", + "recipe": "changed-implementation", + "mode": "success", + "graph": true, + "recovery": "rebuild-current-context", + "expected": { + "code": "stale-branch-invalidated", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + }, + { + "sequence": 3, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + }, + { + "sequence": 4, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "dependency-blocked", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-change", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:c", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "invalidated", + "stale_boundary": "provider", + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "invalidated", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-changed-artifact", + "recipe": "changed-artifact", + "mode": "success", + "graph": true, + "recovery": "rebuild-current-context", + "expected": { + "code": "stale-branch-invalidated", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + }, + { + "sequence": 3, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + }, + { + "sequence": 4, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "dependency-blocked", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-change", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:c", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "invalidated", + "stale_boundary": "artifacts", + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "invalidated", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-changed-validator", + "recipe": "changed-validator", + "mode": "success", + "graph": true, + "recovery": "rebuild-current-context", + "expected": { + "code": "stale-branch-invalidated", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + }, + { + "sequence": 3, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + }, + { + "sequence": 4, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + }, + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + }, + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "dependency-blocked", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-change", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + }, + { + "step_id": "step:c", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 4, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "invalidated", + "stale_boundary": "validation", + "blocked_by": [] + }, + { + "step_id": "step:b", + "recorded_status": "pending", + "eligibility": "invalidated", + "stale_boundary": null, + "blocked_by": [ + "step:a" + ] + }, + { + "step_id": "step:c", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-changed-plan", + "recipe": "changed-plan", + "mode": "success", + "graph": false, + "recovery": "rebuild-current-context", + "expected": { + "code": "changed-plan-refused", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-change", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-changed-validation-profile", + "recipe": "changed-validation-profile", + "mode": "success", + "graph": false, + "recovery": "preserve-and-repair-store", + "expected": { + "code": "invalid-validation-profile", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-change", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-corrupt-checkpoint", + "recipe": "corrupt-checkpoint", + "mode": "success", + "graph": false, + "recovery": "preserve-and-repair-store", + "expected": { + "code": "invalid-checkpoint-context", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-change", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-corrupt-checksum", + "recipe": "corrupt-checksum", + "mode": "success", + "graph": false, + "recovery": "preserve-and-repair-store", + "expected": { + "code": "corrupt-snapshot", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-change", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-incomplete-write", + "recipe": "incomplete-write", + "mode": "success", + "graph": false, + "recovery": "preserve-and-repair-store", + "expected": { + "code": "incomplete-write", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-change", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-future-schema", + "recipe": "future-schema", + "mode": "success", + "graph": false, + "recovery": "preserve-and-repair-store", + "expected": { + "code": "unsupported-schema", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-change", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-malformed-state", + "recipe": "malformed-state", + "mode": "success", + "graph": false, + "recovery": "preserve-and-repair-store", + "expected": { + "code": "malformed-state", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "succeeded", + "attempt": 1, + "failure": null, + "checkpoint": true + } + ], + "recovery": [] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "before-change", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "succeeded", + "eligibility": "reusable", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-retryable-provider-failure", + "recipe": "retryable-provider-failure", + "mode": "retryable-failure", + "graph": false, + "recovery": "retry-with-acknowledgement", + "expected": { + "code": "retryable-provider-failure-approved", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "failed", + "attempt": 1, + "failure": "artifact-acceptance", + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 3, + "steps": [ + { + "status": "pending", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [ + "retry" + ] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "recovery-required", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "failed", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 3, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + }, + { + "scenario_id": "scenario:lifecycle-terminal-provider-failure", + "recipe": "terminal-provider-failure", + "mode": "terminal-failure", + "graph": false, + "recovery": "abandon-invalid-evidence", + "expected": { + "code": "terminal-validation-failure-abandoned", + "history": [ + { + "sequence": 0, + "steps": [ + { + "status": "pending", + "attempt": 0, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 1, + "steps": [ + { + "status": "running", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 2, + "steps": [ + { + "status": "failed", + "attempt": 1, + "failure": "artifact-acceptance", + "checkpoint": false + } + ], + "recovery": [] + }, + { + "sequence": 3, + "steps": [ + { + "status": "abandoned", + "attempt": 1, + "failure": null, + "checkpoint": false + } + ], + "recovery": [ + "abandon" + ] + } + ], + "assessments": [ + { + "at": "initial", + "sequence": 0, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "pending", + "eligibility": "ready", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "recovery-required", + "sequence": 2, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "failed", + "eligibility": "approval-required", + "stale_boundary": null, + "blocked_by": [] + } + ] + }, + { + "at": "final", + "sequence": 3, + "steps": [ + { + "step_id": "step:a", + "recorded_status": "abandoned", + "eligibility": "abandoned", + "stale_boundary": null, + "blocked_by": [] + } + ] + } + ] + } + } + ] +} diff --git a/tests/hermetic_provider_kit.rs b/tests/hermetic_provider_kit.rs index bba0513..e873375 100644 --- a/tests/hermetic_provider_kit.rs +++ b/tests/hermetic_provider_kit.rs @@ -9,6 +9,9 @@ mod durable_execution; #[path = "graph_recovery/mod.rs"] mod graph_recovery; +#[cfg(target_os = "linux")] +mod lifecycle_matrix; + use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; @@ -186,6 +189,7 @@ const MULTI_PROVIDER_COMPOSITION: CompositionFixtureSpec = CompositionFixtureSpe struct TestRoot { path: PathBuf, + owned: bool, } impl TestRoot { @@ -196,7 +200,7 @@ impl TestRoot { std::process::id() )); fs::create_dir(&path).expect("hermetic provider test root must be new"); - Self { path } + Self { path, owned: true } } fn path(&self) -> &Path { @@ -206,7 +210,9 @@ impl TestRoot { impl Drop for TestRoot { fn drop(&mut self) { - let _cleanup_result = fs::remove_dir_all(&self.path); + if self.owned && !std::thread::panicking() { + let _cleanup_result = fs::remove_dir_all(&self.path); + } } } diff --git a/tests/lifecycle_matrix/fixture.rs b/tests/lifecycle_matrix/fixture.rs new file mode 100644 index 0000000..b8a4c1c --- /dev/null +++ b/tests/lifecycle_matrix/fixture.rs @@ -0,0 +1,299 @@ +use super::*; +use crate::{ + BINDINGS_LOCATOR, CAPABILITIES, INPUT_BYTES, INPUT_LOCATOR, KitFixture, PreparedLifecycleRun, + TestRoot, WORKSPACE_LOCATOR, +}; +use flow::{ + ArtifactBindingSet, ExtensionCatalog, ExtensionLock, ExtensionManifest, ExtensionObservation, + HostArtifactObservationSet, PlannedStep, ProcessStepContext, RUN_PLAN_V1, RunPlan, + RunStepContext, +}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +pub(super) const IDS: [&str; 3] = ["step:a", "step:b", "step:c"]; + +pub(super) struct Node { + pub kit: KitFixture, + pub prepared: PreparedLifecycleRun, + artifacts: PathBuf, +} + +impl Node { + fn prepare(kit: KitFixture, mode: &str, index: usize) -> Self { + let prepared = kit.prepare_lifecycle_named( + CAPABILITIES[0], + mode, + mode == "await-interruption", + &format!("lifecycle-{index}"), + ); + let artifacts = kit.root.path().join(WORKSPACE_LOCATOR); + Self { + kit, + prepared, + artifacts, + } + } + + pub fn context(&self) -> ProcessStepContext<'_> { + ProcessStepContext { + execution_root: self.kit.root.path(), + artifact_root: &self.artifacts, + resolved: self.prepared.resolved(), + invocation: &self.prepared.invocation, + subjects: &self.prepared.subject_lock, + authority: &self.prepared.authority, + bindings: &self.kit.bindings, + } + } + + pub fn preserve_and_replace(&self, relative: &str, bytes: &[u8]) { + let path = self.kit.root.path().join(relative); + let backup = self.kit.root.path().join("preserved-original"); + assert!(!backup.exists()); + fs::copy(&path, &backup).unwrap(); + fs::write(path, bytes).unwrap(); + } +} + +pub(super) struct Fixture { + pub nodes: Vec, + pub plan: RunPlan, + mode: String, +} + +// Host-local handoff only: paths and fixture configuration never enter receipts. +// Reopening freshly resolves, observes, and authorizes; no opaque tokens are saved. +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Descriptor { + mode: String, + nodes: Vec, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct NodeDescriptor { + root: PathBuf, + manifest: ExtensionManifest, + lock: ExtensionLock, + bindings: ArtifactBindingSet, + package_digest: String, + executable_digest: String, + executable_locator: String, + grants_digest: String, + package_observations: HostArtifactObservationSet, +} + +impl Fixture { + pub fn new(mode: &str, graph: bool, bytes: &[u8], name: &str) -> Self { + let nodes = (0..if graph { 3 } else { 1 }) + .map(|index| { + let mut kit = KitFixture::new(CAPABILITIES[0], bytes, name); + kit.bindings.outputs[0].artifact_id = format!("artifact:lifecycle-{index}"); + fs::write( + kit.root + .path() + .join(WORKSPACE_LOCATOR) + .join(BINDINGS_LOCATOR), + serde_json::to_vec(&kit.bindings).unwrap(), + ) + .unwrap(); + Node::prepare(kit, mode, index) + }) + .collect(); + Self::from_nodes(nodes, mode.to_owned()) + } + + fn from_nodes(nodes: Vec, mode: String) -> Self { + let plan = RunPlan { + schema_version: RUN_PLAN_V1.to_owned(), + plan_id: "plan:lifecycle-matrix".to_owned(), + run_id: nodes[0].prepared.invocation.run_id.clone(), + steps: nodes + .iter() + .enumerate() + .map(|(index, node)| { + let dependencies = if index == 1 { + vec![IDS[0].to_owned()] + } else { + Vec::new() + }; + PlannedStep::prepare(IDS[index].to_owned(), dependencies, &node.context()) + .unwrap() + }) + .collect(), + }; + plan.validate().unwrap(); + Self { nodes, plan, mode } + } + + pub fn workspace(&self) -> PathBuf { + self.nodes[0].kit.root.path().join("lifecycle state café") + } + + pub fn contexts(&self) -> Vec> { + self.nodes + .iter() + .enumerate() + .map(|(index, node)| RunStepContext { + step_id: IDS[index], + context: node.context(), + }) + .collect() + } + + pub fn execute(&self, store: &mut RunStore, index: usize) { + store + .execute_in_plan( + &self.plan, + IDS[index], + &self.contexts(), + &NoSecrets, + &NeverCancelled, + &mut Vec::new(), + ) + .unwrap(); + } + + pub fn assess(&self, store: &RunStore, at: &str) -> Assessment { + let report = store.assess_run(&self.plan, &self.contexts()).unwrap(); + report.validate_against(store.state()).unwrap(); + Assessment { + at: at.to_owned(), + sequence: report.sequence, + steps: report.steps, + } + } + + pub fn child(&self, mode: &str) { + let descriptor = Descriptor { + mode: self.mode.clone(), + nodes: self + .nodes + .iter() + .map(|node| NodeDescriptor { + root: node.kit.root.path().to_owned(), + manifest: node.kit.manifest.clone(), + lock: node.kit.lock.clone(), + bindings: node.kit.bindings.clone(), + package_digest: node.kit.package_digest.clone(), + executable_digest: node.kit.executable_digest.clone(), + executable_locator: node.kit.executable_locator.clone(), + grants_digest: node.kit.grants_digest.clone(), + package_observations: node.kit.package_observations.clone(), + }) + .collect(), + }; + let path = self.nodes[0].kit.root.path().join("host-handoff.json"); + fs::write(&path, serde_json::to_vec(&descriptor).unwrap()).unwrap(); + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "lifecycle_matrix::lifecycle_child", + "--ignored", + "--nocapture", + ]) + .env("FLOW_LIFECYCLE_HANDOFF", path) + .env("FLOW_LIFECYCLE_CHILD_MODE", mode) + .output() + .unwrap(); + assert!(output.stdout.len() + output.stderr.len() < 16_384); + assert_eq!( + output.status.code(), + Some(if mode.starts_with("exit-") { 73 } else { 0 }), + "child {mode}: {} {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + pub fn from_handoff(path: &Path) -> Self { + let descriptor: Descriptor = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + let nodes = descriptor + .nodes + .into_iter() + .enumerate() + .map(|(index, saved)| { + let observation = ExtensionObservation::new( + saved.manifest.extension_id.clone(), + saved.manifest.version.clone(), + saved.manifest.publisher.id.clone(), + saved.manifest.integrity.clone(), + true, + ); + let catalog = ExtensionCatalog::inspect( + [saved.manifest.clone()], + saved.lock.clone(), + [observation], + ) + .unwrap(); + let kit = KitFixture { + root: TestRoot { + path: saved.root, + owned: false, + }, + catalog, + manifest: saved.manifest, + lock: saved.lock, + bindings: saved.bindings, + package_digest: saved.package_digest, + executable_digest: saved.executable_digest, + executable_locator: saved.executable_locator, + grants_digest: saved.grants_digest, + package_observations: saved.package_observations, + }; + Node::prepare(kit, &descriptor.mode, index) + }) + .collect(); + Self::from_nodes(nodes, descriptor.mode) + } + + pub fn identities(&self) -> Vec { + self.nodes + .iter() + .map(|node| { + json!({ + "package_digest": node.kit.package_digest, + "executable_digest": node.kit.executable_digest, + "manifest_digest": digest_json(&node.kit.manifest), + "input_digest": digest_bytes(INPUT_BYTES), + "bindings_digest": digest_json(&node.kit.bindings), + }) + }) + .collect() + } + + pub fn verify_preservation(&self, recipe: &str) { + for (index, node) in self.nodes.iter().enumerate() { + let root = node.kit.root.path(); + let source = if recipe == "changed-input" && index == 0 { + root.join("preserved-original") + } else { + root.join(WORKSPACE_LOCATOR).join(INPUT_LOCATOR) + }; + assert_eq!(fs::read(source).unwrap(), INPUT_BYTES); + if recipe != "changed-implementation" || index != 0 { + assert_eq!( + digest_bytes( + &fs::read( + root.join(crate::PACKAGE_LOCATOR) + .join(&node.kit.executable_locator) + ) + .unwrap() + ), + node.kit.executable_digest + ); + assert_eq!( + fs::read(root.join(crate::PACKAGE_LOCATOR).join("LICENSE")).unwrap(), + crate::LICENSE_BYTES + ); + } else { + assert_eq!( + digest_bytes(&fs::read(root.join("preserved-original")).unwrap()), + node.kit.executable_digest + ); + } + } + } +} diff --git a/tests/lifecycle_matrix/mod.rs b/tests/lifecycle_matrix/mod.rs new file mode 100644 index 0000000..c0c2ed4 --- /dev/null +++ b/tests/lifecycle_matrix/mod.rs @@ -0,0 +1,273 @@ +//! Versioned, test-owned projections of public durable state, never a runtime model. +use crate::{NoSecrets, digest_bytes, digest_json, provider_binary_snapshot}; +use flow::{ + NeverCancelled, RunFailure, RunRecoveryAction, RunState, RunStepAssessment, RunStepStatus, + RunStore, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeSet; +use std::fs; +use std::path::Path; +use std::time::Instant; + +mod fixture; +mod recipes; +use fixture::Fixture; + +const CATALOG: &str = include_str!("../fixtures/lifecycle-scenarios.v1.json"); + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Catalog { + schema_version: String, + fixture_version: String, + tier: String, + budget: Budget, + known_gaps: Vec, + scenarios: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Budget { + test_threads: usize, + repetitions: usize, + timeout_ms: u64, + max_receipt_bytes: usize, + max_snapshots: usize, + max_history_bytes: u64, + max_artifacts: usize, + max_artifact_bytes: u64, + max_fixture_bytes: u64, + max_process_address_space_bytes: u64, + max_process_file_bytes: u64, + max_driver_output_bytes: u64, + driver_timeout_seconds: u64, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Case { + scenario_id: String, + recipe: String, + mode: String, + graph: bool, + recovery: String, + expected: Outcome, +} + +#[derive(Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Outcome { + code: String, + history: Vec, + assessments: Vec, +} + +#[derive(Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Frame { + sequence: u64, + steps: Vec, + recovery: Vec, +} + +#[derive(Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Step { + status: RunStepStatus, + attempt: u64, + failure: Option, + checkpoint: bool, +} + +#[derive(Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Assessment { + at: String, + sequence: u64, + steps: Vec, +} + +#[derive(Debug, Eq, PartialEq, Serialize)] +struct Receipt { + schema_version: &'static str, + scenario_id: String, + recipe_digest: String, + fixture_identity: Vec, + outcome: Outcome, + recovery: String, + plan_digest: String, + history_digest: String, + artifact_digests: Vec>, +} + +fn history(path: &Path) -> Vec { + snapshot_paths(path) + .iter() + .map(|path| { + let value: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + let state: RunState = serde_json::from_value(value["state"].clone()).unwrap(); + Frame { + sequence: state.sequence, + steps: state + .steps + .iter() + .map(|step| Step { + status: step.status, + attempt: step.attempt, + failure: step.failure, + checkpoint: step.checkpoint.is_some(), + }) + .collect(), + recovery: state + .recovery_decisions + .iter() + .map(|decision| decision.action) + .collect(), + } + }) + .collect() +} + +fn snapshot_paths(path: &Path) -> Vec { + let mut paths: Vec<_> = fs::read_dir(path) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == "json") + }) + .collect(); + paths.sort(); + paths +} + +fn footprint(path: &Path) -> (usize, u64) { + let mut count = 0; + let mut bytes = 0; + for entry in fs::read_dir(path).unwrap() { + let entry = entry.unwrap(); + let metadata = fs::symlink_metadata(entry.path()).unwrap(); + assert!(!metadata.is_symlink()); + if metadata.is_dir() { + let (nested_count, nested_bytes) = footprint(&entry.path()); + count += nested_count; + bytes += nested_bytes; + } else { + count += 1; + bytes += metadata.len(); + } + } + (count, bytes) +} + +fn check_budget(fixture: &Fixture, budget: &Budget) { + let snapshots = snapshot_paths(&fixture.workspace()); + assert!(snapshots.len() <= budget.max_snapshots); + assert!(footprint(&fixture.workspace()).1 <= budget.max_history_bytes); + let mut fixture_bytes = 0; + for node in &fixture.nodes { + fixture_bytes += footprint(node.kit.root.path()).1; + let (artifacts, bytes) = footprint(&node.kit.root.path().join(crate::WORKSPACE_LOCATOR)); + assert!(artifacts <= budget.max_artifacts && bytes <= budget.max_artifact_bytes); + } + assert!(fixture_bytes <= budget.max_fixture_bytes); +} + +#[test] +fn executable_lifecycle_matrix() { + let catalog: Catalog = serde_json::from_str(CATALOG).unwrap(); + assert_eq!(catalog.schema_version, "flow.lifecycle-scenario-catalog/v1"); + assert_eq!(catalog.fixture_version, "1.0.0"); + assert_eq!(catalog.tier, "pull-request-linux"); + assert!(!catalog.known_gaps.is_empty()); + assert_eq!(catalog.budget.repetitions, 2); + assert_eq!(catalog.budget.test_threads, 1); + // These are enforced by the Python runner; direct cargo is a diagnostic run. + assert_eq!( + catalog.budget.max_process_address_space_bytes, + 4_294_967_296 + ); + assert_eq!(catalog.budget.max_process_file_bytes, 16_777_216); + assert_eq!(catalog.budget.max_driver_output_bytes, 4_194_304); + assert_eq!(catalog.budget.driver_timeout_seconds, 600); + let (bytes, name) = provider_binary_snapshot(); + let mut ids = BTreeSet::new(); + let mut recipes = BTreeSet::new(); + for case in &catalog.scenarios { + assert!(ids.insert(&case.scenario_id) && recipes.insert(&case.recipe)); + assert!(case.scenario_id.starts_with("scenario:lifecycle-")); + let mut previous = None; + for _ in 0..catalog.budget.repetitions { + let started = Instant::now(); + let mut fixture = Fixture::new(&case.mode, case.graph, &bytes, &name); + let fixture_identity = fixture.identities(); + let plan_digest = digest_json(&fixture.plan); + let outcome = recipes::run(&mut fixture, &case.recipe); + assert_eq!(outcome, case.expected, "{}", case.scenario_id); + fixture.verify_preservation(&case.recipe); + check_budget(&fixture, &catalog.budget); + let receipt = Receipt { + schema_version: "flow.lifecycle-scenario-receipt/v1", + scenario_id: case.scenario_id.clone(), + recipe_digest: digest_json( + &json!({"fixture_version": catalog.fixture_version, "case": case}), + ), + fixture_identity, + outcome, + recovery: case.recovery.clone(), + plan_digest, + history_digest: digest_json( + &snapshot_paths(&fixture.workspace()) + .iter() + .map(|path| digest_bytes(&fs::read(path).unwrap())) + .collect::>(), + ), + artifact_digests: fixture + .nodes + .iter() + .map(|node| { + node.kit + .output_path() + .is_file() + .then(|| digest_bytes(&fs::read(node.kit.output_path()).unwrap())) + }) + .collect(), + }; + let encoded = serde_json::to_string(&receipt).unwrap(); + assert!(!encoded.contains("FLOW_LIFECYCLE_FAILURE_PRIVATE_CANARY")); + assert!(encoded.len() <= catalog.budget.max_receipt_bytes); + for node in &fixture.nodes { + for private in [ + node.kit.root.path().to_str().unwrap(), + node.prepared.invocation.configuration.values["seed"] + .as_str() + .unwrap(), + std::str::from_utf8(crate::INPUT_BYTES).unwrap(), + ] { + assert!( + !encoded.contains(private), + "portable receipt leaks raw fixture values" + ); + } + } + assert!(started.elapsed().as_millis() <= u128::from(catalog.budget.timeout_ms)); + if let Some(prior) = &previous { + assert_eq!(&receipt, prior, "{}", case.scenario_id); + } + previous = Some(receipt); + } + println!( + "FLOW_LIFECYCLE_RECEIPT={}", + serde_json::to_string(&previous.unwrap()).unwrap() + ); + } +} + +#[test] +#[ignore = "subprocess entry point; invoked by the executable lifecycle matrix"] +fn lifecycle_child() { + recipes::child(); +} diff --git a/tests/lifecycle_matrix/recipes.rs b/tests/lifecycle_matrix/recipes.rs new file mode 100644 index 0000000..cccfaa0 --- /dev/null +++ b/tests/lifecycle_matrix/recipes.rs @@ -0,0 +1,579 @@ +use super::fixture::IDS; +use super::*; +use flow::{ + DurableExecutionError, EventSinkError, ExecutionError, ExtensionEvent, ProcessRunnerError, + ProcessStream, RecoveryApproval, ResumeEligibility, StateBoundary, StateError, +}; +use std::cell::Cell; + +fn approval(action: RunRecoveryAction, acknowledge: bool) -> RecoveryApproval { + RecoveryApproval { + decision_id: "decision:lifecycle-recovery".to_owned(), + action, + acknowledge_uncertain_effects: acknowledge, + } +} + +fn execute_error(fixture: &Fixture, store: &mut RunStore, recipe: &str) -> &'static str { + let checks = Cell::new(0); + let control = fixture.nodes[0].kit.lifecycle_control_path(); + let cancel = || match recipe { + "cancel-before-launch" | "retry-cancelled" | "retry-without-ack" => true, + "cancel-after-intent" => { + checks.set(checks.get() + 1); + checks.get() >= 2 + } + "cancel-during-provider" => control.is_file(), + _ => false, + }; + let error = store + .execute_in_plan( + &fixture.plan, + IDS[0], + &fixture.contexts(), + &NoSecrets, + &cancel, + &mut Vec::new(), + ) + .unwrap_err(); + let code = match error { + DurableExecutionError::CancelledBeforeLaunch => "cancelled-before-launch", + DurableExecutionError::Process(ProcessRunnerError::Cancelled { .. }) => "cancelled", + DurableExecutionError::Process(ProcessRunnerError::TimedOut { + timeout_ms: 5_000, .. + }) => "timed-out", + DurableExecutionError::Process(ProcessRunnerError::Validation { + source: ExecutionError::ProcessExit { code: Some(7) }, + }) => "process-exit-7", + DurableExecutionError::Process(ProcessRunnerError::Validation { + source: ExecutionError::ProcessExit { code: None }, + }) => "process-signal", + DurableExecutionError::Process(ProcessRunnerError::Validation { + source: + ExecutionError::ProcessOutputLimit { + stream, + limit: 65_536, + observed: 65_537, + }, + }) => match stream { + ProcessStream::Stdout => "stdout-limit", + ProcessStream::Stderr => "stderr-limit", + }, + DurableExecutionError::Acceptance(flow::ArtifactAcceptanceError::Mismatch { + ref message, + }) if message == "artifact acceptance requires a complete produced or reused result" => { + "partial-output-rejected" + } + other => panic!("unexpected typed lifecycle failure: {other:?}"), + }; + if matches!(recipe, "cancel-during-provider" | "timeout") { + assert!(control.is_file()); + crate::assert_recorded_process_reaped(&control); + } + code +} + +pub(super) fn run(fixture: &mut Fixture, recipe: &str) -> Outcome { + let mut store = RunStore::create(&fixture.workspace(), fixture.plan.clone()).unwrap(); + let initial = fixture.assess(&store, "initial"); + match recipe { + "retryable-provider-failure" | "terminal-provider-failure" => { + classified_failure(fixture, store, initial, recipe) + } + "completed-restart" + | "partial-restart" + | "host-exit-after-intent" + | "host-exit-after-provider" + | "retry-interrupted" => restart(fixture, store, initial, recipe), + value + if value.starts_with("changed-") + || value.starts_with("corrupt-") + || matches!( + value, + "incomplete-write" | "future-schema" | "malformed-state" + ) => + { + drift(fixture, store, initial, recipe) + } + "cancel-between-steps" => { + fixture.execute(&mut store, 0); + let completed = fixture.assess(&store, "first-complete"); + store.cancel_pending(IDS[1]).unwrap(); + finish( + fixture, + store, + vec![initial, completed], + "cancelled-between-steps", + ) + } + "retry-completed" => { + fixture.execute(&mut store, 0); + let before = history(&fixture.workspace()); + assert!(matches!( + store.decide_recovery( + IDS[0], + &fixture.nodes[0].context(), + approval(RunRecoveryAction::Retry, true) + ), + Err(StateError::Transition) + )); + assert!(matches!( + store.execute_in_plan( + &fixture.plan, + IDS[0], + &fixture.contexts(), + &NoSecrets, + &NeverCancelled, + &mut Vec::new() + ), + Err(DurableExecutionError::State(StateError::Ineligible { + eligibility: ResumeEligibility::Reusable + })) + )); + assert_eq!(history(&fixture.workspace()), before); + finish(fixture, store, vec![initial], "completed-retry-refused") + } + "cancel-before-launch" + | "cancel-after-intent" + | "cancel-during-provider" + | "timeout" + | "nonzero-exit" + | "signal-termination" + | "stdout-limit" + | "stderr-limit" + | "partial-output" + | "retry-cancelled" + | "retry-without-ack" + | "abandon-terminal" => { + let code = execute_error(fixture, &mut store, recipe); + recover_failure(fixture, store, vec![initial], recipe, code) + } + other => panic!("unknown lifecycle recipe: {other}"), + } +} + +fn finish( + fixture: &Fixture, + store: RunStore, + mut assessments: Vec, + code: &str, +) -> Outcome { + let before = history(&fixture.workspace()); + drop(store); + let reopened = RunStore::open(&fixture.workspace()).unwrap(); + assessments.push(fixture.assess(&reopened, "final")); + assert_eq!( + history(&fixture.workspace()), + before, + "assessment must not rewrite history" + ); + Outcome { + code: code.to_owned(), + history: before, + assessments, + } +} + +fn recover_failure( + fixture: &Fixture, + mut store: RunStore, + mut assessments: Vec, + recipe: &str, + code: &str, +) -> Outcome { + match recipe { + "retry-cancelled" => { + assessments.push(fixture.assess(&store, "recovery-required")); + store + .decide_recovery( + IDS[0], + &fixture.nodes[0].context(), + approval(RunRecoveryAction::Retry, true), + ) + .unwrap(); + assessments.push(fixture.assess(&store, "retry-approved")); + assert!(!fixture.nodes[0].kit.output_path().exists()); + fixture.execute(&mut store, 0); + finish(fixture, store, assessments, "explicit-retry-succeeded") + } + "abandon-terminal" => { + assessments.push(fixture.assess(&store, "recovery-required")); + let candidate = fs::read(fixture.nodes[0].kit.output_path()).unwrap(); + store + .decide_recovery( + IDS[0], + &fixture.nodes[0].context(), + approval(RunRecoveryAction::Abandon, true), + ) + .unwrap(); + assert_eq!( + fs::read(fixture.nodes[0].kit.output_path()).unwrap(), + candidate + ); + finish(fixture, store, assessments, "explicit-abandon") + } + "retry-without-ack" => { + let before = history(&fixture.workspace()); + assert!(matches!( + store.decide_recovery( + IDS[0], + &fixture.nodes[0].context(), + approval(RunRecoveryAction::Retry, false) + ), + Err(StateError::Invalid { + rule: "explicit recovery acknowledgement" + }) + )); + assert_eq!(history(&fixture.workspace()), before); + finish(fixture, store, assessments, "unacknowledged-retry-refused") + } + _ => finish(fixture, store, assessments, code), + } +} + +fn restart(fixture: &Fixture, mut store: RunStore, initial: Assessment, recipe: &str) -> Outcome { + let mut assessments = vec![initial]; + if matches!(recipe, "completed-restart" | "partial-restart") { + fixture.execute(&mut store, 0); + assessments.push(fixture.assess(&store, "before-restart")); + } + drop(store); + let mode = match recipe { + "completed-restart" => "inspect", + "partial-restart" => "resume", + "host-exit-after-intent" => "exit-after-intent", + "host-exit-after-provider" | "retry-interrupted" => "exit-after-provider", + _ => unreachable!(), + }; + let before = history(&fixture.workspace()); + fixture.child(mode); + let mut store = RunStore::open(&fixture.workspace()).unwrap(); + assessments.push(fixture.assess(&store, "reopened")); + if recipe == "completed-restart" { + assert_eq!(history(&fixture.workspace()), before); + } + if recipe == "host-exit-after-intent" { + assert!(!fixture.nodes[0].kit.output_path().exists()); + } + if matches!(recipe, "host-exit-after-provider" | "retry-interrupted") { + assert!(fixture.nodes[0].kit.output_path().is_file()); + assert!(store.state().steps[0].checkpoint.is_none()); + } + if recipe == "retry-interrupted" { + let output = fixture.nodes[0].kit.output_path(); + let uncertain = fs::read(&output).unwrap(); + let retained = fixture.nodes[0] + .kit + .root + .path() + .join("retained-interrupted-output.json"); + fs::rename(output, &retained).unwrap(); + store + .decide_recovery( + IDS[0], + &fixture.nodes[0].context(), + approval(RunRecoveryAction::Retry, true), + ) + .unwrap(); + assessments.push(fixture.assess(&store, "retry-approved")); + fixture.execute(&mut store, 0); + assert_eq!(fs::read(retained).unwrap(), uncertain); + } + finish( + fixture, + store, + assessments, + match recipe { + "completed-restart" => "completed-reused", + "partial-restart" => "partial-run-resumed", + "retry-interrupted" => "interrupted-retry-succeeded", + _ => "uncertain-intent-retained", + }, + ) +} + +fn replace_snapshot(fixture: &Fixture, recipe: &str) { + let path = snapshot_paths(&fixture.workspace()).pop().unwrap(); + let original = fs::read(&path).unwrap(); + fs::write( + fixture.nodes[0] + .kit + .root + .path() + .join("preserved-snapshot.json"), + &original, + ) + .unwrap(); + if recipe == "malformed-state" { + fs::write(path, b"{broken").unwrap(); + return; + } + let mut value: Value = serde_json::from_slice(&original).unwrap(); + let checkpoint = &mut value["state"]["steps"][0]["checkpoint"]; + match recipe { + "changed-validator" => { + checkpoint["validation"]["implementation_digest"] = "0".repeat(64).into(); + } + "changed-validation-profile" => { + checkpoint["validation"]["profile"] = "flow.other-profile/v1".into(); + } + "corrupt-checkpoint" => checkpoint["context_digest"] = "0".repeat(64).into(), + "corrupt-checksum" => value["state_digest"] = "0".repeat(64).into(), + "future-schema" => value["schema_version"] = "flow.run-snapshot/v999".into(), + _ => unreachable!(), + } + if recipe != "corrupt-checksum" { + value["state_digest"] = digest_json(&value["state"]).into(); + } + fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap(); +} + +fn inject_drift(fixture: &mut Fixture, recipe: &str) { + match recipe { + "changed-input" => fixture.nodes[0] + .preserve_and_replace("workspace/inputs/source-text.txt", b"changed input"), + "changed-artifact" => fixture.nodes[0].preserve_and_replace( + "workspace/outputs/inspection-report.json", + b"changed output", + ), + "changed-implementation" => { + let path = format!( + "{}/{}", + crate::PACKAGE_LOCATOR, + fixture.nodes[0].kit.executable_locator + ); + fixture.nodes[0].preserve_and_replace(&path, b"changed executable"); + } + "changed-configuration" => { + fixture.nodes[0] + .prepared + .invocation + .configuration + .values + .insert("seed".to_owned(), json!("FLOW_LIFECYCLE_PRIVATE_CANARY")); + } + "changed-provider" => { + "0.2.0".clone_into(&mut fixture.nodes[0].prepared.invocation.extension.version); + } + "changed-plan" => "plan:changed".clone_into(&mut fixture.plan.plan_id), + "incomplete-write" => fs::write( + fixture.workspace().join("snapshot.pending"), + b"partial commit", + ) + .unwrap(), + _ => replace_snapshot(fixture, recipe), + } +} + +fn drift(fixture: &mut Fixture, mut store: RunStore, initial: Assessment, recipe: &str) -> Outcome { + // Complete the independent branch first so a validator mutation changes only + // the final Running -> Succeeded transition, not an older accepted checkpoint. + if fixture.nodes.len() == 3 { + fixture.execute(&mut store, 2); + } + fixture.execute(&mut store, 0); + let before = history(&fixture.workspace()); + let assessments = vec![initial, fixture.assess(&store, "before-change")]; + drop(store); + inject_drift(fixture, recipe); + let refused = match recipe { + "changed-validation-profile" => Some("invalid-validation-profile"), + "corrupt-checkpoint" => Some("invalid-checkpoint-context"), + "corrupt-checksum" => Some("corrupt-snapshot"), + "future-schema" => Some("unsupported-schema"), + "malformed-state" => Some("malformed-state"), + "incomplete-write" => Some("incomplete-write"), + _ => None, + }; + if let Some(code) = refused { + let error = RunStore::open(&fixture.workspace()).unwrap_err(); + assert!(matches!( + (recipe, error), + ( + "changed-validation-profile", + StateError::Invalid { + rule: "validation profile" + } + ) | ( + "corrupt-checkpoint", + StateError::Invalid { + rule: "checkpoint context" + } + ) | ("corrupt-checksum", StateError::Corrupt) + | ("future-schema", StateError::UnsupportedSchema) + | ("malformed-state", StateError::Malformed) + | ("incomplete-write", StateError::IncompleteWrite) + )); + // A corrupt replacement cannot produce a legitimate later state frame. + return Outcome { + code: code.to_owned(), + history: before, + assessments, + }; + } + let mut store = RunStore::open(&fixture.workspace()).unwrap(); + if recipe == "changed-plan" { + assert!(matches!( + store.assess_run(&fixture.plan, &fixture.contexts()), + Err(StateError::Stale { + boundary: StateBoundary::Plan + }) + )); + assert_eq!(history(&fixture.workspace()), before); + return Outcome { + code: "changed-plan-refused".to_owned(), + history: before, + assessments, + }; + } + let after = history(&fixture.workspace()); + assert!(matches!( + store.execute_in_plan( + &fixture.plan, + IDS[1], + &fixture.contexts(), + &NoSecrets, + &NeverCancelled, + &mut Vec::new() + ), + Err(DurableExecutionError::State(StateError::Ineligible { + eligibility: ResumeEligibility::Invalidated + })) + )); + assert!(!fixture.nodes[1].kit.output_path().exists()); + assert_eq!(history(&fixture.workspace()), after); + finish(fixture, store, assessments, "stale-branch-invalidated") +} + +pub(super) fn child() { + let path = std::env::var_os("FLOW_LIFECYCLE_HANDOFF").expect("matrix handoff"); + let fixture = Fixture::from_handoff(Path::new(&path)); + let mode = std::env::var("FLOW_LIFECYCLE_CHILD_MODE").unwrap(); + let mut store = RunStore::open(&fixture.workspace()).unwrap(); + match mode.as_str() { + "inspect" => assert_eq!( + fixture.assess(&store, "child").steps[0].eligibility, + ResumeEligibility::Reusable + ), + "resume" => { + assert_eq!( + fixture.assess(&store, "child").steps[0].eligibility, + ResumeEligibility::Reusable + ); + fixture.execute(&mut store, 1); + } + "exit-after-intent" => { + let checks = Cell::new(0); + let cancel = || { + checks.set(checks.get() + 1); + if checks.get() == 2 { + std::process::exit(73); + } + false + }; + store + .execute_in_plan( + &fixture.plan, + IDS[0], + &fixture.contexts(), + &NoSecrets, + &cancel, + &mut Vec::new(), + ) + .unwrap(); + panic!("host must exit after intent"); + } + "exit-after-provider" => { + // LocalProcessRunner forwards validated events after reaping its child. + let mut sink = + |_: &ExtensionEvent| -> Result<(), EventSinkError> { std::process::exit(73) }; + store + .execute_in_plan( + &fixture.plan, + IDS[0], + &fixture.contexts(), + &NoSecrets, + &NeverCancelled, + &mut sink, + ) + .unwrap(); + panic!("host must exit before acceptance"); + } + _ => panic!("unknown child mode"), + } +} + +fn classified_failure( + fixture: &Fixture, + mut store: RunStore, + initial: Assessment, + recipe: &str, +) -> Outcome { + let mut events = Vec::new(); + let error = store + .execute_in_plan( + &fixture.plan, + IDS[0], + &fixture.contexts(), + &NoSecrets, + &NeverCancelled, + &mut events, + ) + .unwrap_err(); + assert!(matches!(error, DurableExecutionError::Acceptance( + flow::ArtifactAcceptanceError::Mismatch { ref message } + ) if message == "artifact acceptance requires a complete produced or reused result")); + let node = &fixture.nodes[0]; + let transcript_path = node + .kit + .root + .path() + .join("workspace/outputs/failure-transcript.jsonl"); + let transcript = fs::read(&transcript_path).unwrap(); + let execution = flow::Orchestrator::validate_process_transcript( + node.prepared.resolved(), + &node.prepared.invocation, + &node.prepared.subject_lock, + &node.prepared.subjects, + &node.prepared.authority, + flow::ProcessTranscript::new( + flow::ProcessCompletion::Exited { code: Some(0) }, + &transcript, + &[], + ), + &mut Vec::new(), + ) + .unwrap(); + assert_eq!(execution.events(), events); + assert_eq!(execution.result().outcome, flow::Outcome::Failed); + let (action, code) = match ( + &execution.result().failure.classification, + execution.result().failure.retryable, + ) { + (flow::FailureClassification::Provider, true) => { + assert_eq!(recipe, "retryable-provider-failure"); + ( + RunRecoveryAction::Retry, + "retryable-provider-failure-approved", + ) + } + (flow::FailureClassification::Validation, false) => { + assert_eq!(recipe, "terminal-provider-failure"); + ( + RunRecoveryAction::Abandon, + "terminal-validation-failure-abandoned", + ) + } + other => panic!("unexpected provider failure classification: {other:?}"), + }; + let assessments = vec![initial, fixture.assess(&store, "recovery-required")]; + let candidate = fs::read(node.kit.output_path()).unwrap(); + // The hint alone never retries. The test operator makes an explicit decision; + // Retry leaves work pending and retains failed artifacts for operator handling. + store + .decide_recovery(IDS[0], &node.context(), approval(action, true)) + .unwrap(); + assert_eq!(fs::read(node.kit.output_path()).unwrap(), candidate); + assert_eq!(fs::read(transcript_path).unwrap(), transcript); + finish(fixture, store, assessments, code) +} diff --git a/tools/bounded_test_process.py b/tools/bounded_test_process.py new file mode 100644 index 0000000..ab03af1 --- /dev/null +++ b/tools/bounded_test_process.py @@ -0,0 +1,67 @@ +"""Bound capture while supervising a test command, without shell strings or retries.""" +import os +from pathlib import Path +import selectors +import signal +import subprocess +import sys +import time + + +class BudgetExceeded(ValueError): + def __init__(self, message, output): + super().__init__(message) + self.output = output + + +def run(command, *, cwd, timeout=600, max_output=4_194_304): + """Return merged output; on timeout/flood kill this test process group and reap.""" + output = bytearray() + deadline = time.monotonic() + timeout + with subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, start_new_session=True) as process: + try: + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ) + while selector.get_map(): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise BudgetExceeded("test command exceeded runtime budget", bytes(output)) + for key, _ in selector.select(remaining): + data = os.read(key.fileobj.fileno(), 65_536) + if not data: + selector.unregister(key.fileobj) + break + available = max_output - len(output) + output.extend(data[:available]) + if len(data) > available: + raise BudgetExceeded("test command exceeded output budget", bytes(output)) + try: + code = process.wait(timeout=max(0, deadline - time.monotonic())) + except subprocess.TimeoutExpired as error: + raise BudgetExceeded("test command exceeded runtime budget", bytes(output)) from error + if code: + _kill_group(process) + return subprocess.CompletedProcess(command, code, bytes(output).decode("utf-8", errors="replace")) + except BaseException: + _kill_group(process) + process.wait() + raise + + +def _kill_group(process): + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def cargo_command(cargo, root): + """Limit test executables and their children, not the compiler/linker.""" + if sys.platform != "linux": + raise ValueError("bounded scenario qualification requires Linux") + version = subprocess.check_output([cargo, "--version", "--verbose"], text=True, timeout=10) + host = next(line.split(": ", 1)[1] for line in version.splitlines() if line.startswith("host: ")) + import json + runner = [sys.executable, str(Path(root) / "tools/bounded_test_runner.py")] + return [cargo, "--config", f"target.{host}.runner={json.dumps(runner)}", "test", "--locked", "--offline"] diff --git a/tools/bounded_test_runner.py b/tools/bounded_test_runner.py new file mode 100644 index 0000000..2ac14a0 --- /dev/null +++ b/tools/bounded_test_runner.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Cargo target runner for Linux conformance tests; inherited per-process limits.""" +import os +import resource +import sys + +ADDRESS_SPACE_BYTES = 4_294_967_296 +FILE_BYTES = 16_777_216 + + +def limit(kind, ceiling): + _, hard = resource.getrlimit(kind) + value = ceiling if hard == resource.RLIM_INFINITY else min(ceiling, hard) + resource.setrlimit(kind, (value, value)) + + +def main(): + limit(resource.RLIMIT_AS, ADDRESS_SPACE_BYTES) + limit(resource.RLIMIT_FSIZE, FILE_BYTES) + limit(resource.RLIMIT_CORE, 0) + os.execv(sys.argv[1], sys.argv[1:]) + + +if __name__ == "__main__": + main() diff --git a/tools/lifecycle_reports.py b/tools/lifecycle_reports.py new file mode 100644 index 0000000..622cb16 --- /dev/null +++ b/tools/lifecycle_reports.py @@ -0,0 +1,94 @@ +"""Strict allowlist and completeness checks for test-owned lifecycle receipts.""" +import hashlib +import json +import re + +MARKER = "FLOW_LIFECYCLE_RECEIPT=" +RECEIPT_FIELDS = {"schema_version", "scenario_id", "recipe_digest", "fixture_identity", + "outcome", "recovery", "plan_digest", "history_digest", "artifact_digests"} +IDENTITY_FIELDS = {"package_digest", "executable_digest", "manifest_digest", "input_digest", "bindings_digest"} +RECOVERY = {"operator-decision-required", "continue-ready-work", "reuse-accepted-work", + "retry-with-acknowledgement", "refuse-retry", "abandon-invalid-evidence", + "rebuild-current-context", "preserve-and-repair-store"} +BUDGET = {"test_threads": 1, "repetitions": 2, "timeout_ms": 60_000, "max_receipt_bytes": 32_768, + "max_snapshots": 16, "max_history_bytes": 2_097_152, "max_artifacts": 16, + "max_artifact_bytes": 1_048_576, "max_fixture_bytes": 134_217_728, + "max_process_address_space_bytes": 4_294_967_296, "max_process_file_bytes": 16_777_216, + "max_driver_output_bytes": 4_194_304, "driver_timeout_seconds": 600} + + +def canonical_digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()).hexdigest() + + +def require_digest(value): + if not isinstance(value, str) or not re.fullmatch("[0-9a-f]{64}", value): + raise ValueError("invalid identity digest") + + +def verify_receipts(catalog, output, source_identity): + if set(catalog) != {"schema_version", "fixture_version", "tier", "budget", "known_gaps", "scenarios"}: + raise ValueError("unknown or missing catalog fields") + if (catalog["schema_version"] != "flow.lifecycle-scenario-catalog/v1" + or catalog["fixture_version"] != "1.0.0" or catalog["tier"] != "pull-request-linux"): + raise ValueError("unsupported lifecycle catalog") + if catalog["budget"] != BUDGET or not catalog["known_gaps"]: + raise ValueError("unsupported lifecycle budgets or missing gaps") + scenarios = catalog["scenarios"] + expected = {case["scenario_id"]: case for case in scenarios} + if not expected or len(expected) != len(scenarios) or len({case["recipe"] for case in scenarios}) != len(scenarios): + raise ValueError("empty or duplicate lifecycle recipes") + for case in scenarios: + if set(case) != {"scenario_id", "recipe", "mode", "graph", "recovery", "expected"} or case["recovery"] not in RECOVERY: + raise ValueError("unsupported lifecycle recipe fields or recovery classification") + receipts = {} + for line in output.splitlines(): + if MARKER not in line: + continue + encoded = line.split(MARKER, 1)[1] + if len(encoded.encode()) > BUDGET["max_receipt_bytes"]: + raise ValueError("lifecycle receipt exceeds budget") + receipt = json.loads(encoded) + if set(receipt) != RECEIPT_FIELDS or receipt["schema_version"] != "flow.lifecycle-scenario-receipt/v1": + raise ValueError("unsupported lifecycle receipt fields or version") + identifier = receipt["scenario_id"] + if identifier not in expected or identifier in receipts: + raise ValueError("unknown or duplicate lifecycle receipt") + case = expected[identifier] + if receipt["recipe_digest"] != canonical_digest({"fixture_version": catalog["fixture_version"], "case": case}): + raise ValueError("lifecycle recipe identity drift") + if receipt["outcome"] != case["expected"] or receipt["recovery"] != case["recovery"]: + raise ValueError(f"unexpected lifecycle trace or recovery: {identifier}") + identities = receipt["fixture_identity"] + count = 3 if case["graph"] else 1 + if len(identities) != count or len(receipt["artifact_digests"]) != count: + raise ValueError("incomplete fixture inventory") + for identity in identities: + if set(identity) != IDENTITY_FIELDS: + raise ValueError("unsupported fixture identity fields") + for value in identity.values(): + require_digest(value) + if identity["input_digest"] != source_identity["files"]["contracts/fixtures/scenarios/sources/source-text.txt"]: + raise ValueError("fixture source identity drift") + require_digest(receipt["plan_digest"]) + require_digest(receipt["history_digest"]) + for value in receipt["artifact_digests"]: + if value is not None: + require_digest(value) + receipts[identifier] = receipt + if receipts.keys() != expected.keys(): + raise ValueError("missing lifecycle receipts") + # Every recipe used the same exact compiled provider package. Per-toolchain + # executables may differ, so pin them in the report rather than in the catalog. + for field in ["executable_digest", "package_digest", "manifest_digest"]: + if len({identity[field] for receipt in receipts.values() for identity in receipt["fixture_identity"]}) != 1: + raise ValueError("provider fixture identity drift") + return [receipts[key] for key in sorted(receipts)] + + +def report(catalog, catalog_bytes, receipts, source_identity, toolchain): + return {"schema_version": "flow.lifecycle-scenario-report/v1", "status": "passed", + "catalog_digest": hashlib.sha256(catalog_bytes).hexdigest(), "source_identity": source_identity, + "toolchain": toolchain, "tier": catalog["tier"], "scenario_count": len(receipts), + "executions_per_scenario": BUDGET["repetitions"], "budgets": BUDGET, + "known_gaps": catalog["known_gaps"], "receipts": receipts} diff --git a/tools/run_acceptance_scenarios.py b/tools/run_acceptance_scenarios.py index 712e605..b7fc5e6 100644 --- a/tools/run_acceptance_scenarios.py +++ b/tools/run_acceptance_scenarios.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Execute Flow #30 recipes and retain only checked, normalized portable receipts. +"""Execute Flow #30/#65 recipes and retain checked, normalized portable receipts. This is a test driver, not a Flow runtime. Rust owns assertions and public API execution; this driver checks completeness and writes a bounded coverage report. @@ -8,13 +8,15 @@ import argparse import hashlib import json -import os from pathlib import Path import subprocess import sys +import bounded_test_process +import lifecycle_reports ROOT = Path(__file__).resolve().parents[1] CATALOG = ROOT / "tests/fixtures/acceptance-scenarios.v1.json" +LIFECYCLE_CATALOG = ROOT / "tests/fixtures/lifecycle-scenarios.v1.json" MARKER = "FLOW_ACCEPTANCE_RECEIPT=" FAMILIES = {"resolution", "contract", "artifact", "provider", "privacy"} @@ -26,9 +28,11 @@ def digest(data): def source_identity(): """Pin the tested recipe, provider, contracts, and implementation bytes.""" paths = {"Cargo.toml", "Cargo.lock", "LICENSE", "tests/hermetic_provider_kit.rs", - "tools/run_acceptance_scenarios.py"} + "tools/run_acceptance_scenarios.py", "tools/bounded_test_process.py", + "tools/bounded_test_runner.py", "tools/lifecycle_reports.py", + "tools/test_lifecycle_report.py", ".github/workflows/ci.yml"} paths.add("tests/durable_state.rs") - for directory in ["src", "contracts", "tests/scenario_matrix", "tests/durable_execution", "tests/graph_recovery", "tests/fixtures", "tests/common"]: + for directory in ["src", "contracts", "tests/scenario_matrix", "tests/durable_execution", "tests/graph_recovery", "tests/lifecycle_matrix", "tests/fixtures", "tests/common"]: paths.update(str(path.relative_to(ROOT)) for path in (ROOT / directory).rglob("*") if path.is_file() and "__pycache__" not in path.parts) entries = {path: digest((ROOT / path).read_bytes()) for path in sorted(paths)} @@ -68,43 +72,59 @@ def verify_receipts(catalog, output): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", type=Path, default=Path("target/acceptance-scenarios.v1.report.json")) - parser.add_argument("--all-targets", action="store_true", help="Also execute the rest of the required Rust test suite.") + parser.add_argument("--output", type=Path, help="Primary report path (defaults to target/-scenarios.v1.report.json).") + selection = parser.add_mutually_exclusive_group() + selection.add_argument("--all-targets", action="store_true", help="Execute all Rust targets and verify both acceptance and lifecycle coverage.") + selection.add_argument("--lifecycle-only", action="store_true", help="Execute and verify the durable lifecycle corpus only.") + parser.add_argument("--lifecycle-output", type=Path, default=Path("target/lifecycle-scenarios.v1.report.json"), help="Additional lifecycle report path with --all-targets.") parser.add_argument("--cargo", default="cargo", help="Cargo executable; dependencies must already be cached.") args = parser.parse_args() - report_path = args.output if args.output.is_absolute() else ROOT / args.output + selected = "lifecycle" if args.lifecycle_only else "acceptance" + output = args.output or Path(f"target/{selected}-scenarios.v1.report.json") + report_path = output if output.is_absolute() else ROOT / output + lifecycle_path = args.lifecycle_output if args.lifecycle_output.is_absolute() else ROOT / args.lifecycle_output + if args.all_targets and lifecycle_path.resolve() == report_path.resolve(): + raise ValueError("acceptance and lifecycle reports require distinct paths") # A failed run must never leave an older successful report at this target. report_path.parent.mkdir(parents=True, exist_ok=True) report_path.unlink(missing_ok=True) report_path.with_suffix(".failure.log").unlink(missing_ok=True) + if args.all_targets: + lifecycle_path.parent.mkdir(parents=True, exist_ok=True) + lifecycle_path.unlink(missing_ok=True) catalog = json.loads(CATALOG.read_text()) if catalog["schema_version"] != "flow.acceptance-scenario-catalog/v1": raise ValueError("unsupported catalog version") - if os.name != "posix": - raise ValueError("the PR matrix requires a Unix symlink-capable host") before = source_identity() - command = [args.cargo, "test", "--locked", "--offline"] - command += ["--all-targets"] if args.all_targets else ["--test", "hermetic_provider_kit", "scenario_matrix::"] - command += ["--", "--nocapture"] - result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, timeout=600, check=False) + command = bounded_test_process.cargo_command(args.cargo, ROOT) + command += ["--all-targets"] if args.all_targets else ["--test", "hermetic_provider_kit", "lifecycle_matrix::executable_lifecycle_matrix" if args.lifecycle_only else "scenario_matrix::"] + # A concurrent fork briefly inherits other workers' flock descriptors before + # exec. Isolate test-owned run workspaces; explicit contention tests still run. + command += ["--", "--nocapture", "--test-threads=1"] + try: + result = bounded_test_process.run(command, cwd=ROOT) + except bounded_test_process.BudgetExceeded as error: + report_path.with_suffix(".failure.log").write_bytes(error.output) + raise if result.returncode: # Retained locally for diagnosis. Raw test/provider text is not portable. log = report_path.with_suffix(".failure.log") - log.write_text((result.stdout + result.stderr)[-4_194_304:]) + log.write_text(result.stdout) raise ValueError(f"Rust tests failed; inspect local diagnostics at {log}") - if len(result.stdout.encode()) + len(result.stderr.encode()) > 4_194_304: - raise ValueError("test output exceeds the 4 MiB driver budget") - receipts = verify_receipts(catalog, result.stdout) + receipts = [] if args.lifecycle_only else verify_receipts(catalog, result.stdout) + lifecycle_catalog = json.loads(LIFECYCLE_CATALOG.read_text()) + lifecycle_receipts = lifecycle_reports.verify_receipts(lifecycle_catalog, result.stdout, before) if args.all_targets or args.lifecycle_only else [] after = source_identity() if before != after: changed = sorted(path for path in before["files"].keys() | after["files"].keys() if before["files"].get(path) != after["files"].get(path)) raise ValueError(f"source bytes changed during validation: {', '.join(changed)}") + toolchain = subprocess.check_output([args.cargo, "--version"], text=True, timeout=10).strip() report = { "schema_version": "flow.acceptance-scenario-report/v1", "catalog_digest": digest(CATALOG.read_bytes()), "source_identity": before, - "toolchain": subprocess.check_output([args.cargo, "--version"], text=True).strip(), + "toolchain": toolchain, "tier": catalog["tier"], "status": "passed", "scenario_count": len(receipts), @@ -115,10 +135,19 @@ def main(): "known_gaps": catalog["known_gaps"], "receipts": receipts, } - temporary = report_path.with_suffix(".tmp") + if args.lifecycle_only: + report = lifecycle_reports.report(lifecycle_catalog, LIFECYCLE_CATALOG.read_bytes(), lifecycle_receipts, before, toolchain) + write_report(report_path, report) + print(f"PASS: {report['scenario_count']} {selected} scenarios, two fresh roots each; {report_path}") + if args.all_targets: + write_report(lifecycle_path, lifecycle_reports.report(lifecycle_catalog, LIFECYCLE_CATALOG.read_bytes(), lifecycle_receipts, before, toolchain)) + print(f"PASS: {len(lifecycle_receipts)} lifecycle scenarios, two fresh roots each; {lifecycle_path}") + + +def write_report(path, report): + temporary = path.with_suffix(".tmp") temporary.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") - temporary.replace(report_path) - print(f"PASS: {len(receipts)} acceptance scenarios, two fresh roots each; {report_path}") + temporary.replace(path) if __name__ == "__main__": diff --git a/tools/test_lifecycle_report.py b/tools/test_lifecycle_report.py new file mode 100644 index 0000000..56219fa --- /dev/null +++ b/tools/test_lifecycle_report.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Exercise failure gates independently of the Rust matrix and enforce supervision.""" +from copy import deepcopy +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +import bounded_test_process as bounded +import lifecycle_reports as reports + +ROOT = Path(__file__).resolve().parents[1] +CATALOG = json.loads((ROOT / "tests/fixtures/lifecycle-scenarios.v1.json").read_text()) +SOURCE = {"files": {"contracts/fixtures/scenarios/sources/source-text.txt": hashlib.sha256( + (ROOT / "contracts/fixtures/scenarios/sources/source-text.txt").read_bytes()).hexdigest()}} + + +def receipt(case): + identity = dict.fromkeys(reports.IDENTITY_FIELDS, "a" * 64) + identity["input_digest"] = next(iter(SOURCE["files"].values())) + return dict(schema_version="flow.lifecycle-scenario-receipt/v1", scenario_id=case["scenario_id"], + recipe_digest=reports.canonical_digest({"fixture_version": "1.0.0", "case": case}), + fixture_identity=[identity] * (3 if case["graph"] else 1), + outcome=deepcopy(case["expected"]), recovery=case["recovery"], + plan_digest="b" * 64, history_digest="c" * 64, + artifact_digests=[None] * (3 if case["graph"] else 1)) + + +class ReceiptGate(unittest.TestCase): + def setUp(self): + self.receipts = [receipt(case) for case in CATALOG["scenarios"]] + + def verify(self, receipts=None, catalog=None, source=None): + output = "\n".join(reports.MARKER + json.dumps(row) for row in + (self.receipts if receipts is None else receipts)) + return reports.verify_receipts(catalog or CATALOG, output, source or SOURCE) + + def test_complete_exact_inventory(self): + self.assertEqual(len(self.verify()), len(CATALOG["scenarios"])) + + def test_missing_duplicate_unknown(self): + for rows in [self.receipts[:-1], self.receipts + [self.receipts[0]], + [dict(self.receipts[0], scenario_id="scenario:unknown"), *self.receipts[1:]]]: + with self.subTest(rows=len(rows)), self.assertRaises(ValueError): + self.verify(rows) + + def test_transition_order_eligibility_and_recovery(self): + mutations = [ + lambda row: row["outcome"]["history"].reverse(), + lambda row: row["outcome"]["assessments"][-1]["steps"][0].update(eligibility="reusable"), + lambda row: row.update(recovery="reuse-accepted-work"), + lambda row: row["outcome"].update(code="succeeded"), + ] + for mutate in mutations: + rows = deepcopy(self.receipts) + mutate(rows[0]) + with self.subTest(mutation=mutate), self.assertRaises(ValueError): + self.verify(rows) + + def test_versions_identities_and_private_fields(self): + mutations = [ + lambda row: row.update(schema_version="flow.lifecycle-scenario-receipt/v2"), + lambda row: row.update(recipe_digest="d" * 64), + lambda row: row.update(plan_digest=None), + lambda row: row.update(history_digest="private path"), + lambda row: row.update(raw_stderr="private provider text"), + lambda row: row["fixture_identity"][0].update(input_digest="d" * 64), + lambda row: row["fixture_identity"][0].update(executable_digest="d" * 64), + lambda row: row["fixture_identity"][0].update(raw_configuration="private"), + lambda row: row.update(artifact_digests=["private text"]), + ] + for mutate in mutations: + rows = deepcopy(self.receipts) + mutate(rows[0]) + with self.subTest(mutation=mutate), self.assertRaises(ValueError): + self.verify(rows) + + def test_catalog_drift_and_source_drift(self): + changed = deepcopy(CATALOG) + changed["scenarios"][0]["mode"] = "different-provider-behavior" + with self.assertRaisesRegex(ValueError, "recipe identity drift"): + self.verify(catalog=changed) + with self.assertRaisesRegex(ValueError, "source identity drift"): + self.verify(source={"files": dict.fromkeys(SOURCE["files"], "f" * 64)}) + for mutation in [{"fixture_version": "2.0.0"}, {"budget": {}}, {"known_gaps": []}]: + with self.assertRaises(ValueError): + self.verify(catalog=dict(CATALOG, **mutation)) + + def test_oversized_receipt(self): + self.receipts[0]["raw_stderr"] = "x" * reports.BUDGET["max_receipt_bytes"] + with self.assertRaisesRegex(ValueError, "exceeds budget"): + self.verify() + + +@unittest.skipUnless(sys.platform == "linux", "Linux resource qualification") +class TestSupervision(unittest.TestCase): + def test_combines_streams_without_shell(self): + result = bounded.run([sys.executable, "-c", "import sys; print('out'); print('err', file=sys.stderr)"], cwd=ROOT) + self.assertEqual(result.returncode, 0) + self.assertEqual(set(result.stdout.splitlines()), {"out", "err"}) + + def test_output_flood_is_stopped_during_capture(self): + with self.assertRaises(bounded.BudgetExceeded) as caught: + bounded.run([sys.executable, "-c", "import os\nwhile True: os.write(1, b'x' * 65536)"], + cwd=ROOT, timeout=5, max_output=8192) + self.assertEqual(len(caught.exception.output), 8192) + self.assertIn("output budget", str(caught.exception)) + + def test_hung_command_is_killed_and_reaped(self): + with self.assertRaisesRegex(bounded.BudgetExceeded, "runtime budget"): + bounded.run([sys.executable, "-c", "while True: pass"], cwd=ROOT, timeout=0.1) + + def test_kernel_rejects_excess_address_space_and_file_size(self): + program = """ +import errno, mmap, resource +assert resource.getrlimit(resource.RLIMIT_AS) == (4294967296, 4294967296) +assert resource.getrlimit(resource.RLIMIT_FSIZE) == (16777216, 16777216) +try: + mmap.mmap(-1, 4294967296) +except OSError as error: + assert error.errno == errno.ENOMEM +else: + raise AssertionError('memory limit did not apply') +with open('oversized-test-file', 'wb', buffering=0) as stream: + stream.seek(16777216) + try: + stream.write(b'x') + except OSError as error: + assert error.errno == errno.EFBIG + else: + raise AssertionError('file limit did not apply') +""" + with tempfile.TemporaryDirectory() as directory: + result = bounded.run([sys.executable, str(ROOT / "tools/bounded_test_runner.py"), + sys.executable, "-c", program], cwd=directory) + self.assertEqual(result.returncode, 0, result.stdout) + + +if __name__ == "__main__": + unittest.main()