From 3fee6f84df7cc7a5acfafa7ea1577c4540070a9a Mon Sep 17 00:00:00 2001 From: Shaw Zheng Date: Thu, 10 Sep 2026 06:09:22 +0800 Subject: [PATCH 01/11] feat(testing): define execution authorization lineage receipts --- README.md | 1 + .../execution-authorization-lineage.v1.md | 83 +++ .../execution_authorization_lineage.lua | 486 ++++++++++++++++++ libraries/contract/fkst.toml | 2 +- ...on_authorization_lineage_contract_test.lua | 360 +++++++++++++ 5 files changed, 931 insertions(+), 1 deletion(-) create mode 100644 contracts/execution-authorization-lineage.v1.md create mode 100644 libraries/contract/execution_authorization_lineage.lua create mode 100644 packages/testing-runner/tests/execution_authorization_lineage_contract_test.lua diff --git a/README.md b/README.md index b48402f6..86552efa 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ contracts/ agentic-browser-execution.v1.md defect-publication.v1.md environment-factory.v1.md + execution-authorization-lineage.v1.md project-profile.v1.md qa-publication.v1.md structured-execution.v2.md diff --git a/contracts/execution-authorization-lineage.v1.md b/contracts/execution-authorization-lineage.v1.md new file mode 100644 index 00000000..20e5cd85 --- /dev/null +++ b/contracts/execution-authorization-lineage.v1.md @@ -0,0 +1,83 @@ +# Execution authorization lineage v1 + +`contract.execution_authorization_lineage` defines closed audit receipts for Host-owned authorization +effects. The receipts let downstream consumers verify the exact authorization chain without turning +logs, counters, self-digests, or replay handles into an execution capability. + +## Boundary + +Every exported receipt fixes `evidence_role = audit-only`, `authorization_capability = false`, +`reusable = false`, and `source_max_uses = 1`. It binds one immutable repository commit plus the run, +trace, and dedup identities. Receipt validators require complete source bindings; shape validation or +partial caller-provided expectations are insufficient. + +The authoritative claim state remains in the Host durable store. Raw claim IDs, fence tokens, state +MACs, runtime configuration secrets, physical workspace paths, credentials, commands, and capability +payloads are forbidden from this exported lineage. Each Host profile-policy, authorization-policy, +attestation, and verifier field fixes one non-interchangeable kind and relative-key grammar; URLs, +traversal, query, fragment, userinfo, and local +path syntax are rejected. Claim receipts expose only a domain-separated +`claim_fingerprint_sha256` computed by the trusted Host from the internal claim handle. A fingerprint +cannot be submitted to complete or replay the claim. + +Possession or validation of a receipt does not authorize checkout, startup, test execution, +publication, promotion, or a gate effect. Receipts are one-way projections of authenticated Host state, +never inputs from which authority state may be reconstructed. + +## Receipt chain + +The fixed chain is: + +```text +testing-project-profile-approval-claim-receipt.v1 + -> testing-structured-preauthorization-claim-receipt.v1 + -> testing-structured-execution-grant-verification-receipt.v1 + -> testing-structured-execution-claim-receipt.v1 + -> testing-structured-execution-completion-receipt.v1 + -> testing-execution-authorization-lineage-index.v1 +``` + +The Profile receipt keeps persisted artifact SHA-256 and canonical Profile/Approval SHA-256 in +separate named fields. It is emitted only after trusted Approval authentication, point-of-use receipt +freshness validation, and a successful atomic single-use Profile claim. + +The Preauthorization receipt binds the Profile claim receipt, preauthorization, Profile digest, Case +Catalog, StructuredPlan, ready Environment Receipt, trusted policy, and the preauthorization claim +fingerprint. The Grant verification receipt then binds the Preauthorization claim receipt, Grant, +parent authorization, plan, environment, trusted authority attestation, and exact verifier identity. + +Execution claim and completion are distinct immutable receipts. The claim receipt binds the Grant +verification and Preauthorization claim receipts plus the exact operation and safe run-scoped execution +artifact root. It never contains result or completion fields. The completion receipt binds the claim +receipt to `/execution.json`, `/case-result-set.json`, +`/evidence-manifest.json`, and the completion time. + +The lineage index references all five receipts at fixed paths under: + +```text +.testing/runs//authorization-lineage/ +``` + +Receipt artifacts are persisted as canonical JSON. Index validation recomputes each receipt's canonical +JSON SHA-256 and requires its immutable ref, digest, value, and complete native source bindings. It +validates every receipt and all cross-receipt links. Every ref must resolve to the same run root; +cross-run, cross-repository, cross-plan, cross-environment, and cross-Grant substitutions fail closed +even if an attacker can reserialize the outer receipt. The chain also requires one continuous Host +authority, policy revision, Plan, and ready Environment Receipt from Preauthorization through Grant +and execution, plus monotonically ordered claim, verification, execution, completion, and index +timestamps. + +## Host integration + +The trusted Host must produce these receipts only after the corresponding real verifier or atomic +claim succeeds and persist them immutably. A fixture verifier may exercise the contract, but it is not +a production trust root. Loss of an exported receipt may be repaired only as an idempotent projection +of the same authenticated durable state; a receipt must never be imported to recreate a claim. + +Existing Project Profile, Grant request/result, structured execution request-v3, and summary-v1 +contracts remain unchanged. Production consumers requiring authorization lineage must use a future +explicit request/result version or a separate lineage index; adding receipt fields to strict v1 payloads +is not backward compatible. + +Diagnostic counters such as `claim_count = 1` or `grant_write_count = 1` do not substitute for this +receipt chain. diff --git a/libraries/contract/execution_authorization_lineage.lua b/libraries/contract/execution_authorization_lineage.lua new file mode 100644 index 00000000..36bfcf9a --- /dev/null +++ b/libraries/contract/execution_authorization_lineage.lua @@ -0,0 +1,486 @@ +-- contract.execution_authorization_lineage: non-capability audit evidence for Host authorization. +local error_facts = require("contract.error_facts") +local canonical_json = require("contract.canonical_json") +local sha256 = require("contract.sha256") +local time = require("contract.time") + +local M = {} + +M.schemas = { + profile_claim = "testing-project-profile-approval-claim-receipt.v1", + preauthorization_claim = "testing-structured-preauthorization-claim-receipt.v1", + grant_verification = "testing-structured-execution-grant-verification-receipt.v1", + execution_claim = "testing-structured-execution-claim-receipt.v1", + execution_completion = "testing-structured-execution-completion-receipt.v1", + lineage_index = "testing-execution-authorization-lineage-index.v1", +} + +M.paths = { + profile_claim = "authorization-lineage/profile-approval-claim.json", + preauthorization_claim = "authorization-lineage/preauthorization-claim.json", + grant_verification = "authorization-lineage/grant-verification.json", + execution_claim = "authorization-lineage/execution-claim.json", + execution_completion = "authorization-lineage/execution-completion.json", +} + +local common_fields = { + schema = true, status = true, receipt_id = true, repository = true, run_id = true, + trace_id = true, dedup_key = true, recorded_at = true, source_max_uses = true, + evidence_role = true, authorization_capability = true, reusable = true, +} + +local function fail(classification, message) + error(error_facts.error_message("contract.execution-authorization-lineage", classification, message)) +end + +local function field_set(specific) + local result = {} + for key in pairs(common_fields) do result[key] = true end + for _, key in ipairs(specific) do result[key] = true end + return result +end + +local function only_fields(value, allowed, context) + if type(value) ~= "table" then fail("malformed-" .. context, context .. " must be a table") end + for key in pairs(value) do + if type(key) ~= "string" or allowed[key] ~= true then + fail("malformed-" .. context, "unsupported field " .. tostring(key)) + end + end +end + +local function bounded(value, field, limit) + if type(value) ~= "string" or value == "" or #value > (limit or 1024) + or value:find("[%z\1-\31\127]") ~= nil then + fail("malformed-field", field .. " must be a bounded string") + end + return value +end + +local function identity(value, field) + bounded(value, field, 180) + if value:find("%s") ~= nil then fail("malformed-field", field .. " must not contain whitespace") end + return value +end + +local function digest(value, field) + if type(value) ~= "string" or #value ~= 64 or value:match("^[0-9a-f]+$") == nil then + fail("malformed-digest", field .. " must be a lowercase SHA-256 digest") + end + return value +end + +local function timestamp(value, field) + local epoch = time.iso_timestamp_epoch_seconds(value) + if epoch == nil then fail("malformed-time", field .. " must be a UTC timestamp") end + return epoch +end + +local function repository(value, field) + only_fields(value, { url = true, commit_sha = true }, field) + bounded(value.url, field .. ".url", 2048) + if value.url:match("^https://[^%s@/?#]+/[^%s?#]+$") == nil + or value.url:sub(-1) == "/" or value.url:find("\\", 1, true) ~= nil then + fail("malformed-repository", field .. ".url must be a canonical credential-free HTTPS URL") + end + if type(value.commit_sha) ~= "string" or #value.commit_sha ~= 40 + or value.commit_sha:match("^[0-9a-f]+$") == nil then + fail("mutable-revision", field .. ".commit_sha must be an immutable lowercase commit SHA") + end +end + +local function source_ref(value, field, expected_kind) + only_fields(value, { kind = true, ref = true }, field) + identity(value.kind, field .. ".kind") + bounded(value.ref, field .. ".ref", 4096) + local prefixes = { + ["host-profile-policy"] = "policies/", + ["host-policy"] = "policies/", + ["signed-attestation"] = "attestations/", + ["host-verifier"] = "verifiers/", + } + local prefix = prefixes[value.kind] + if value.kind ~= expected_kind or prefix == nil or value.ref:sub(1, #prefix) ~= prefix + or value.ref:match("^[A-Za-z0-9][A-Za-z0-9._/-]*$") == nil + or value.ref:find("//", 1, true) ~= nil or value.ref:sub(-1) == "/" then + fail("unsafe-reference", field .. " must be a closed Host identity reference") + end + for segment in value.ref:gmatch("[^/]+") do + if segment == "." or segment == ".." then + fail("unsafe-reference", field .. ".ref contains traversal") + end + end +end + +local function artifact_pointer(value, field) + bounded(value, field, 4096) + if value:match("^%.testing/runs/[A-Za-z0-9._-]+/.+$") == nil + or value:find("\\", 1, true) ~= nil or value:find("//", 1, true) ~= nil + or value:find("%s") ~= nil or value:find("?", 1, true) ~= nil + or value:find("#", 1, true) ~= nil then + fail("unsafe-reference", field .. " must be a run-scoped artifact pointer") + end + for segment in value:gmatch("[^/]+") do + if segment == "." or segment == ".." then fail("unsafe-reference", field .. " contains traversal") end + end +end + +local function run_root(value) + return value:match("^(%.testing/runs/[A-Za-z0-9._-]+)") +end + +local function validate_run_pointers(value, fields, context) + local expected_root = ".testing/runs/" .. value.run_id + for _, field in ipairs(fields) do + artifact_pointer(value[field], context .. "." .. field) + if run_root(value[field]) ~= expected_root then + fail("cross-run-reference", context .. "." .. field .. " is outside the receipt run") + end + end +end + +local function equal(left, right, seen) + if type(left) ~= type(right) then return false end + if type(left) ~= "table" then return left == right end + seen = seen or {} + if seen[left] == right then return true end + seen[left] = right + for key, value in pairs(left) do if not equal(value, right[key], seen) then return false end end + for key in pairs(right) do if left[key] == nil then return false end end + return true +end + +local function require_bindings(value, expected, fields, context) + if type(expected) ~= "table" then + fail("missing-source-bindings", context .. " requires complete source bindings") + end + local allowed = {} + for _, field in ipairs(fields) do allowed[field] = true end + only_fields(expected, allowed, context .. "-source-bindings") + for _, field in ipairs(fields) do + if expected[field] == nil then + fail("missing-source-binding", context .. "." .. field .. " source binding is required") + end + if not equal(value[field], expected[field]) then + fail("foreign-receipt", context .. "." .. field .. " differs from the authorized effect") + end + end +end + +local function validate_common(value, schema, status, context) + if value.schema ~= schema or value.status ~= status then + fail("malformed-receipt", context .. " schema or status is invalid") + end + identity(value.receipt_id, context .. ".receipt_id") + identity(value.run_id, context .. ".run_id") + identity(value.trace_id, context .. ".trace_id") + identity(value.dedup_key, context .. ".dedup_key") + timestamp(value.recorded_at, context .. ".recorded_at") + repository(value.repository, context .. ".repository") + if value.source_max_uses ~= 1 or value.evidence_role ~= "audit-only" + or value.authorization_capability ~= false or value.reusable ~= false then + fail("capability-confusion", context .. " must remain non-reusable audit evidence") + end +end + +local profile_fields = { + "repository", "run_id", "trace_id", "dedup_key", "profile_source_ref", + "profile_artifact_ref", "profile_artifact_sha256", "profile_sha256", "profile_revision", + "approval_artifact_ref", "approval_artifact_sha256", "approval_id", "approval_sha256", + "approval_authority", "policy_revision", "evidence_ref", "validation_receipt_ref", + "validation_receipt_sha256", "claim_fingerprint_sha256", "claimed_at", +} + +function M.validate_profile_claim_receipt(value, expected) + local context = "profile-approval-claim-receipt" + only_fields(value, field_set({ + "profile_source_ref", "profile_artifact_ref", "profile_artifact_sha256", "profile_sha256", + "profile_revision", "approval_artifact_ref", "approval_artifact_sha256", "approval_id", + "approval_sha256", "approval_authority", "policy_revision", "evidence_ref", + "validation_receipt_ref", "validation_receipt_sha256", "claim_fingerprint_sha256", + "claimed_at", + }), context) + validate_common(value, M.schemas.profile_claim, "claimed", context) + source_ref(value.profile_source_ref, context .. ".profile_source_ref", "host-profile-policy") + source_ref(value.approval_authority, context .. ".approval_authority", "host-policy") + source_ref(value.evidence_ref, context .. ".evidence_ref", "signed-attestation") + identity(value.profile_revision, context .. ".profile_revision") + identity(value.approval_id, context .. ".approval_id") + identity(value.policy_revision, context .. ".policy_revision") + validate_run_pointers(value, { + "profile_artifact_ref", "approval_artifact_ref", "validation_receipt_ref", + }, context) + for _, field in ipairs({ + "profile_artifact_sha256", "profile_sha256", "approval_artifact_sha256", + "approval_sha256", "validation_receipt_sha256", "claim_fingerprint_sha256", + }) do digest(value[field], context .. "." .. field) end + if timestamp(value.claimed_at, context .. ".claimed_at") > timestamp(value.recorded_at, context .. ".recorded_at") then + fail("malformed-time", context .. " cannot be recorded before its claim") + end + require_bindings(value, expected, profile_fields, context) + return value +end + +local preauthorization_fields = { + "repository", "run_id", "trace_id", "dedup_key", "profile_claim_receipt_ref", + "profile_claim_receipt_sha256", "preauthorization_ref", "preauthorization_sha256", + "authorization_id", "profile_sha256", "case_catalog_ref", "case_catalog_sha256", + "plan_ref", "plan_sha256", "environment_receipt_ref", "environment_receipt_sha256", + "authority", "policy_revision", "evidence_ref", "claim_fingerprint_sha256", "claimed_at", +} + +function M.validate_preauthorization_claim_receipt(value, expected) + local context = "preauthorization-claim-receipt" + only_fields(value, field_set({ + "profile_claim_receipt_ref", "profile_claim_receipt_sha256", "preauthorization_ref", + "preauthorization_sha256", "authorization_id", "profile_sha256", "case_catalog_ref", + "case_catalog_sha256", "plan_ref", "plan_sha256", "environment_receipt_ref", + "environment_receipt_sha256", "authority", "policy_revision", "evidence_ref", + "claim_fingerprint_sha256", "claimed_at", + }), context) + validate_common(value, M.schemas.preauthorization_claim, "claimed", context) + identity(value.authorization_id, context .. ".authorization_id") + identity(value.policy_revision, context .. ".policy_revision") + source_ref(value.authority, context .. ".authority", "host-policy") + source_ref(value.evidence_ref, context .. ".evidence_ref", "signed-attestation") + validate_run_pointers(value, { + "profile_claim_receipt_ref", "preauthorization_ref", "case_catalog_ref", "plan_ref", + "environment_receipt_ref", + }, context) + for _, field in ipairs({ + "profile_claim_receipt_sha256", "preauthorization_sha256", "profile_sha256", + "case_catalog_sha256", "plan_sha256", "environment_receipt_sha256", + "claim_fingerprint_sha256", + }) do digest(value[field], context .. "." .. field) end + if timestamp(value.claimed_at, context .. ".claimed_at") > timestamp(value.recorded_at, context .. ".recorded_at") then + fail("malformed-time", context .. " cannot be recorded before its claim") + end + require_bindings(value, expected, preauthorization_fields, context) + return value +end + +local grant_fields = { + "repository", "run_id", "trace_id", "dedup_key", "preauthorization_claim_receipt_ref", + "preauthorization_claim_receipt_sha256", "grant_ref", "grant_sha256", "grant_id", + "parent_authorization_ref", "parent_authorization_sha256", "plan_ref", "plan_sha256", + "environment_receipt_ref", "environment_receipt_sha256", "authority", "policy_revision", + "evidence_ref", "verifier_ref", "verification_id", "verified_at", +} + +function M.validate_grant_verification_receipt(value, expected) + local context = "grant-verification-receipt" + only_fields(value, field_set({ + "preauthorization_claim_receipt_ref", "preauthorization_claim_receipt_sha256", + "grant_ref", "grant_sha256", "grant_id", "parent_authorization_ref", + "parent_authorization_sha256", "plan_ref", "plan_sha256", "environment_receipt_ref", + "environment_receipt_sha256", "authority", "policy_revision", "evidence_ref", + "verifier_ref", "verification_id", "verified_at", + }), context) + validate_common(value, M.schemas.grant_verification, "authenticated", context) + identity(value.grant_id, context .. ".grant_id") + identity(value.policy_revision, context .. ".policy_revision") + identity(value.verification_id, context .. ".verification_id") + source_ref(value.authority, context .. ".authority", "host-policy") + source_ref(value.evidence_ref, context .. ".evidence_ref", "signed-attestation") + source_ref(value.verifier_ref, context .. ".verifier_ref", "host-verifier") + validate_run_pointers(value, { + "preauthorization_claim_receipt_ref", "grant_ref", "parent_authorization_ref", + "plan_ref", "environment_receipt_ref", + }, context) + for _, field in ipairs({ + "preauthorization_claim_receipt_sha256", "grant_sha256", + "parent_authorization_sha256", "plan_sha256", "environment_receipt_sha256", + }) do digest(value[field], context .. "." .. field) end + if timestamp(value.verified_at, context .. ".verified_at") > timestamp(value.recorded_at, context .. ".recorded_at") then + fail("malformed-time", context .. " cannot be recorded before verification") + end + require_bindings(value, expected, grant_fields, context) + return value +end + +local execution_claim_fields = { + "repository", "run_id", "trace_id", "dedup_key", "grant_verification_receipt_ref", + "grant_verification_receipt_sha256", "preauthorization_claim_receipt_ref", + "preauthorization_claim_receipt_sha256", "grant_ref", "grant_sha256", "grant_id", + "plan_ref", "plan_sha256", "environment_receipt_ref", "environment_receipt_sha256", + "artifact_root", "operation_id", "claim_fingerprint_sha256", "claimed_at", +} + +function M.validate_execution_claim_receipt(value, expected) + local context = "execution-claim-receipt" + only_fields(value, field_set({ + "grant_verification_receipt_ref", "grant_verification_receipt_sha256", + "preauthorization_claim_receipt_ref", "preauthorization_claim_receipt_sha256", + "grant_ref", "grant_sha256", "grant_id", "plan_ref", "plan_sha256", + "environment_receipt_ref", "environment_receipt_sha256", "artifact_root", + "operation_id", "claim_fingerprint_sha256", "claimed_at", + }), context) + validate_common(value, M.schemas.execution_claim, "claimed", context) + identity(value.grant_id, context .. ".grant_id") + identity(value.operation_id, context .. ".operation_id") + artifact_pointer(value.artifact_root, context .. ".artifact_root") + if run_root(value.artifact_root) ~= ".testing/runs/" .. value.run_id + or value.artifact_root:sub(-1) == "/" then + fail("cross-run-reference", context .. ".artifact_root must be inside the exact receipt run") + end + validate_run_pointers(value, { + "grant_verification_receipt_ref", "preauthorization_claim_receipt_ref", "grant_ref", + "plan_ref", "environment_receipt_ref", + }, context) + for _, field in ipairs({ + "grant_verification_receipt_sha256", "preauthorization_claim_receipt_sha256", + "grant_sha256", "plan_sha256", "environment_receipt_sha256", + "claim_fingerprint_sha256", + }) do digest(value[field], context .. "." .. field) end + if timestamp(value.claimed_at, context .. ".claimed_at") > timestamp(value.recorded_at, context .. ".recorded_at") then + fail("malformed-time", context .. " cannot be recorded before its claim") + end + require_bindings(value, expected, execution_claim_fields, context) + return value +end + +local completion_fields = { + "repository", "run_id", "trace_id", "dedup_key", "execution_claim_receipt_ref", + "execution_claim_receipt_sha256", "result_ref", "result_sha256", "case_result_set_ref", + "case_result_set_artifact_sha256", "evidence_manifest_ref", + "evidence_manifest_artifact_sha256", "completed_at", +} + +function M.validate_execution_completion_receipt(value, expected) + local context = "execution-completion-receipt" + only_fields(value, field_set({ + "execution_claim_receipt_ref", "execution_claim_receipt_sha256", "result_ref", + "result_sha256", "case_result_set_ref", "case_result_set_artifact_sha256", + "evidence_manifest_ref", "evidence_manifest_artifact_sha256", "completed_at", + }), context) + validate_common(value, M.schemas.execution_completion, "completed", context) + validate_run_pointers(value, { + "execution_claim_receipt_ref", "result_ref", "case_result_set_ref", + "evidence_manifest_ref", + }, context) + for _, field in ipairs({ + "execution_claim_receipt_sha256", "result_sha256", "case_result_set_artifact_sha256", + "evidence_manifest_artifact_sha256", + }) do digest(value[field], context .. "." .. field) end + if timestamp(value.completed_at, context .. ".completed_at") > timestamp(value.recorded_at, context .. ".recorded_at") then + fail("malformed-time", context .. " cannot be recorded before completion") + end + require_bindings(value, expected, completion_fields, context) + return value +end + +local receipt_names = { + "profile_claim", "preauthorization_claim", "grant_verification", + "execution_claim", "execution_completion", +} + +function M.validate_lineage_index(value, artifacts, expected) + local context = "authorization-lineage-index" + only_fields(value, { + schema = true, status = true, repository = true, run_id = true, trace_id = true, + dedup_key = true, recorded_at = true, receipts = true, lineage_complete = true, + source_max_uses = true, evidence_role = true, authorization_capability = true, + reusable = true, + }, context) + if value.schema ~= M.schemas.lineage_index or value.status ~= "complete" + or value.lineage_complete ~= true or value.evidence_role ~= "audit-only" + or value.source_max_uses ~= 1 or value.authorization_capability ~= false + or value.reusable ~= false then + fail("capability-confusion", context .. " must be complete non-reusable audit evidence") + end + identity(value.run_id, context .. ".run_id") + identity(value.trace_id, context .. ".trace_id") + identity(value.dedup_key, context .. ".dedup_key") + timestamp(value.recorded_at, context .. ".recorded_at") + repository(value.repository, context .. ".repository") + only_fields(value.receipts, { + profile_claim = true, preauthorization_claim = true, grant_verification = true, + execution_claim = true, execution_completion = true, + }, context .. ".receipts") + if type(artifacts) ~= "table" or type(expected) ~= "table" then + fail("missing-source-bindings", context .. " requires all receipt artifacts and source bindings") + end + local validators = { + profile_claim = M.validate_profile_claim_receipt, + preauthorization_claim = M.validate_preauthorization_claim_receipt, + grant_verification = M.validate_grant_verification_receipt, + execution_claim = M.validate_execution_claim_receipt, + execution_completion = M.validate_execution_completion_receipt, + } + local root = ".testing/runs/" .. value.run_id .. "/" + for _, name in ipairs(receipt_names) do + local binding = value.receipts[name] + only_fields(binding, { ref = true, sha256 = true }, context .. ".receipts." .. name) + artifact_pointer(binding.ref, context .. ".receipts." .. name .. ".ref") + digest(binding.sha256, context .. ".receipts." .. name .. ".sha256") + if binding.ref ~= root .. M.paths[name] then + fail("noncanonical-reference", context .. ".receipts." .. name .. " path is not canonical") + end + local artifact = artifacts[name] + if type(artifact) ~= "table" or artifact.ref ~= binding.ref or artifact.sha256 ~= binding.sha256 + or type(artifact.value) ~= "table" then + fail("immutable-binding-failed", context .. ".receipts." .. name .. " bytes are not bound") + end + if sha256.hex(canonical_json.encode(artifact.value)) ~= binding.sha256 then + fail("immutable-binding-failed", context .. ".receipts." .. name .. " canonical bytes differ") + end + validators[name](artifact.value, expected[name]) + if not equal(artifact.value.repository, value.repository) + or artifact.value.run_id ~= value.run_id or artifact.value.trace_id ~= value.trace_id + or artifact.value.dedup_key ~= value.dedup_key then + fail("foreign-receipt", context .. ".receipts." .. name .. " belongs to another run") + end + end + local profile = artifacts.profile_claim.value + local preauthorization = artifacts.preauthorization_claim.value + local grant = artifacts.grant_verification.value + local claim = artifacts.execution_claim.value + local completion = artifacts.execution_completion.value + if preauthorization.profile_claim_receipt_ref ~= value.receipts.profile_claim.ref + or preauthorization.profile_claim_receipt_sha256 ~= value.receipts.profile_claim.sha256 + or grant.preauthorization_claim_receipt_ref ~= value.receipts.preauthorization_claim.ref + or grant.preauthorization_claim_receipt_sha256 ~= value.receipts.preauthorization_claim.sha256 + or claim.grant_verification_receipt_ref ~= value.receipts.grant_verification.ref + or claim.grant_verification_receipt_sha256 ~= value.receipts.grant_verification.sha256 + or claim.preauthorization_claim_receipt_ref ~= value.receipts.preauthorization_claim.ref + or claim.preauthorization_claim_receipt_sha256 ~= value.receipts.preauthorization_claim.sha256 + or completion.execution_claim_receipt_ref ~= value.receipts.execution_claim.ref + or completion.execution_claim_receipt_sha256 ~= value.receipts.execution_claim.sha256 + or profile.profile_sha256 ~= preauthorization.profile_sha256 + or preauthorization.preauthorization_ref ~= grant.parent_authorization_ref + or preauthorization.preauthorization_sha256 ~= grant.parent_authorization_sha256 + or grant.grant_ref ~= claim.grant_ref or grant.grant_sha256 ~= claim.grant_sha256 + or grant.grant_id ~= claim.grant_id + or not equal(preauthorization.authority, grant.authority) + or preauthorization.policy_revision ~= grant.policy_revision + or preauthorization.plan_ref ~= grant.plan_ref + or preauthorization.plan_sha256 ~= grant.plan_sha256 + or preauthorization.environment_receipt_ref ~= grant.environment_receipt_ref + or preauthorization.environment_receipt_sha256 ~= grant.environment_receipt_sha256 + or grant.plan_ref ~= claim.plan_ref or grant.plan_sha256 ~= claim.plan_sha256 + or grant.environment_receipt_ref ~= claim.environment_receipt_ref + or grant.environment_receipt_sha256 ~= claim.environment_receipt_sha256 then + fail("lineage-binding-mismatch", context .. " receipt chain is incomplete or inconsistent") + end + if completion.result_ref ~= claim.artifact_root .. "/execution.json" + or completion.case_result_set_ref ~= claim.artifact_root .. "/case-result-set.json" + or completion.evidence_manifest_ref ~= claim.artifact_root .. "/evidence-manifest.json" then + fail("lineage-binding-mismatch", context .. " completion artifacts differ from the claimed execution root") + end + local profile_epoch = timestamp(profile.claimed_at, context .. ".profile_claim.claimed_at") + local preauthorization_epoch = timestamp( + preauthorization.claimed_at, context .. ".preauthorization_claim.claimed_at") + local grant_epoch = timestamp(grant.verified_at, context .. ".grant_verification.verified_at") + local claim_epoch = timestamp(claim.claimed_at, context .. ".execution_claim.claimed_at") + local completion_epoch = timestamp( + completion.completed_at, context .. ".execution_completion.completed_at") + local index_epoch = timestamp(value.recorded_at, context .. ".recorded_at") + if profile_epoch > preauthorization_epoch or preauthorization_epoch > grant_epoch + or grant_epoch > claim_epoch or claim_epoch > completion_epoch + or completion_epoch > index_epoch then + fail("lineage-chronology-invalid", context .. " authorization events are out of order") + end + return value +end + +return M diff --git a/libraries/contract/fkst.toml b/libraries/contract/fkst.toml index f8b7ba5c..f11c6bb1 100644 --- a/libraries/contract/fkst.toml +++ b/libraries/contract/fkst.toml @@ -18,7 +18,7 @@ version = "workspace" [exports] exact = true -public = ["contract.browser_control", "contract.browser_readiness", "contract.canonical_json", "contract.context_bundle_identity", "contract.convergence_identity", "contract.environment_factory", "contract.error_facts", "contract.github_issue_create", "contract.payload", "contract.sha256", "contract.project_profile", "contract.source_ref", "contract.strings", "contract.structured_execution", "contract.testing", "contract.testing_design", "contract.testing_design_generation", "contract.testing_execution", "contract.testing_package_executor", "contract.testing_package_release", "contract.testing_result_authority", "contract.testing_runner_invocation", "contract.testing_schema_catalog", "contract.testing_results", "contract.testing_results_compat", "contract.testing_evidence_manifest", "contract.testing_package_manifest", "contract.time", "contract.transition_version", "contract.workflow_qa"] +public = ["contract.browser_control", "contract.browser_readiness", "contract.canonical_json", "contract.context_bundle_identity", "contract.convergence_identity", "contract.environment_factory", "contract.error_facts", "contract.execution_authorization_lineage", "contract.github_issue_create", "contract.payload", "contract.sha256", "contract.project_profile", "contract.source_ref", "contract.strings", "contract.structured_execution", "contract.testing", "contract.testing_design", "contract.testing_design_generation", "contract.testing_execution", "contract.testing_package_executor", "contract.testing_package_release", "contract.testing_result_authority", "contract.testing_runner_invocation", "contract.testing_schema_catalog", "contract.testing_results", "contract.testing_results_compat", "contract.testing_evidence_manifest", "contract.testing_package_manifest", "contract.time", "contract.transition_version", "contract.workflow_qa"] [visibility] public = true diff --git a/packages/testing-runner/tests/execution_authorization_lineage_contract_test.lua b/packages/testing-runner/tests/execution_authorization_lineage_contract_test.lua new file mode 100644 index 00000000..abdc67e6 --- /dev/null +++ b/packages/testing-runner/tests/execution_authorization_lineage_contract_test.lua @@ -0,0 +1,360 @@ +local lineage = require("contract.execution_authorization_lineage") +local canonical_json = require("contract.canonical_json") +local sha256 = require("contract.sha256") +local t = fkst.test + +local function digest(char) return string.rep(char, 64) end +local function canonical_digest(value) return sha256.hex(canonical_json.encode(value)) end + +local function copy(value) + if type(value) ~= "table" then return value end + local result = {} + for key, item in pairs(value) do result[copy(key)] = copy(item) end + return result +end + +local function ref(run_id, suffix) return ".testing/runs/" .. run_id .. "/" .. suffix end + +local run_id = "authorization-lineage-run" +local repository = { + url = "https://example.invalid/testing/fixture.git", + commit_sha = string.rep("1", 40), +} + +local function envelope(value) + local result = { + repository = copy(repository), run_id = run_id, + trace_id = "trace-authorization-lineage", dedup_key = run_id, + recorded_at = "2026-09-10T00:10:00Z", source_max_uses = 1, + evidence_role = "audit-only", authorization_capability = false, reusable = false, + } + for key, item in pairs(value) do result[key] = copy(item) end + return result +end + +local function expected(value) + local result = copy(value) + for _, key in ipairs({ + "schema", "status", "receipt_id", "recorded_at", "source_max_uses", + "evidence_role", "authorization_capability", "reusable", + }) do result[key] = nil end + return result +end + +local paths = {} +for name, suffix in pairs(lineage.paths) do paths[name] = ref(run_id, suffix) end + +local artifact_digests = { + profile_claim = digest("1"), preauthorization_claim = digest("2"), + grant_verification = digest("3"), execution_claim = digest("4"), + execution_completion = digest("5"), +} + +local function profile_claim() + return envelope({ + schema = lineage.schemas.profile_claim, status = "claimed", + receipt_id = "profile-approval-claim-receipt-1", + profile_source_ref = { kind = "host-profile-policy", ref = "policies/profile-v1" }, + profile_artifact_ref = ref(run_id, "authorization/project-profile.json"), + profile_artifact_sha256 = digest("6"), profile_sha256 = digest("7"), + profile_revision = "fixture-profile-v1", + approval_artifact_ref = ref(run_id, "authorization/profile-approval.json"), + approval_artifact_sha256 = digest("8"), approval_id = "profile-approval-1", + approval_sha256 = digest("9"), + approval_authority = { kind = "host-policy", ref = "policies/profile-approval-v1" }, + policy_revision = "profile-approval-v1", + evidence_ref = { kind = "signed-attestation", ref = "attestations/profile-approval-1" }, + validation_receipt_ref = ref(run_id, "authorization/profile-validation-receipt.json"), + validation_receipt_sha256 = digest("a"), claim_fingerprint_sha256 = digest("b"), + claimed_at = "2026-09-10T00:01:00Z", + }) +end + +local function preauthorization_claim() + return envelope({ + schema = lineage.schemas.preauthorization_claim, status = "claimed", + receipt_id = "preauthorization-claim-receipt-1", + profile_claim_receipt_ref = paths.profile_claim, + profile_claim_receipt_sha256 = artifact_digests.profile_claim, + preauthorization_ref = ref(run_id, "execution/preauthorization.json"), + preauthorization_sha256 = digest("c"), authorization_id = "preauthorization-1", + profile_sha256 = digest("7"), case_catalog_ref = ref(run_id, "execution/case-catalog.json"), + case_catalog_sha256 = digest("d"), plan_ref = ref(run_id, "execution/structured-plan.json"), + plan_sha256 = digest("e"), + environment_receipt_ref = ref(run_id, "environment/environment-receipt-ready.json"), + environment_receipt_sha256 = digest("f"), + authority = { kind = "host-policy", ref = "policies/execution-v1" }, + policy_revision = "execution-v1", + evidence_ref = { kind = "signed-attestation", ref = "attestations/preauthorization-1" }, + claim_fingerprint_sha256 = digest("0"), claimed_at = "2026-09-10T00:02:00Z", + }) +end + +local function grant_verification() + return envelope({ + schema = lineage.schemas.grant_verification, status = "authenticated", + receipt_id = "grant-verification-receipt-1", + preauthorization_claim_receipt_ref = paths.preauthorization_claim, + preauthorization_claim_receipt_sha256 = artifact_digests.preauthorization_claim, + grant_ref = ref(run_id, "execution/execution-grant.json"), grant_sha256 = digest("a"), + grant_id = "execution-grant-1", + parent_authorization_ref = ref(run_id, "execution/preauthorization.json"), + parent_authorization_sha256 = digest("c"), + plan_ref = ref(run_id, "execution/structured-plan.json"), plan_sha256 = digest("e"), + environment_receipt_ref = ref(run_id, "environment/environment-receipt-ready.json"), + environment_receipt_sha256 = digest("f"), + authority = { kind = "host-policy", ref = "policies/execution-v1" }, + policy_revision = "execution-v1", + evidence_ref = { kind = "signed-attestation", ref = "attestations/execution-grant-1" }, + verifier_ref = { kind = "host-verifier", ref = "verifiers/execution-v1" }, + verification_id = "grant-verification-1", verified_at = "2026-09-10T00:03:00Z", + }) +end + +local function execution_claim() + return envelope({ + schema = lineage.schemas.execution_claim, status = "claimed", + receipt_id = "execution-claim-receipt-1", + grant_verification_receipt_ref = paths.grant_verification, + grant_verification_receipt_sha256 = artifact_digests.grant_verification, + preauthorization_claim_receipt_ref = paths.preauthorization_claim, + preauthorization_claim_receipt_sha256 = artifact_digests.preauthorization_claim, + grant_ref = ref(run_id, "execution/execution-grant.json"), grant_sha256 = digest("a"), + grant_id = "execution-grant-1", plan_ref = ref(run_id, "execution/structured-plan.json"), + plan_sha256 = digest("e"), + environment_receipt_ref = ref(run_id, "environment/environment-receipt-ready.json"), + environment_receipt_sha256 = digest("f"), artifact_root = ref(run_id, "execution"), + operation_id = run_id, claim_fingerprint_sha256 = digest("1"), + claimed_at = "2026-09-10T00:04:00Z", + }) +end + +local function execution_completion() + return envelope({ + schema = lineage.schemas.execution_completion, status = "completed", + receipt_id = "execution-completion-receipt-1", + execution_claim_receipt_ref = paths.execution_claim, + execution_claim_receipt_sha256 = artifact_digests.execution_claim, + result_ref = ref(run_id, "execution/execution.json"), result_sha256 = digest("2"), + case_result_set_ref = ref(run_id, "execution/case-result-set.json"), + case_result_set_artifact_sha256 = digest("3"), + evidence_manifest_ref = ref(run_id, "execution/evidence-manifest.json"), + evidence_manifest_artifact_sha256 = digest("4"), completed_at = "2026-09-10T00:05:00Z", + }) +end + +local function lineage_fixture() + local values = { + profile_claim = profile_claim(), preauthorization_claim = preauthorization_claim(), + grant_verification = grant_verification(), execution_claim = execution_claim(), + execution_completion = execution_completion(), + } + local bound_digests = {} + bound_digests.profile_claim = canonical_digest(values.profile_claim) + values.preauthorization_claim.profile_claim_receipt_sha256 = bound_digests.profile_claim + bound_digests.preauthorization_claim = canonical_digest(values.preauthorization_claim) + values.grant_verification.preauthorization_claim_receipt_sha256 = bound_digests.preauthorization_claim + bound_digests.grant_verification = canonical_digest(values.grant_verification) + values.execution_claim.grant_verification_receipt_sha256 = bound_digests.grant_verification + values.execution_claim.preauthorization_claim_receipt_sha256 = bound_digests.preauthorization_claim + bound_digests.execution_claim = canonical_digest(values.execution_claim) + values.execution_completion.execution_claim_receipt_sha256 = bound_digests.execution_claim + bound_digests.execution_completion = canonical_digest(values.execution_completion) + local artifacts, bindings, expected_values = {}, {}, {} + for name, value in pairs(values) do + artifacts[name] = { ref = paths[name], sha256 = bound_digests[name], value = value } + bindings[name] = { ref = paths[name], sha256 = bound_digests[name] } + expected_values[name] = expected(value) + end + local index = { + schema = lineage.schemas.lineage_index, status = "complete", repository = copy(repository), + run_id = run_id, trace_id = "trace-authorization-lineage", dedup_key = run_id, + recorded_at = "2026-09-10T00:10:00Z", receipts = bindings, lineage_complete = true, + source_max_uses = 1, evidence_role = "audit-only", + authorization_capability = false, reusable = false, + } + return index, artifacts, expected_values +end + +local function reseal_lineage(index, artifacts, expected_values) + local function bind(name) + local value = artifacts[name].value + local value_digest = canonical_digest(value) + artifacts[name].sha256 = value_digest + index.receipts[name].sha256 = value_digest + expected_values[name] = expected(value) + return value_digest + end + + local profile_digest = bind("profile_claim") + artifacts.preauthorization_claim.value.profile_claim_receipt_sha256 = profile_digest + local preauthorization_digest = bind("preauthorization_claim") + artifacts.grant_verification.value.preauthorization_claim_receipt_sha256 = preauthorization_digest + artifacts.execution_claim.value.preauthorization_claim_receipt_sha256 = preauthorization_digest + local grant_digest = bind("grant_verification") + artifacts.execution_claim.value.grant_verification_receipt_sha256 = grant_digest + local claim_digest = bind("execution_claim") + artifacts.execution_completion.value.execution_claim_receipt_sha256 = claim_digest + bind("execution_completion") +end + +return { + test_accepts_complete_non_capability_authorization_lineage = function() + local index, artifacts, expected_values = lineage_fixture() + t.eq(lineage.validate_lineage_index(index, artifacts, expected_values), index) + end, + + test_individual_receipts_require_complete_source_bindings = function() + local value = profile_claim() + t.raises(function() lineage.validate_profile_claim_receipt(value) end) + local expected_value = expected(value) + expected_value.profile_sha256 = nil + t.raises(function() lineage.validate_profile_claim_receipt(value, expected_value) end) + t.eq(lineage.validate_profile_claim_receipt(value, expected(value)), value) + end, + + test_exported_claim_receipts_reject_raw_claim_handles_and_capability_flags = function() + local value = execution_claim() + value.claim_id = "runtime-fence-handle" + t.raises(function() lineage.validate_execution_claim_receipt(value, expected(value)) end) + for _, mutate in ipairs({ + function(item) item.authorization_capability = true end, + function(item) item.reusable = true end, + function(item) item.source_max_uses = 2 end, + function(item) item.evidence_role = "authorization" end, + }) do + value = profile_claim() + mutate(value) + t.raises(function() lineage.validate_profile_claim_receipt(value, expected(value)) end) + end + end, + + test_source_identity_references_reject_credentials_paths_and_url_metadata = function() + for _, changed in ipairs({ + { field = "profile_source_ref", ref = "file:///private/tmp/profile" }, + { field = "profile_source_ref", ref = "https://user:token@example.test/profile" }, + { field = "profile_source_ref", ref = "policies/../durable-store/claim" }, + { field = "approval_authority", ref = "~/.ssh/id_ed25519" }, + { field = "approval_authority", ref = "policies/profile?id=secret" }, + { field = "evidence_ref", ref = "attestations/profile#replay-handle" }, + }) do + local value = profile_claim() + value[changed.field].ref = changed.ref + t.raises(function() lineage.validate_profile_claim_receipt(value, expected(value)) end) + end + end, + + test_source_identity_fields_reject_interchangeable_kinds = function() + for _, changed in ipairs({ + { field = "profile_source_ref", kind = "host-policy", ref = "policies/profile-v1" }, + { field = "approval_authority", kind = "signed-attestation", ref = "attestations/approval" }, + { field = "evidence_ref", kind = "host-verifier", ref = "verifiers/profile" }, + }) do + local value = profile_claim() + value[changed.field] = { kind = changed.kind, ref = changed.ref } + t.raises(function() lineage.validate_profile_claim_receipt(value, expected(value)) end) + end + local value = grant_verification() + value.verifier_ref = { kind = "host-policy", ref = "policies/execution-v1" } + t.raises(function() lineage.validate_grant_verification_receipt(value, expected(value)) end) + end, + + test_rejects_digest_domain_cross_run_and_cross_repository_substitution = function() + local value = profile_claim() + local source = expected(value) + value.profile_artifact_sha256, value.profile_sha256 = value.profile_sha256, value.profile_artifact_sha256 + t.raises(function() lineage.validate_profile_claim_receipt(value, source) end) + + value = grant_verification() + source = expected(value) + value.plan_ref = ref("another-run", "execution/structured-plan.json") + t.raises(function() lineage.validate_grant_verification_receipt(value, source) end) + + value = preauthorization_claim() + source = expected(value) + value.repository.commit_sha = string.rep("2", 40) + t.raises(function() lineage.validate_preauthorization_claim_receipt(value, source) end) + end, + + test_claim_and_completion_are_distinct_immutable_receipts = function() + local value = execution_claim() + value.result_ref = ref(run_id, "execution/execution.json") + value.result_sha256 = digest("2") + t.raises(function() lineage.validate_execution_claim_receipt(value, expected(value)) end) + + local completion = execution_completion() + completion.claim_fingerprint_sha256 = digest("1") + t.raises(function() lineage.validate_execution_completion_receipt(completion, expected(completion)) end) + end, + + test_index_rejects_incomplete_or_resealed_cross_artifact_lineage = function() + local index, artifacts, expected_values = lineage_fixture() + index.receipts.execution_completion = nil + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + + index, artifacts, expected_values = lineage_fixture() + artifacts.preauthorization_claim.value.profile_sha256 = digest("5") + expected_values.preauthorization_claim = expected(artifacts.preauthorization_claim.value) + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + + index, artifacts, expected_values = lineage_fixture() + artifacts.execution_claim.sha256 = digest("9") + index.receipts.execution_claim.sha256 = digest("9") + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + end, + + test_index_rejects_resealed_cross_grant_identity = function() + local index, artifacts, expected_values = lineage_fixture() + artifacts.execution_claim.value.grant_id = "foreign-execution-grant" + reseal_lineage(index, artifacts, expected_values) + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + end, + + test_index_rejects_completion_outside_claimed_execution_root = function() + local index, artifacts, expected_values = lineage_fixture() + artifacts.execution_completion.value.result_ref = ref(run_id, "foreign/execution.json") + reseal_lineage(index, artifacts, expected_values) + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + end, + + test_index_rejects_resealed_authority_or_policy_discontinuity = function() + for _, mutate in ipairs({ + function(value) value.authority.ref = "policies/foreign-execution-v1" end, + function(value) value.policy_revision = "foreign-execution-v1" end, + }) do + local index, artifacts, expected_values = lineage_fixture() + mutate(artifacts.grant_verification.value) + reseal_lineage(index, artifacts, expected_values) + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + end + end, + + test_index_rejects_resealed_plan_or_environment_substitution = function() + for _, mutate in ipairs({ + function(grant, claim) + grant.plan_ref = ref(run_id, "execution/foreign-plan.json") + grant.plan_sha256 = digest("6") + claim.plan_ref = grant.plan_ref + claim.plan_sha256 = grant.plan_sha256 + end, + function(grant, claim) + grant.environment_receipt_ref = ref(run_id, "environment/foreign-ready.json") + grant.environment_receipt_sha256 = digest("6") + claim.environment_receipt_ref = grant.environment_receipt_ref + claim.environment_receipt_sha256 = grant.environment_receipt_sha256 + end, + }) do + local index, artifacts, expected_values = lineage_fixture() + mutate(artifacts.grant_verification.value, artifacts.execution_claim.value) + reseal_lineage(index, artifacts, expected_values) + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + end + end, + + test_index_rejects_resealed_out_of_order_authorization_events = function() + local index, artifacts, expected_values = lineage_fixture() + artifacts.preauthorization_claim.value.claimed_at = "2026-09-09T23:59:59Z" + reseal_lineage(index, artifacts, expected_values) + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + end, +} From e4553d8334877fe529ddcf5b8304a1b29a632a3d Mon Sep 17 00:00:00 2001 From: Shaw Zheng Date: Thu, 10 Sep 2026 06:25:02 +0800 Subject: [PATCH 02/11] fix(testing): accept host identity reference keys --- libraries/contract/execution_authorization_lineage.lua | 9 +-------- .../execution_authorization_lineage_contract_test.lua | 8 ++++++++ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/libraries/contract/execution_authorization_lineage.lua b/libraries/contract/execution_authorization_lineage.lua index 36bfcf9a..2921e711 100644 --- a/libraries/contract/execution_authorization_lineage.lua +++ b/libraries/contract/execution_authorization_lineage.lua @@ -93,14 +93,7 @@ local function source_ref(value, field, expected_kind) only_fields(value, { kind = true, ref = true }, field) identity(value.kind, field .. ".kind") bounded(value.ref, field .. ".ref", 4096) - local prefixes = { - ["host-profile-policy"] = "policies/", - ["host-policy"] = "policies/", - ["signed-attestation"] = "attestations/", - ["host-verifier"] = "verifiers/", - } - local prefix = prefixes[value.kind] - if value.kind ~= expected_kind or prefix == nil or value.ref:sub(1, #prefix) ~= prefix + if value.kind ~= expected_kind or value.ref:match("^[A-Za-z0-9][A-Za-z0-9._/-]*$") == nil or value.ref:find("//", 1, true) ~= nil or value.ref:sub(-1) == "/" then fail("unsafe-reference", field .. " must be a closed Host identity reference") diff --git a/packages/testing-runner/tests/execution_authorization_lineage_contract_test.lua b/packages/testing-runner/tests/execution_authorization_lineage_contract_test.lua index abdc67e6..66b7404f 100644 --- a/packages/testing-runner/tests/execution_authorization_lineage_contract_test.lua +++ b/packages/testing-runner/tests/execution_authorization_lineage_contract_test.lua @@ -259,6 +259,14 @@ return { t.raises(function() lineage.validate_grant_verification_receipt(value, expected(value)) end) end, + test_source_identity_fields_accept_existing_host_relative_identity_keys = function() + local value = profile_claim() + value.profile_source_ref.ref = "fixtures/canonical-qa-profile" + value.approval_authority.ref = "fixtures/canonical-qa" + value.evidence_ref.ref = "fixtures/approvals/canonical-qa" + t.eq(lineage.validate_profile_claim_receipt(value, expected(value)), value) + end, + test_rejects_digest_domain_cross_run_and_cross_repository_substitution = function() local value = profile_claim() local source = expected(value) From 995efbb070878597ebc27aa80a58407aef74afb5 Mon Sep 17 00:00:00 2001 From: Shaw Zheng Date: Thu, 10 Sep 2026 23:26:37 +0800 Subject: [PATCH 03/11] feat(testing): enforce isolated execution authorization lineage --- README.md | 18 +- .../execution-authorization-lineage.v1.md | 18 + contracts/target-execution-boundary.v1.md | 48 + .../generic-host/bin/authorization-lineage.js | 389 ++++++ .../generic-host/bin/generic-host-runtime.js | 1059 +++++++++++++++-- .../bin/structured-execution-artifacts.js | 12 +- .../host_canonical_workflow_qa.lua | 54 +- .../generic-host/host_durable_workflow_qa.lua | 609 +++++++++- .../host_workflow_qa_supervisor.lua | 6 + ...thorization_lineage_node_validator_test.js | 374 ++++++ ...horization_lineage_node_validator_test.lua | 18 + ...ownstream_local_qa_acceptance_e2e_test.lua | 6 +- .../tests/durable_host_store_test.lua | 75 +- .../durable_workflow_qa_crash_matrix_test.lua | 1 + .../durable_workflow_qa_recovery_test.lua | 204 +++- .../tests/workflow_qa_adapter_test.lua | 7 + .../authorization_lineage_projection.lua | 168 +++ .../bin/fkst-structured-execution-runtime.js | 111 +- .../generic_host_workflow_qa.lua | 12 +- .../structured_execution_runtime_test.js | 112 +- libraries/testing_runtime/workflow_qa.lua | 8 +- .../workflow_qa_host_adapter.lua | 52 +- .../bin/environment-factory-runtime.js | 326 ++--- .../bin/runtime/budgets.js | 4 + .../environment-factory/bin/runtime/common.js | 448 ++++++- .../bin/runtime/lock-holder.js | 70 ++ .../bin/runtime/platform.js | 9 +- .../bin/runtime/supervised-process.js | 476 ++++++++ .../bin/runtime/target-execution-boundary.js | 114 ++ .../bin/runtime/workspace.js | 48 +- packages/environment-factory/runtime.lua | 1 + .../tests/fixtures/runtime/source/app.js | 3 + .../runtime/source/credential-isolation.js | 27 + .../runtime/source/database_service.py | 26 + .../fixtures/runtime/source/middleware.js | 3 + .../tests/fixtures/runtime/source/phase.js | 2 + .../tests/hermetic_e2e_test.lua | 10 + .../tests/node_runtime_test.js | 380 +++++- .../tests/host_boundary_test.lua | 2 + .../testing-runner/structured_execution.lua | 13 + .../tests/workflow_qa_host_adapter_test.lua | 96 ++ packages/workflow-qa/core.lua | 2 +- 42 files changed, 5014 insertions(+), 407 deletions(-) create mode 100644 contracts/target-execution-boundary.v1.md create mode 100644 examples/generic-host/bin/authorization-lineage.js create mode 100644 examples/generic-host/tests/authorization_lineage_node_validator_test.js create mode 100644 examples/generic-host/tests/authorization_lineage_node_validator_test.lua create mode 100644 libraries/testing_runtime/authorization_lineage_projection.lua create mode 100644 packages/environment-factory/bin/runtime/lock-holder.js create mode 100644 packages/environment-factory/bin/runtime/supervised-process.js create mode 100644 packages/environment-factory/bin/runtime/target-execution-boundary.js create mode 100644 packages/environment-factory/tests/fixtures/runtime/source/credential-isolation.js diff --git a/README.md b/README.md index 86552efa..f7409575 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Host repositories compose these packages and provide their own app-specific defa The formal full-FKST host flow is: -1. The downstream Host creates a product-specific `testing-project-profile.v1`, authenticates one-use approval/preauthorization artifacts, and persists sanitized validation receipts. +1. The downstream Host creates a product-specific `testing-project-profile.v1`, applies deterministic policy admission to the one-use profile/preauthorization artifacts, and persists sanitized validation receipts. The legacy `approval` schema name does not require a routine human action; the trusted Host policy remains the execution authority. 2. The Host submits `workflow-qa.run-request.v2` on `workflow-qa.qa_run_request`; product names, commands, URLs, and credential locations remain Host-owned. 3. `environment-factory` checks out, builds, starts, and publishes the immutable ready environment receipt. 4. `testing-design` produces repository and traceability context; `workflow-qa` then revalidates the exact browser session through `browser-readiness`. @@ -94,6 +94,10 @@ Issue seam and durable issue-written acknowledgement to the pinned `github-proxy Project startup configuration uses the separate `testing-project-profile.v1` and `testing-project-profile-approval.v1` contracts documented in `contracts/project-profile.v1.md`. +Hosts may issue the latter through deterministic machine policy; this package does not require a +routine human approval step. Removing human interaction does not merge the authority layers: +profile admission, run preauthorization, Grant verification, and the atomic single-use execution +claim remain distinct and fail closed. Profile validity and canonical digest identity never grant execution permission: a host trust root must authenticate the exact approval, and `contract.project_profile.authorize_execution` must recheck the profile, immutable repository commit, approval, validation receipt, freshness, and replay claim @@ -113,7 +117,17 @@ binds `{ url, commit_sha }` repository identity and the sanitized browser readin Factory does not start or acknowledge testing. Its production adapter is `packages/environment-factory/runtime.lua`, backed by the shell-free Node effect runner at `packages/environment-factory/bin/environment-factory-runtime.js`; the hermetic package test drives -that adapter through real Git, process, readiness, receipt, replay, and cleanup effects. +that adapter through real Git, process, readiness, receipt, replay, and cleanup effects. All target +checkout, build, start, readiness, test, and cleanup commands use a private leased home with GitHub, +Git credential-helper, SSH agent, askpass, hooks, and fsmonitor authority removed. Host command +environment cannot override those controls, and one-shot leases are removed immediately while a +supervised-process lease is retained only until verified cleanup. + +Those environment controls are defense in depth, not an operating-system sandbox. Every target +effect also requires the Host-owned `testing-host.target-execution-boundary.v1` contract documented +in `contracts/target-execution-boundary.v1.md`. The current production package accepts only an exact +immutable `trusted-fixture-exact` repository binding. Unknown or mismatched repositories fail closed +with `HOST_RUNTIME_ISOLATION_REQUIRED` until a future Host supplies verifiable OS/container isolation. Terminal Environment Factory results include an immutable typed cleanup-receipt pointer. The receipt lists attempted resources, verified removals, and remaining owner-bound cleanup handles; cross-run diff --git a/contracts/execution-authorization-lineage.v1.md b/contracts/execution-authorization-lineage.v1.md index 20e5cd85..dc1cbccf 100644 --- a/contracts/execution-authorization-lineage.v1.md +++ b/contracts/execution-authorization-lineage.v1.md @@ -74,6 +74,24 @@ claim succeeds and persist them immutably. A fixture verifier may exercise the c a production trust root. Loss of an exported receipt may be repaired only as an idempotent projection of the same authenticated durable state; a receipt must never be imported to recreate a claim. +The durable generic Host reference implementation projects all five receipts at the Profile claim, +Preauthorization claim, Grant verification, execution claim, and completion effect points. It writes +canonical JSON without a trailing newline so the persisted byte digest is the receipt canonical +digest, then validates the complete lineage index through this contract. Restart paths project the +same bytes from authenticated durable state and reject an existing Grant that cannot be reconciled to +its earlier claim. Raw durable handles never leave the Host. + +Routine human approval is not a requirement of this lineage. A trusted Host may make Profile and +Preauthorization decisions through deterministic machine policy. That automation does not collapse +the distinct single-use claims, turn audit receipts into capabilities, or grant publication, +promotion, regression, or gating authority. + +Authorization lineage also does not replace target isolation admission. Before a target effect, the +runtime independently requires the exact `testing-host.target-execution-boundary.v1` repository +binding documented in `contracts/target-execution-boundary.v1.md`. Compatibility, machine policy +admission, Preauthorization, and a valid Grant cannot authorize an unknown repository when that +boundary is absent or mismatched. + Existing Project Profile, Grant request/result, structured execution request-v3, and summary-v1 contracts remain unchanged. Production consumers requiring authorization lineage must use a future explicit request/result version or a separate lineage index; adding receipt fields to strict v1 payloads diff --git a/contracts/target-execution-boundary.v1.md b/contracts/target-execution-boundary.v1.md new file mode 100644 index 00000000..19feb040 --- /dev/null +++ b/contracts/target-execution-boundary.v1.md @@ -0,0 +1,48 @@ +# Target Execution Boundary v1 + +`testing-host.target-execution-boundary.v1` is a Host-owned admission contract for every command or +HTTP effect that can reach a target repository or its running application. It is not emitted by the +target repository, the implementation worker, PQL, or a run-scoped artifact producer. + +The current package accepts exactly one mode: + +```json +{ + "schema": "testing-host.target-execution-boundary.v1", + "mode": "trusted-fixture-exact", + "target_class": "host-owned-exact-trusted-fixture", + "repository": { + "url": "https://example.invalid/testing/fixture.git", + "commit_sha": "0123456789012345678901234567890123456789" + }, + "authority": { + "kind": "host-policy", + "ref": "fixtures/reviewed-fixture-boundary" + }, + "policy_revision": "reviewed-fixture-policy-v1", + "authorization_capability": false, + "execution_authorized": false +} +``` + +The repository URL must be canonical, credential-free HTTPS and the commit must be a full lowercase +40-character SHA. The Host runtime config must live under `.testing/host/**`, while the operation +artifact root must live under `.testing/runs/**`; the two namespaces cannot overlap. Environment +Factory validates the boundary before checkout, target commands, readiness, and target cleanup. +Structured Execution independently validates it before issuing a CLI effect receipt, consuming that +receipt, or sending a target HTTP request. The Generic Host persists the same exact binding in its +durable Host config. + +`trusted-fixture-exact` means that the Host operator has reviewed and admitted that exact immutable +fixture. It must not be inferred from repository contents, a PQL compatibility result, an asset +admission, an execution preauthorization, or a Grant. A URL or commit mismatch fails closed with +`HOST_RUNTIME_ISOLATION_REQUIRED`. + +No other execution-boundary mode is currently accepted. In particular, a boolean such as +`isolated=true` is not evidence of isolation. A future untrusted-repository mode requires a separate, +verifiable Host-owned receipt from an OS identity, container, VM, or equivalent filesystem boundary +that prevents target code from reading or replacing Host credentials. + +The private HOME lease, disabled Git credential helpers/hooks/fsmonitor, and removed GitHub and SSH +environment variables are defense-in-depth controls for an admitted fixture. They are not an +operating-system sandbox and do not make arbitrary code under the Host UID safe. diff --git a/examples/generic-host/bin/authorization-lineage.js b/examples/generic-host/bin/authorization-lineage.js new file mode 100644 index 00000000..e80b2106 --- /dev/null +++ b/examples/generic-host/bin/authorization-lineage.js @@ -0,0 +1,389 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { stable } = require('./durable-host-store'); + +const schemas = Object.freeze({ + profile_claim: 'testing-project-profile-approval-claim-receipt.v1', + preauthorization_claim: 'testing-structured-preauthorization-claim-receipt.v1', + grant_verification: 'testing-structured-execution-grant-verification-receipt.v1', + execution_claim: 'testing-structured-execution-claim-receipt.v1', + execution_completion: 'testing-structured-execution-completion-receipt.v1', + lineage_index: 'testing-execution-authorization-lineage-index.v1', +}); + +const paths = Object.freeze({ + profile_claim: 'authorization-lineage/profile-approval-claim.json', + preauthorization_claim: 'authorization-lineage/preauthorization-claim.json', + grant_verification: 'authorization-lineage/grant-verification.json', + execution_claim: 'authorization-lineage/execution-claim.json', + execution_completion: 'authorization-lineage/execution-completion.json', +}); + +const receiptNames = Object.freeze(Object.keys(paths)); +const commonFields = [ + 'schema', 'status', 'receipt_id', 'repository', 'run_id', 'trace_id', 'dedup_key', + 'recorded_at', 'source_max_uses', 'evidence_role', 'authorization_capability', 'reusable', +]; + +const definitions = Object.freeze({ + profile_claim: { + status: 'claimed', event: 'claimed_at', + fields: [ + 'profile_source_ref', 'profile_artifact_ref', 'profile_artifact_sha256', 'profile_sha256', + 'profile_revision', 'approval_artifact_ref', 'approval_artifact_sha256', 'approval_id', + 'approval_sha256', 'approval_authority', 'policy_revision', 'evidence_ref', + 'validation_receipt_ref', 'validation_receipt_sha256', 'claim_fingerprint_sha256', 'claimed_at', + ], + sources: [ + 'repository', 'run_id', 'trace_id', 'dedup_key', 'profile_source_ref', + 'profile_artifact_ref', 'profile_artifact_sha256', 'profile_sha256', 'profile_revision', + 'approval_artifact_ref', 'approval_artifact_sha256', 'approval_id', 'approval_sha256', + 'approval_authority', 'policy_revision', 'evidence_ref', 'validation_receipt_ref', + 'validation_receipt_sha256', 'claim_fingerprint_sha256', 'claimed_at', + ], + pointers: ['profile_artifact_ref', 'approval_artifact_ref', 'validation_receipt_ref'], + digests: [ + 'profile_artifact_sha256', 'profile_sha256', 'approval_artifact_sha256', + 'approval_sha256', 'validation_receipt_sha256', 'claim_fingerprint_sha256', + ], + }, + preauthorization_claim: { + status: 'claimed', event: 'claimed_at', + fields: [ + 'profile_claim_receipt_ref', 'profile_claim_receipt_sha256', 'preauthorization_ref', + 'preauthorization_sha256', 'authorization_id', 'profile_sha256', 'case_catalog_ref', + 'case_catalog_sha256', 'plan_ref', 'plan_sha256', 'environment_receipt_ref', + 'environment_receipt_sha256', 'authority', 'policy_revision', 'evidence_ref', + 'claim_fingerprint_sha256', 'claimed_at', + ], + sources: [ + 'repository', 'run_id', 'trace_id', 'dedup_key', 'profile_claim_receipt_ref', + 'profile_claim_receipt_sha256', 'preauthorization_ref', 'preauthorization_sha256', + 'authorization_id', 'profile_sha256', 'case_catalog_ref', 'case_catalog_sha256', + 'plan_ref', 'plan_sha256', 'environment_receipt_ref', 'environment_receipt_sha256', + 'authority', 'policy_revision', 'evidence_ref', 'claim_fingerprint_sha256', 'claimed_at', + ], + pointers: [ + 'profile_claim_receipt_ref', 'preauthorization_ref', 'case_catalog_ref', 'plan_ref', + 'environment_receipt_ref', + ], + digests: [ + 'profile_claim_receipt_sha256', 'preauthorization_sha256', 'profile_sha256', + 'case_catalog_sha256', 'plan_sha256', 'environment_receipt_sha256', + 'claim_fingerprint_sha256', + ], + }, + grant_verification: { + status: 'authenticated', event: 'verified_at', + fields: [ + 'preauthorization_claim_receipt_ref', 'preauthorization_claim_receipt_sha256', + 'grant_ref', 'grant_sha256', 'grant_id', 'parent_authorization_ref', + 'parent_authorization_sha256', 'plan_ref', 'plan_sha256', 'environment_receipt_ref', + 'environment_receipt_sha256', 'authority', 'policy_revision', 'evidence_ref', + 'verifier_ref', 'verification_id', 'verified_at', + ], + sources: [ + 'repository', 'run_id', 'trace_id', 'dedup_key', 'preauthorization_claim_receipt_ref', + 'preauthorization_claim_receipt_sha256', 'grant_ref', 'grant_sha256', 'grant_id', + 'parent_authorization_ref', 'parent_authorization_sha256', 'plan_ref', 'plan_sha256', + 'environment_receipt_ref', 'environment_receipt_sha256', 'authority', 'policy_revision', + 'evidence_ref', 'verifier_ref', 'verification_id', 'verified_at', + ], + pointers: [ + 'preauthorization_claim_receipt_ref', 'grant_ref', 'parent_authorization_ref', + 'plan_ref', 'environment_receipt_ref', + ], + digests: [ + 'preauthorization_claim_receipt_sha256', 'grant_sha256', 'parent_authorization_sha256', + 'plan_sha256', 'environment_receipt_sha256', + ], + }, + execution_claim: { + status: 'claimed', event: 'claimed_at', + fields: [ + 'grant_verification_receipt_ref', 'grant_verification_receipt_sha256', + 'preauthorization_claim_receipt_ref', 'preauthorization_claim_receipt_sha256', + 'grant_ref', 'grant_sha256', 'grant_id', 'plan_ref', 'plan_sha256', + 'environment_receipt_ref', 'environment_receipt_sha256', 'artifact_root', + 'operation_id', 'claim_fingerprint_sha256', 'claimed_at', + ], + sources: [ + 'repository', 'run_id', 'trace_id', 'dedup_key', 'grant_verification_receipt_ref', + 'grant_verification_receipt_sha256', 'preauthorization_claim_receipt_ref', + 'preauthorization_claim_receipt_sha256', 'grant_ref', 'grant_sha256', 'grant_id', + 'plan_ref', 'plan_sha256', 'environment_receipt_ref', 'environment_receipt_sha256', + 'artifact_root', 'operation_id', 'claim_fingerprint_sha256', 'claimed_at', + ], + pointers: [ + 'grant_verification_receipt_ref', 'preauthorization_claim_receipt_ref', 'grant_ref', + 'plan_ref', 'environment_receipt_ref', 'artifact_root', + ], + digests: [ + 'grant_verification_receipt_sha256', 'preauthorization_claim_receipt_sha256', + 'grant_sha256', 'plan_sha256', 'environment_receipt_sha256', 'claim_fingerprint_sha256', + ], + }, + execution_completion: { + status: 'completed', event: 'completed_at', + fields: [ + 'execution_claim_receipt_ref', 'execution_claim_receipt_sha256', 'result_ref', + 'result_sha256', 'case_result_set_ref', 'case_result_set_artifact_sha256', + 'evidence_manifest_ref', 'evidence_manifest_artifact_sha256', 'completed_at', + ], + sources: [ + 'repository', 'run_id', 'trace_id', 'dedup_key', 'execution_claim_receipt_ref', + 'execution_claim_receipt_sha256', 'result_ref', 'result_sha256', 'case_result_set_ref', + 'case_result_set_artifact_sha256', 'evidence_manifest_ref', + 'evidence_manifest_artifact_sha256', 'completed_at', + ], + pointers: [ + 'execution_claim_receipt_ref', 'result_ref', 'case_result_set_ref', 'evidence_manifest_ref', + ], + digests: [ + 'execution_claim_receipt_sha256', 'result_sha256', 'case_result_set_artifact_sha256', + 'evidence_manifest_artifact_sha256', + ], + }, +}); + +function fail(message) { + throw new Error(`authorization-lineage contract: ${message}`); +} + +function object(value, field) { + if (!value || typeof value !== 'object' || Array.isArray(value)) fail(`${field} must be an object`); + return value; +} + +function exactKeys(value, keys, field) { + object(value, field); + const expected = [...keys].sort(); + const actual = Object.keys(value).sort(); + if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { + fail(`${field} fields are not closed`); + } +} + +function bounded(value, field, limit = 1024) { + if (typeof value !== 'string' || value === '' || Buffer.byteLength(value, 'utf8') > limit + || /[\u0000-\u001f\u007f]/.test(value)) { + fail(`${field} must be a bounded string`); + } +} + +function identity(value, field) { + bounded(value, field, 180); + if (/\s/.test(value)) fail(`${field} must not contain whitespace`); +} + +function digest(value, field) { + if (typeof value !== 'string' || !/^[0-9a-f]{64}$/.test(value)) fail(`${field} must be a SHA-256`); +} + +function timestamp(value, field) { + const match = typeof value === 'string' + ? /^(\d{4})-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)Z$/.exec(value) : null; + if (!match) fail(`${field} must be a UTC timestamp`); + const [year, month, day, hour, minute, second] = match.slice(1).map(Number); + const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); + const daysInMonth = [31, leap ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + if (month < 1 || month > 12 || day < 1 || day > daysInMonth[month - 1] + || hour > 23 || minute > 59 || second > 59) fail(`${field} must be a UTC timestamp`); + let adjustedYear = year; + let adjustedMonth = month; + if (adjustedMonth <= 2) { + adjustedYear -= 1; + adjustedMonth += 12; + } + const era = Math.floor(adjustedYear / 400); + const yearOfEra = adjustedYear - era * 400; + const dayOfYear = Math.floor((153 * (adjustedMonth - 3) + 2) / 5) + day - 1; + const dayOfEra = yearOfEra * 365 + Math.floor(yearOfEra / 4) + - Math.floor(yearOfEra / 100) + dayOfYear; + const daysSinceEpoch = era * 146097 + dayOfEra - 719468; + return (daysSinceEpoch * 86400 + hour * 3600 + minute * 60 + second) * 1000; +} + +function repository(value, field) { + exactKeys(value, ['url', 'commit_sha'], field); + bounded(value.url, `${field}.url`, 2048); + if (!/^https:\/\/[^\s@/?#]+\/[^\s?#]+$/.test(value.url) || value.url.endsWith('/') || value.url.includes('\\')) { + fail(`${field}.url must be a canonical credential-free HTTPS URL`); + } + if (!/^[0-9a-f]{40}$/.test(value.commit_sha)) fail(`${field}.commit_sha must be immutable`); +} + +function sourceRef(value, field, kind) { + exactKeys(value, ['kind', 'ref'], field); + identity(value.kind, `${field}.kind`); + bounded(value.ref, `${field}.ref`, 4096); + if (value.kind !== kind || !/^[A-Za-z0-9][A-Za-z0-9._/-]*$/.test(value.ref) + || value.ref.includes('//') || value.ref.endsWith('/') + || value.ref.split('/').some((segment) => segment === '.' || segment === '..')) { + fail(`${field} must be a closed Host identity reference`); + } +} + +function artifactPointer(value, field) { + bounded(value, field, 4096); + if (!/^\.testing\/runs\/[A-Za-z0-9._-]+\/.+$/.test(value) || value.includes('\\') + || value.includes('//') || /\s/.test(value) || value.includes('?') || value.includes('#') + || value.split('/').some((segment) => segment === '.' || segment === '..')) { + fail(`${field} must be a run-scoped artifact pointer`); + } +} + +function same(left, right) { + return stable(left) === stable(right); +} + +function runRoot(pointer) { + const match = /^(\.testing\/runs\/[A-Za-z0-9._-]+)/.exec(pointer); + return match && match[1]; +} + +function validateCommon(value, name, definition) { + exactKeys(value, [...commonFields, ...definition.fields], `${name} receipt`); + if (value.schema !== schemas[name] || value.status !== definition.status) fail(`${name} schema or status differs`); + identity(value.receipt_id, `${name}.receipt_id`); + identity(value.run_id, `${name}.run_id`); + identity(value.trace_id, `${name}.trace_id`); + identity(value.dedup_key, `${name}.dedup_key`); + timestamp(value.recorded_at, `${name}.recorded_at`); + repository(value.repository, `${name}.repository`); + if (value.source_max_uses !== 1 || value.evidence_role !== 'audit-only' + || value.authorization_capability !== false || value.reusable !== false) { + fail(`${name} must remain non-reusable audit evidence`); + } + for (const field of definition.pointers) { + artifactPointer(value[field], `${name}.${field}`); + if (runRoot(value[field]) !== `.testing/runs/${value.run_id}`) fail(`${name}.${field} belongs to another run`); + } + for (const field of definition.digests) digest(value[field], `${name}.${field}`); + if (timestamp(value[definition.event], `${name}.${definition.event}`) > Date.parse(value.recorded_at)) { + fail(`${name} was recorded before its source event`); + } +} + +function validateSpecific(value, name) { + if (name === 'profile_claim') { + sourceRef(value.profile_source_ref, 'profile_claim.profile_source_ref', 'host-profile-policy'); + sourceRef(value.approval_authority, 'profile_claim.approval_authority', 'host-policy'); + sourceRef(value.evidence_ref, 'profile_claim.evidence_ref', 'signed-attestation'); + identity(value.profile_revision, 'profile_claim.profile_revision'); + identity(value.approval_id, 'profile_claim.approval_id'); + identity(value.policy_revision, 'profile_claim.policy_revision'); + } else if (name === 'preauthorization_claim') { + identity(value.authorization_id, 'preauthorization_claim.authorization_id'); + identity(value.policy_revision, 'preauthorization_claim.policy_revision'); + sourceRef(value.authority, 'preauthorization_claim.authority', 'host-policy'); + sourceRef(value.evidence_ref, 'preauthorization_claim.evidence_ref', 'signed-attestation'); + } else if (name === 'grant_verification') { + identity(value.grant_id, 'grant_verification.grant_id'); + identity(value.policy_revision, 'grant_verification.policy_revision'); + identity(value.verification_id, 'grant_verification.verification_id'); + sourceRef(value.authority, 'grant_verification.authority', 'host-policy'); + sourceRef(value.evidence_ref, 'grant_verification.evidence_ref', 'signed-attestation'); + sourceRef(value.verifier_ref, 'grant_verification.verifier_ref', 'host-verifier'); + } else if (name === 'execution_claim') { + identity(value.grant_id, 'execution_claim.grant_id'); + identity(value.operation_id, 'execution_claim.operation_id'); + if (value.artifact_root.endsWith('/')) fail('execution_claim.artifact_root is not exact'); + } +} + +function validateReceipt(name, value, expected) { + const definition = definitions[name]; + if (!definition) fail(`unsupported receipt ${name}`); + validateCommon(value, name, definition); + validateSpecific(value, name); + exactKeys(expected, definition.sources, `${name} trusted source bindings`); + for (const field of definition.sources) { + if (!same(value[field], expected[field])) fail(`${name}.${field} differs from its trusted source`); + } + return value; +} + +function canonicalDigest(value) { + return crypto.createHash('sha256').update(stable(value)).digest('hex'); +} + +function validateLineageIndex(value, artifacts, expected) { + exactKeys(value, [ + 'schema', 'status', 'repository', 'run_id', 'trace_id', 'dedup_key', 'recorded_at', + 'receipts', 'lineage_complete', 'source_max_uses', 'evidence_role', + 'authorization_capability', 'reusable', + ], 'lineage index'); + if (value.schema !== schemas.lineage_index || value.status !== 'complete' + || value.lineage_complete !== true || value.source_max_uses !== 1 + || value.evidence_role !== 'audit-only' || value.authorization_capability !== false + || value.reusable !== false) fail('lineage index must remain complete non-reusable audit evidence'); + repository(value.repository, 'lineage index.repository'); + identity(value.run_id, 'lineage index.run_id'); + identity(value.trace_id, 'lineage index.trace_id'); + identity(value.dedup_key, 'lineage index.dedup_key'); + const indexTime = timestamp(value.recorded_at, 'lineage index.recorded_at'); + exactKeys(value.receipts, receiptNames, 'lineage index.receipts'); + object(artifacts, 'lineage artifacts'); + object(expected, 'lineage trusted source bindings'); + const root = `.testing/runs/${value.run_id}`; + for (const name of receiptNames) { + exactKeys(value.receipts[name], ['ref', 'sha256'], `lineage index.receipts.${name}`); + const binding = value.receipts[name]; + artifactPointer(binding.ref, `lineage index.receipts.${name}.ref`); + digest(binding.sha256, `lineage index.receipts.${name}.sha256`); + if (binding.ref !== `${root}/${paths[name]}`) fail(`${name} receipt path is not canonical`); + const artifact = artifacts[name]; + if (!artifact || artifact.ref !== binding.ref || artifact.sha256 !== binding.sha256 + || canonicalDigest(artifact.value) !== binding.sha256) fail(`${name} immutable binding differs`); + validateReceipt(name, artifact.value, expected[name]); + if (!same(artifact.value.repository, value.repository) || artifact.value.run_id !== value.run_id + || artifact.value.trace_id !== value.trace_id || artifact.value.dedup_key !== value.dedup_key) { + fail(`${name} receipt belongs to another run`); + } + } + const profile = artifacts.profile_claim.value; + const preauthorization = artifacts.preauthorization_claim.value; + const grant = artifacts.grant_verification.value; + const claim = artifacts.execution_claim.value; + const completion = artifacts.execution_completion.value; + if (preauthorization.profile_claim_receipt_ref !== value.receipts.profile_claim.ref + || preauthorization.profile_claim_receipt_sha256 !== value.receipts.profile_claim.sha256 + || grant.preauthorization_claim_receipt_ref !== value.receipts.preauthorization_claim.ref + || grant.preauthorization_claim_receipt_sha256 !== value.receipts.preauthorization_claim.sha256 + || claim.grant_verification_receipt_ref !== value.receipts.grant_verification.ref + || claim.grant_verification_receipt_sha256 !== value.receipts.grant_verification.sha256 + || claim.preauthorization_claim_receipt_ref !== value.receipts.preauthorization_claim.ref + || claim.preauthorization_claim_receipt_sha256 !== value.receipts.preauthorization_claim.sha256 + || completion.execution_claim_receipt_ref !== value.receipts.execution_claim.ref + || completion.execution_claim_receipt_sha256 !== value.receipts.execution_claim.sha256 + || profile.profile_sha256 !== preauthorization.profile_sha256 + || preauthorization.preauthorization_ref !== grant.parent_authorization_ref + || preauthorization.preauthorization_sha256 !== grant.parent_authorization_sha256 + || grant.grant_ref !== claim.grant_ref || grant.grant_sha256 !== claim.grant_sha256 + || grant.grant_id !== claim.grant_id || !same(preauthorization.authority, grant.authority) + || preauthorization.policy_revision !== grant.policy_revision + || preauthorization.plan_ref !== grant.plan_ref || preauthorization.plan_sha256 !== grant.plan_sha256 + || preauthorization.environment_receipt_ref !== grant.environment_receipt_ref + || preauthorization.environment_receipt_sha256 !== grant.environment_receipt_sha256 + || grant.plan_ref !== claim.plan_ref || grant.plan_sha256 !== claim.plan_sha256 + || grant.environment_receipt_ref !== claim.environment_receipt_ref + || grant.environment_receipt_sha256 !== claim.environment_receipt_sha256) { + fail('receipt chain is incomplete or inconsistent'); + } + if (completion.result_ref !== `${claim.artifact_root}/execution.json` + || completion.case_result_set_ref !== `${claim.artifact_root}/case-result-set.json` + || completion.evidence_manifest_ref !== `${claim.artifact_root}/evidence-manifest.json`) { + fail('completion artifacts differ from the claimed execution root'); + } + const chronology = [ + profile.claimed_at, preauthorization.claimed_at, grant.verified_at, + claim.claimed_at, completion.completed_at, value.recorded_at, + ].map((item, index) => timestamp(item, `lineage chronology ${index}`)); + if (chronology.some((item, index) => index > 0 && chronology[index - 1] > item) + || chronology[chronology.length - 1] !== indexTime) fail('authorization events are out of order'); + return value; +} + +module.exports = { definitions, paths, receiptNames, schemas, validateLineageIndex, validateReceipt }; diff --git a/examples/generic-host/bin/generic-host-runtime.js b/examples/generic-host/bin/generic-host-runtime.js index 889ae84a..cc1daace 100755 --- a/examples/generic-host/bin/generic-host-runtime.js +++ b/examples/generic-host/bin/generic-host-runtime.js @@ -4,8 +4,9 @@ const crypto = require('node:crypto'); const fs = require('node:fs'); const path = require('node:path'); -const { spawn, spawnSync } = require('node:child_process'); +const { spawnSync } = require('node:child_process'); const { execute: storeExecute, stable } = require('./durable-host-store'); +const lineageContract = require('./authorization-lineage'); function environmentRuntimeHelper(name) { const candidates = [ @@ -19,8 +20,12 @@ function environmentRuntimeHelper(name) { } const { - pathIdentity, processAlive, processStartIdentity, removeOwnedDirectory, samePathIdentity, sleep, + minimalEnvironment, pathIdentity, releaseWorkerEnvironment, + releaseWorkerEnvironmentLease, removeOwnedDirectory, samePathIdentity, sleep, + verifyWorkerEnvironment, verifyWorkerEnvironmentLease, } = environmentRuntimeHelper('common'); +const { validateTargetExecutionBoundary } = environmentRuntimeHelper('target-execution-boundary'); +const { startOrRecoverSupervisedProcess } = environmentRuntimeHelper('supervised-process'); const { listenerOwners, listenersOwnedByProcessGroup, listenersReleased, processGroupState, terminateProcessGroup, } = environmentRuntimeHelper('platform'); @@ -33,6 +38,13 @@ function sha256(value) { return crypto.createHash('sha256').update(String(value)).digest('hex'); } +function childProcessEnvironment(cwd) { + return minimalEnvironment({}, { + schema: 'generic-host.worker-isolation.v1', + cwd_sha256: sha256(path.resolve(cwd)), + }); +} + function durableRoot() { const value = process.env.FKST_GENERIC_HOST_DURABLE_ROOT || process.env.FKST_DURABLE_ROOT; if (typeof value !== 'string' || !path.isAbsolute(value)) fail('generic Host durable root must be absolute'); @@ -128,9 +140,35 @@ function loadConfig(projectRoot, runId) { || config.run_id !== runId || path.resolve(config.project_root) !== path.resolve(projectRoot)) { fail('durable run config is unavailable or foreign'); } + validateTargetExecutionBoundary(config.target_execution_boundary, config.profile.repository); return config; } +function expectedProfileReplayBinding(config) { + const repository = config.profile && config.profile.repository; + const approval = config.approval; + const receipt = config.validation_receipt; + if (!validRepository(repository) || !approval || !receipt + || !sameRepository(repository, approval.repository) + || approval.approval_id !== receipt.approval_id + || !validDigest(receipt.approval_sha256) + || !validDigest(receipt.profile_sha256) + || approval.max_uses !== 1 + || approval.trace_id !== receipt.trace_id + || approval.dedup_key !== receipt.dedup_key) { + fail('trusted Profile replay binding is unavailable'); + } + return { + approval_id: approval.approval_id, + approval_sha256: receipt.approval_sha256, + profile_sha256: receipt.profile_sha256, + repository: { url: repository.url, commit_sha: repository.commit_sha }, + trace_id: receipt.trace_id, + dedup_key: receipt.dedup_key, + max_uses: approval.max_uses, + }; +} + function artifactFile(projectRoot, logicalPath) { if (!safeArtifactPath(logicalPath)) fail('artifact path is invalid'); const target = path.resolve(projectRoot, logicalPath); @@ -152,20 +190,168 @@ function atomicWrite(filePath, body) { fs.renameSync(temporary, filePath); } -function artifactRead(projectRoot, logicalPath) { +function stableDirectoryIdentity(boundary, directory) { + const root = path.resolve(boundary); + const target = path.resolve(directory); + if (target !== root && !target.startsWith(`${root}${path.sep}`)) { + fail('materialized artifact directory escaped its boundary'); + } + let cursor = root; + const segments = path.relative(root, target).split(path.sep).filter(Boolean); + for (const segment of ['', ...segments]) { + if (segment !== '') cursor = path.join(cursor, segment); + try { + fs.mkdirSync(cursor, { mode: 0o700 }); + } catch (error) { + if (!error || error.code !== 'EEXIST') throw error; + } + const stat = fs.lstatSync(cursor); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + fail('materialized artifact directory is not a physical directory'); + } + } + const stat = fs.lstatSync(target); + return { dev: stat.dev, ino: stat.ino }; +} + +function sameDirectoryIdentity(left, right) { + return left.dev === right.dev && left.ino === right.ino; +} + +function withAnchoredMaterializedDirectory(boundary, directory, operation) { + const expected = stableDirectoryIdentity(boundary, directory); + const original = process.cwd(); + let anchored = false; + try { + process.chdir(directory); + anchored = true; + if (!sameDirectoryIdentity(expected, fs.statSync('.'))) { + fail('materialized artifact directory identity changed'); + } + const result = operation(); + if (!sameDirectoryIdentity(expected, fs.statSync('.'))) { + fail('materialized artifact directory identity changed'); + } + return result; + } finally { + if (anchored) process.chdir(original); + } +} + +function readAnchoredPhysicalFile(name) { + if (path.basename(name) !== name || name === '.' || name === '..') { + fail('materialized artifact filename is invalid'); + } + if (typeof fs.constants.O_NOFOLLOW !== 'number') { + fail('no-follow materialized artifact reads are unsupported'); + } + let fd; + try { + fd = fs.openSync(name, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + } catch (error) { + if (error && error.code === 'ENOENT') return null; + throw error; + } + try { + const before = fs.fstatSync(fd); + if (!before.isFile() || before.isSymbolicLink()) { + fail('materialized artifact is not a physical file'); + } + const body = fs.readFileSync(fd, 'utf8'); + const after = fs.fstatSync(fd); + if (before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size) { + fail('materialized artifact identity changed while reading'); + } + return body; + } finally { + fs.closeSync(fd); + } +} + +function verifyMaterializedImmutable(filePath, body, boundary = path.dirname(filePath)) { + return withAnchoredMaterializedDirectory(boundary, path.dirname(filePath), () => { + const materialized = readAnchoredPhysicalFile(path.basename(filePath)); + if (materialized === null) return false; + if (materialized !== body) fail('materialized artifact differs'); + return true; + }); +} + +function materializeImmutableNoReplace(filePath, body, boundary = path.dirname(filePath)) { + const directory = path.dirname(filePath); + const name = path.basename(filePath); + return withAnchoredMaterializedDirectory(boundary, directory, () => { + const temporary = `.${name}.tmp-${process.pid}-${crypto.randomBytes(8).toString('hex')}`; + const fd = fs.openSync(temporary, 'wx', 0o600); + try { + fs.writeFileSync(fd, body, 'utf8'); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + try { + fs.linkSync(temporary, name); + } catch (error) { + if (!error || error.code !== 'EEXIST') throw error; + const materialized = readAnchoredPhysicalFile(name); + if (materialized !== body) fail('materialized artifact differs'); + return false; + } finally { + fs.unlinkSync(temporary); + } + return true; + }); +} + +function generatedArtifactDigestPath(config, logicalPath) { + const request = config && config.request; + const design = request && request.design_module_start; + const execution = request && request.structured_execution; + const allowed = []; + if (design && typeof design.artifact_root === 'string') { + allowed.push(`${design.artifact_root}/test-plan.json`); + } + if (execution && typeof execution.structured_plan_ref === 'string') { + allowed.push(execution.structured_plan_ref); + } + return allowed.includes(logicalPath); +} + +function artifactRead(projectRoot, logicalPath, expectedDigest, options = {}) { const runId = runIdFromPath(logicalPath); if (!runId) fail('artifact path has no run id'); - loadConfig(projectRoot, runId); + const config = loadConfig(projectRoot, runId); let result = storeExecute({ root: runRoot(runId), operation: 'artifact-read', path: logicalPath }); if (!result.found) { + if (options.durableOnly === true) return null; + const unboundDigestImport = options.allowGeneratedDigestImport === true + && generatedArtifactDigestPath(config, logicalPath); + if (!validDigest(expectedDigest) && !unboundDigestImport) { + fail(`unbound materialized artifact import is denied: ${logicalPath}`); + } const target = artifactFile(projectRoot, logicalPath); - if (!fs.existsSync(target)) return null; - const body = fs.readFileSync(target, 'utf8'); - storeExecute({ root: runRoot(runId), operation: 'artifact-write', path: logicalPath, body }); - result = { found: true, body, digest: sha256(body) }; + const boundary = path.resolve(projectRoot, '.testing'); + const body = withAnchoredMaterializedDirectory(boundary, path.dirname(target), () => + readAnchoredPhysicalFile(path.basename(target))); + if (body === null) return null; + const observedDigest = sha256(body); + if (expectedDigest && observedDigest !== expectedDigest) { + fail('materialized artifact import digest differs'); + } + const imported = storeExecute({ + root: runRoot(runId), operation: 'artifact-write', path: logicalPath, body, + }); + if (!imported.written || imported.digest !== observedDigest) { + fail('materialized artifact import differs'); + } + result = { found: true, body, digest: imported.digest }; } + if (expectedDigest && result.digest !== expectedDigest) fail('artifact digest binding differs'); const target = artifactFile(projectRoot, logicalPath); - if (!fs.existsSync(target)) atomicWrite(target, result.body); + const boundary = path.resolve(projectRoot, '.testing'); + if (!verifyMaterializedImmutable(target, result.body, boundary)) { + materializeImmutableNoReplace(target, result.body, boundary); + } let value; try { value = JSON.parse(result.body); } catch (_error) { value = result.body; } return { value, raw: result.body, digest: result.digest }; @@ -179,10 +365,9 @@ function artifactWrite(projectRoot, logicalPath, value) { const result = storeExecute({ root: runRoot(runId), operation: 'artifact-write', path: logicalPath, body }); if (!result.written) fail('immutable artifact differs'); const target = artifactFile(projectRoot, logicalPath); - if (fs.existsSync(target)) { - if (fs.readFileSync(target, 'utf8') !== body) fail('materialized artifact differs'); - } else { - atomicWrite(target, body); + const boundary = path.resolve(projectRoot, '.testing'); + if (!verifyMaterializedImmutable(target, body, boundary)) { + materializeImmutableNoReplace(target, body, boundary); } return { written: true, replayed: result.replayed === true, digest: result.digest }; } @@ -194,14 +379,435 @@ function artifactWriteRaw(projectRoot, logicalPath, body) { const result = storeExecute({ root: runRoot(runId), operation: 'artifact-write', path: logicalPath, body }); if (!result.written) fail('immutable artifact differs'); const target = artifactFile(projectRoot, logicalPath); - if (fs.existsSync(target)) { - if (fs.readFileSync(target, 'utf8') !== body) fail('materialized artifact differs'); - } else { - atomicWrite(target, body); + const boundary = path.resolve(projectRoot, '.testing'); + if (!verifyMaterializedImmutable(target, body, boundary)) { + materializeImmutableNoReplace(target, body, boundary); } return { written: true, replayed: result.replayed === true, digest: result.digest }; } +const lineagePaths = lineageContract.paths; +const lineageSchemas = lineageContract.schemas; + +function lineageRoot(config) { + const expected = `.testing/runs/${config.run_id}`; + if (config.artifact_root !== expected) fail('authorization lineage requires the canonical run root'); + return expected; +} + +function lineagePath(config, name) { + if (!lineagePaths[name]) fail(`unsupported authorization lineage receipt ${name}`); + return `${lineageRoot(config)}/${lineagePaths[name]}`; +} + +function claimFingerprint(config, domain, privateClaimId) { + if (typeof privateClaimId !== 'string' || privateClaimId === '') fail('authorization claim id is unavailable'); + if (typeof config.lineage_projection_secret !== 'string' + || config.lineage_projection_secret.length < 32) fail('lineage projection secret is unavailable'); + const inner = sha256(`${config.lineage_projection_secret}\0${domain}\0${privateClaimId}`); + return sha256(`fkst-authorization-lineage.v1\0${domain}\0${inner}`); +} + +function lineageEnvelope(config, schema, status, receiptId, recordedAt, fields) { + return { + schema, status, receipt_id: receiptId, + repository: { url: config.repository.url, commit_sha: config.repository.commit_sha }, + run_id: config.run_id, trace_id: config.request.trace_id, dedup_key: config.request.dedup_key, + recorded_at: recordedAt, source_max_uses: 1, evidence_role: 'audit-only', + authorization_capability: false, reusable: false, ...fields, + }; +} + +function writeLineageReceipt(projectRoot, config, name, value, expected) { + lineageContract.validateReceipt(name, value, expected); + const logicalPath = lineagePath(config, name); + const body = stable(value); + const written = artifactWriteRaw(projectRoot, logicalPath, body); + const persisted = artifactRead(projectRoot, logicalPath); + if (!persisted || persisted.raw !== body || persisted.digest !== sha256(body) + || stable(persisted.value) !== body) fail(`authorization lineage receipt differs: ${name}`); + lineageContract.validateReceipt(name, persisted.value, expected); + return { ref: logicalPath, sha256: persisted.digest, value: persisted.value, + expected, replayed: written.replayed === true }; +} + +function boundLineageArtifact(projectRoot, ref, expectedDigest, label) { + const artifact = artifactRead(projectRoot, ref, expectedDigest); + if (!artifact || (expectedDigest && artifact.digest !== expectedDigest)) { + fail(`${label} artifact binding differs`); + } + return artifact; +} + +function lineageSource(config, fields) { + return { + repository: { url: config.repository.url, commit_sha: config.repository.commit_sha }, + run_id: config.run_id, + trace_id: config.request.trace_id, + dedup_key: config.request.dedup_key, + ...fields, + }; +} + +function legacyPreauthorizationBinding(request) { + return { + authorization_id: request.authorization_id, + preauthorization_sha256: request.preauthorization_sha256, + repository: request.repository, + plan_sha256: request.plan_sha256, + environment_receipt_sha256: request.environment_receipt_sha256, + trace_id: request.trace_id, + dedup_key: request.dedup_key, + }; +} + +function canonicalPreauthorizationBinding(request) { + return { + authorization_id: request.authorization_id, + preauthorization_ref: request.preauthorization_ref, + preauthorization_sha256: request.preauthorization_sha256, + repository: request.repository, + plan_ref: request.plan_ref, + plan_sha256: request.plan_sha256, + environment_receipt_ref: request.environment_receipt_ref, + environment_receipt_sha256: request.environment_receipt_sha256, + trace_id: request.trace_id, + dedup_key: request.dedup_key, + }; +} + +function trustedPreauthorizationRefs(config) { + const request = config && config.request; + const structured = request && request.structured_execution; + const environment = request && request.environment_start; + if (!structured || !environment || typeof structured.preauthorization_ref !== 'string' + || typeof structured.structured_plan_ref !== 'string' + || typeof environment.artifact_root !== 'string') return null; + return { + preauthorization_ref: structured.preauthorization_ref, + plan_ref: structured.structured_plan_ref, + environment_receipt_ref: `${environment.artifact_root}/environment-receipt-ready.json`, + }; +} + +function compatibleStoredPreauthorizationBinding(stored) { + if (!stored || typeof stored !== 'object' || Array.isArray(stored) + || !Object.prototype.hasOwnProperty.call(stored, 'runtime_config_ref')) return stored; + if (stable(stored.runtime_config_ref) !== stable({ + kind: 'artifact', ref: '.testing/generic-host-runtime.json', + })) return stored; + const normalized = { ...stored }; + delete normalized.runtime_config_ref; + return normalized; +} + +function preauthorizationBindingMatches(stored, request, trustedRefs) { + if (!trustedRefs || request.preauthorization_ref !== trustedRefs.preauthorization_ref + || request.plan_ref !== trustedRefs.plan_ref + || request.environment_receipt_ref !== trustedRefs.environment_receipt_ref) return false; + const normalized = compatibleStoredPreauthorizationBinding(stored); + return stable(normalized) === stable(canonicalPreauthorizationBinding(request)) + || stable(normalized) === stable(legacyPreauthorizationBinding(request)); +} + +function completePreauthorizationRequest(preauthorization, request) { + return { + authorization_id: preauthorization.value.authorization_id, + preauthorization_ref: request.preauthorization_ref, + preauthorization_sha256: preauthorization.digest, + repository: request.repository, + plan_ref: request.plan_ref, + plan_sha256: request.plan_sha256, + environment_receipt_ref: request.environment_receipt_ref, + environment_receipt_sha256: request.environment_receipt_sha256, + trace_id: request.trace_id, + dedup_key: request.dedup_key, + }; +} + +function durableProfileClaim(config) { + const claim = recordRead(runRoot(config.run_id), `generic-host/profile-approval/${config.run_id}`); + if (!claim || typeof claim.claim_id !== 'string') fail('durable Profile claim is unavailable'); + return claim; +} + +function durablePreauthorizationClaim(config, authorizationId) { + const claim = recordRead(runRoot(config.run_id), + `generic-host/preauthorization/${sha256(stable(authorizationId))}`); + if (!claim || typeof claim.claim_id !== 'string') fail('durable Preauthorization claim is unavailable'); + return claim; +} + +function durableGrantVerification(config, grantDigest) { + const verification = recordRead(runRoot(config.run_id), + `testing-runner/grant-verifications/${sha256(grantDigest)}`); + if (!verification || !verification.binding) fail('durable Grant verification is unavailable'); + return verification; +} + +function durableExecutionClaim(config) { + const entries = recordList(runRoot(config.run_id), 'testing-runner/replay'); + if (entries.length !== 1 || !entries[0].value || typeof entries[0].value.claim_id !== 'string') { + fail('durable execution claim is unavailable'); + } + return entries[0].value; +} + +function writeProfileClaimReceipt(projectRoot, config, durableClaim) { + const claimedAt = durableClaim && (durableClaim.claimed_at || config.authorization_now); + if (!durableClaim + || stable(durableClaim.binding) !== stable(expectedProfileReplayBinding(config)) + || typeof durableClaim.claim_id !== 'string' || durableClaim.claim_id === '' + || typeof claimedAt !== 'string' || claimedAt === '') { + fail('environment authorization approval claim is unavailable'); + } + const start = config.request.environment_start; + const profile = boundLineageArtifact(projectRoot, start.profile_ref.ref, null, 'profile'); + const approval = boundLineageArtifact(projectRoot, start.approval_ref.ref, null, 'profile approval'); + const validation = boundLineageArtifact(projectRoot, start.validation_receipt_ref.ref, null, 'profile validation'); + if (profile.value.revision !== validation.value.profile_revision + || approval.value.approval_id !== validation.value.approval_id + || validation.value.profile_sha256 !== config.validation_receipt.profile_sha256 + || validation.value.approval_sha256 !== config.validation_receipt.approval_sha256) { + fail('profile claim source artifacts differ'); + } + const fingerprint = claimFingerprint(config, 'project-profile-approval-claim', durableClaim.claim_id); + const source = lineageSource(config, { + profile_source_ref: config.profile_source_ref, + profile_artifact_ref: start.profile_ref.ref, profile_artifact_sha256: profile.digest, + profile_sha256: validation.value.profile_sha256, profile_revision: profile.value.revision, + approval_artifact_ref: start.approval_ref.ref, approval_artifact_sha256: approval.digest, + approval_id: approval.value.approval_id, approval_sha256: validation.value.approval_sha256, + approval_authority: approval.value.authority, policy_revision: approval.value.policy_revision, + evidence_ref: approval.value.evidence_ref, + validation_receipt_ref: start.validation_receipt_ref.ref, + validation_receipt_sha256: validation.digest, + claim_fingerprint_sha256: fingerprint, claimed_at: claimedAt, + }); + const value = lineageEnvelope(config, lineageSchemas.profile_claim, 'claimed', + `profile-claim-${fingerprint.slice(0, 32)}`, claimedAt, source); + return writeLineageReceipt(projectRoot, config, 'profile_claim', value, source); +} + +function writePreauthorizationClaimReceipt(projectRoot, config, request, durableClaim) { + const claimedAt = durableClaim.claimed_at || config.execution_authorization_now; + if (!preauthorizationBindingMatches( + durableClaim.binding, request, trustedPreauthorizationRefs(config), + )) { + fail('durable Preauthorization claim binding differs'); + } + const profileReceipt = writeProfileClaimReceipt(projectRoot, config, durableProfileClaim(config)); + const preauthorization = boundLineageArtifact(projectRoot, request.preauthorization_ref, + request.preauthorization_sha256, 'preauthorization'); + const catalogRef = config.request.structured_execution.case_catalog_ref; + const catalog = boundLineageArtifact(projectRoot, catalogRef, + config.request.structured_execution.case_catalog_sha256, 'case catalog'); + const plan = boundLineageArtifact(projectRoot, request.plan_ref, request.plan_sha256, 'structured plan'); + const environment = boundLineageArtifact(projectRoot, request.environment_receipt_ref, + request.environment_receipt_sha256, 'environment receipt'); + if (preauthorization.value.authorization_id !== request.authorization_id + || preauthorization.value.profile_sha256 !== profileReceipt.value.profile_sha256 + || preauthorization.value.case_catalog_sha256 !== catalog.digest + || plan.value.environment_receipt_sha256 !== environment.digest) { + fail('preauthorization lineage source differs'); + } + const fingerprint = claimFingerprint(config, 'structured-preauthorization-claim', durableClaim.claim_id); + const source = lineageSource(config, { + profile_claim_receipt_ref: profileReceipt.ref, + profile_claim_receipt_sha256: profileReceipt.sha256, + preauthorization_ref: request.preauthorization_ref, + preauthorization_sha256: preauthorization.digest, + authorization_id: preauthorization.value.authorization_id, + profile_sha256: preauthorization.value.profile_sha256, + case_catalog_ref: catalogRef, case_catalog_sha256: catalog.digest, + plan_ref: request.plan_ref, plan_sha256: plan.digest, + environment_receipt_ref: request.environment_receipt_ref, + environment_receipt_sha256: environment.digest, + authority: preauthorization.value.authority, + policy_revision: preauthorization.value.policy_revision, + evidence_ref: preauthorization.value.evidence_ref, + claim_fingerprint_sha256: fingerprint, claimed_at: claimedAt, + }); + const value = lineageEnvelope(config, lineageSchemas.preauthorization_claim, 'claimed', + `preauthorization-claim-${fingerprint.slice(0, 32)}`, claimedAt, source); + return writeLineageReceipt(projectRoot, config, 'preauthorization_claim', value, source); +} + +function writeGrantVerificationReceipt(projectRoot, config, request) { + const preauthorization = boundLineageArtifact(projectRoot, request.preauthorization_ref, + request.preauthorization_sha256, 'preauthorization'); + const grant = boundLineageArtifact(projectRoot, request.grant_ref, request.grant_sha256, 'execution grant'); + const plan = boundLineageArtifact(projectRoot, request.plan_ref, request.plan_sha256, 'structured plan'); + const environment = boundLineageArtifact(projectRoot, request.environment_receipt_ref, + request.environment_receipt_sha256, 'environment receipt'); + if (grant.value.parent_authorization_sha256 !== preauthorization.digest + || grant.value.plan_sha256 !== plan.digest + || grant.value.environment_receipt_sha256 !== environment.digest + || stable(grant.value.repository) !== stable(request.repository)) { + fail('grant verification source differs'); + } + assertStructuredGrantDerivation(config, request, preauthorization, plan, environment, grant); + const preauthorizationRequest = completePreauthorizationRequest(preauthorization, request); + const preauthorizationReceipt = writePreauthorizationClaimReceipt(projectRoot, config, + preauthorizationRequest, + durablePreauthorizationClaim(config, preauthorization.value.authorization_id)); + const verificationId = `grant-verification-${grant.digest.slice(0, 32)}`; + const root = runRoot(config.run_id); + const durable = recordImmutable(root, `testing-runner/grant-verifications/${sha256(grant.digest)}`, { + binding: { + grant_ref: request.grant_ref, grant_sha256: grant.digest, + preauthorization_ref: request.preauthorization_ref, preauthorization_sha256: preauthorization.digest, + plan_ref: request.plan_ref, plan_sha256: plan.digest, + environment_receipt_ref: request.environment_receipt_ref, + environment_receipt_sha256: environment.digest, repository: request.repository, + trace_id: request.trace_id, dedup_key: request.dedup_key, + }, + verification_id: verificationId, verified_at: config.execution_authorization_now, + }); + if (!durable.written && !durable.replayed) fail('grant verification durable record differs'); + const source = lineageSource(config, { + preauthorization_claim_receipt_ref: preauthorizationReceipt.ref, + preauthorization_claim_receipt_sha256: preauthorizationReceipt.sha256, + grant_ref: request.grant_ref, grant_sha256: grant.digest, grant_id: grant.value.grant_id, + parent_authorization_ref: request.preauthorization_ref, + parent_authorization_sha256: preauthorization.digest, + plan_ref: request.plan_ref, plan_sha256: plan.digest, + environment_receipt_ref: request.environment_receipt_ref, + environment_receipt_sha256: environment.digest, + authority: grant.value.authority, policy_revision: grant.value.policy_revision, + evidence_ref: grant.value.evidence_ref, verifier_ref: config.grant_verifier_ref, + verification_id: verificationId, verified_at: durable.value.verified_at, + }); + const value = lineageEnvelope(config, lineageSchemas.grant_verification, 'authenticated', + verificationId, durable.value.verified_at, source); + return writeLineageReceipt(projectRoot, config, 'grant_verification', value, source); +} + +function writeExecutionClaimReceipt(projectRoot, config, request, durableClaim) { + const claimedAt = durableClaim && (durableClaim.claimed_at || config.execution_authorization_now); + if (!durableClaim || stable(durableClaim.binding) !== stable(request) + || typeof durableClaim.claim_id !== 'string' || typeof claimedAt !== 'string' + || durableClaim.fence_id !== claimFingerprint( + config, 'structured-execution-fence', durableClaim.claim_id, + )) { + fail('execution claim durable binding differs'); + } + const verification = durableGrantVerification(config, request.grant_sha256); + const grantReceipt = writeGrantVerificationReceipt(projectRoot, config, verification.binding); + const preauthorization = boundLineageArtifact(projectRoot, request.preauthorization_ref, + request.preauthorization_sha256, 'preauthorization'); + const preauthorizationRequest = completePreauthorizationRequest(preauthorization, verification.binding); + const preauthorizationReceipt = writePreauthorizationClaimReceipt(projectRoot, config, + preauthorizationRequest, + durablePreauthorizationClaim(config, preauthorization.value.authorization_id)); + const fingerprint = claimFingerprint(config, 'structured-execution-claim', durableClaim.claim_id); + const source = lineageSource(config, { + grant_verification_receipt_ref: grantReceipt.ref, + grant_verification_receipt_sha256: grantReceipt.sha256, + preauthorization_claim_receipt_ref: preauthorizationReceipt.ref, + preauthorization_claim_receipt_sha256: preauthorizationReceipt.sha256, + grant_ref: request.grant_ref, grant_sha256: request.grant_sha256, + grant_id: request.grant_id, plan_ref: request.plan_ref, plan_sha256: request.plan_sha256, + environment_receipt_ref: request.environment_receipt_ref, + environment_receipt_sha256: request.environment_receipt_sha256, + artifact_root: request.artifact_root, operation_id: request.operation_id, + claim_fingerprint_sha256: fingerprint, claimed_at: claimedAt, + }); + const value = lineageEnvelope(config, lineageSchemas.execution_claim, 'claimed', + `execution-claim-${fingerprint.slice(0, 32)}`, claimedAt, source); + return writeLineageReceipt(projectRoot, config, 'execution_claim', value, source); +} + +function assertExecutionMatchesClaim(execution, claimBinding) { + if (!execution || typeof execution !== 'object' || Array.isArray(execution) + || !claimBinding || typeof claimBinding !== 'object' || Array.isArray(claimBinding) + || !validDigest(execution.plan_sha256) || !validDigest(claimBinding.plan_sha256) + || execution.plan_sha256 !== claimBinding.plan_sha256) { + fail('completed execution Plan binding differs from the durable claim'); + } + return true; +} + +function writeExecutionCompletionReceipt(projectRoot, config, durableClaim) { + const completedAt = durableClaim && durableClaim.completion + && (durableClaim.completion.completed_at || config.execution_authorization_now); + if (!durableClaim || durableClaim.status !== 'completed' || !durableClaim.completion + || typeof completedAt !== 'string') { + fail('execution completion durable binding differs'); + } + const executionClaim = writeExecutionClaimReceipt(projectRoot, config, + durableClaim.binding, durableClaim); + const completion = durableClaim.completion; + const artifacts = structuredExecutionArtifacts(projectRoot, completion.result_ref, + completion.result_sha256); + if (!artifacts.caseResultSet || !artifacts.evidenceManifest + || artifacts.execution.digest !== completion.result_sha256) { + fail('canonical execution completion artifacts are unavailable'); + } + assertExecutionMatchesClaim(artifacts.execution.value, durableClaim.binding); + const source = lineageSource(config, { + execution_claim_receipt_ref: executionClaim.ref, + execution_claim_receipt_sha256: executionClaim.sha256, + result_ref: completion.result_ref, result_sha256: artifacts.execution.digest, + case_result_set_ref: artifacts.execution.value.case_result_set_path, + case_result_set_artifact_sha256: artifacts.caseResultSet.digest, + evidence_manifest_ref: artifacts.execution.value.evidence_manifest_path, + evidence_manifest_artifact_sha256: artifacts.evidenceManifest.digest, + completed_at: completedAt, + }); + const value = lineageEnvelope(config, lineageSchemas.execution_completion, 'completed', + `execution-completion-${artifacts.execution.digest.slice(0, 32)}`, completedAt, source); + const receipt = writeLineageReceipt(projectRoot, config, 'execution_completion', value, source); + writeLineageIndex(projectRoot, config, completedAt, receipt); + return receipt; +} + +function writeLineageIndex(projectRoot, config, recordedAt, completionReceipt) { + const preauthorization = boundLineageArtifact(projectRoot, + config.request.structured_execution.preauthorization_ref, + config.request.structured_execution.preauthorization_sha256, 'preauthorization'); + const grant = boundLineageArtifact(projectRoot, config.request.structured_execution.grant_ref, + null, 'execution grant'); + const grantVerification = durableGrantVerification(config, grant.digest); + const preauthorizationRequest = completePreauthorizationRequest( + preauthorization, grantVerification.binding); + const executionClaim = durableExecutionClaim(config); + if (executionClaim.status !== 'completed') fail('durable completed execution claim is unavailable'); + const artifacts = { + profile_claim: writeProfileClaimReceipt(projectRoot, config, durableProfileClaim(config)), + preauthorization_claim: writePreauthorizationClaimReceipt(projectRoot, config, + preauthorizationRequest, + durablePreauthorizationClaim(config, preauthorization.value.authorization_id)), + grant_verification: writeGrantVerificationReceipt(projectRoot, config, grantVerification.binding), + execution_claim: writeExecutionClaimReceipt(projectRoot, config, + executionClaim.binding, executionClaim), + execution_completion: completionReceipt, + }; + const expected = {}; + const receipts = {}; + for (const name of lineageContract.receiptNames) { + expected[name] = artifacts[name].expected; + receipts[name] = { ref: artifacts[name].ref, sha256: artifacts[name].sha256 }; + } + const value = { + schema: lineageSchemas.lineage_index, status: 'complete', + repository: { url: config.repository.url, commit_sha: config.repository.commit_sha }, + run_id: config.run_id, trace_id: config.request.trace_id, dedup_key: config.request.dedup_key, + recorded_at: recordedAt, receipts, lineage_complete: true, source_max_uses: 1, + evidence_role: 'audit-only', authorization_capability: false, reusable: false, + }; + lineageContract.validateLineageIndex(value, artifacts, expected); + const path = `${lineageRoot(config)}/authorization-lineage/index.json`; + const body = stable(value); + artifactWriteRaw(projectRoot, path, body); + const persisted = artifactRead(projectRoot, path); + if (!persisted || persisted.raw !== body || persisted.digest !== sha256(body)) { + fail('authorization lineage index differs'); + } + lineageContract.validateLineageIndex(persisted.value, artifacts, expected); + return { ref: path, sha256: persisted.digest, value: persisted.value }; +} + function listIndexedRuns(projectRoot) { const runs = []; for (const entry of recordList(hostRoot(), 'runs')) { @@ -297,19 +903,25 @@ function directExec(argv, cwd, timeoutSeconds, outputBytes) { if (!Array.isArray(argv) || argv.length === 0 || argv.some((item) => typeof item !== 'string')) { fail('argv must be a non-empty string list'); } - const options = { - cwd, - encoding: 'utf8', - timeout: Math.max(1, Number(timeoutSeconds) || 30) * 1000, - env: process.env, - }; - if (Number.isInteger(outputBytes) && outputBytes >= 1024) options.maxBuffer = outputBytes; - const result = spawnSync(argv[0], argv.slice(1), options); - return { - exit_code: result.status == null ? -1 : result.status, - stdout: result.stdout || '', - stderr: result.stderr || (result.error ? String(result.error.message || result.error) : ''), - }; + const environment = childProcessEnvironment(cwd); + try { + verifyWorkerEnvironment(environment); + const options = { + cwd, + encoding: 'utf8', + timeout: Math.max(1, Number(timeoutSeconds) || 30) * 1000, + env: environment, + }; + if (Number.isInteger(outputBytes) && outputBytes >= 1024) options.maxBuffer = outputBytes; + const result = spawnSync(argv[0], argv.slice(1), options); + return { + exit_code: result.status == null ? -1 : result.status, + stdout: result.stdout || '', + stderr: result.stderr || (result.error ? String(result.error.message || result.error) : ''), + }; + } finally { + releaseWorkerEnvironment(environment); + } } function environmentStateKey(ref) { @@ -402,51 +1014,77 @@ function startApplication(projectRoot, payload) { || payload.argv.length === 0 || !samePorts(ports, [{ name: 'application', port: config.port }])) { fail('application start binding differs'); } + const workspace = workspaceResource(root, payload.workspace_ref); + const workspaceState = verifyWorkspace(config, workspace); + if (!workspaceState.owned) fail(`workspace ownership failed: ${workspaceState.reason}`); + const binding = { + schema: 'generic-host.environment-resource.v1', kind: 'process', operation_id: runId, + effect_id: payload.effect_id, cleanup_ref: payload.cleanup_ref, workspace_ref: payload.workspace_ref, + workspace_path: workspace.path, workspace_identity: workspace.path_identity, + argv_sha256: sha256(stable(payload.argv)), + ownership_token: sha256(stable({ + schema: 'generic-host.process-ownership.v1', run_id: runId, + effect_id: payload.effect_id, cleanup_ref: payload.cleanup_ref, + })), + runtime_ports: ports, repository: config.profile.repository, + }; const existing = recordRead(root, environmentResourceKey(payload.cleanup_ref)); if (existing) { - const status = inspectResources(projectRoot, { run_id: runId }); - if (!status.owned) fail(`application replay ownership failed: ${status.reason}`); + const volatile = new Set([ + 'startup_state', 'startup_token_sha256', 'pid', 'pgid', + 'process_start_identity', 'worker_environment_lease', + ]); + const existingBinding = Object.fromEntries( + Object.entries(existing).filter(([key]) => !volatile.has(key)), + ); + if (stable(existingBinding) !== stable(binding)) fail('application replay binding differs'); + try { + verifyWorkerEnvironmentLease(existing.worker_environment_lease); + } catch (_error) { + fail('application replay worker environment binding changed'); + } + const group = processGroupState(existing); + if (!group.supported || group.foreign) fail('application replay process ownership cannot be verified'); + if (!group.alive) { + return { status: 'blocked', cleanup_ref: payload.cleanup_ref, early_exit: true, runtime_ports: ports }; + } + const listenerState = listenersOwnedByProcessGroup(ports, existing.pgid); + if (!listenerState.supported || !listenerState.owned) { + fail(`application replay listener ownership failed: ${listenerState.reason}`); + } return { status: 'running', cleanup_ref: payload.cleanup_ref, early_exit: false, runtime_ports: ports }; } - const workspace = workspaceResource(root, payload.workspace_ref); - const workspaceState = verifyWorkspace(config, workspace); - if (!workspaceState.owned) fail(`workspace ownership failed: ${workspaceState.reason}`); - const child = spawn(payload.argv[0], payload.argv.slice(1), { - cwd: workspace.path, - env: process.env, - shell: false, - detached: true, - stdio: 'ignore', + const startupClaimPath = path.join(root, 'private', 'supervised-process-startup.json'); + const launch = startOrRecoverSupervisedProcess({ + claimPath: startupClaimPath, + argv: payload.argv, + cwd: workspace.path, + createEnvironment: () => childProcessEnvironment(workspace.path), + binding, }); - child.once('error', () => {}); - child.unref(); + if (launch.interrupted || !launch.resource) fail('application startup was interrupted before registration'); + const resource = launch.resource; + if (launch.state !== 'running') { + const stored = recordImmutable(root, environmentResourceKey(payload.cleanup_ref), resource); + if (!stored.written && !stored.replayed) fail('application resource binding differs'); + return { status: 'blocked', cleanup_ref: payload.cleanup_ref, early_exit: true, runtime_ports: ports }; + } const deadline = Date.now() + 5_000; - let processIdentity = null; let listenerState = null; while (Date.now() < deadline) { - processIdentity = processStartIdentity(child.pid); - if (processIdentity !== null) { - listenerState = listenersOwnedByProcessGroup(ports, child.pid); - if (listenerState.supported && listenerState.owned) break; - } + listenerState = listenersOwnedByProcessGroup(ports, resource.pgid); + if (listenerState.supported && listenerState.owned) break; sleep(25); } - if (processIdentity === null || !listenerState || !listenerState.supported || !listenerState.owned) { - terminateProcessGroup({ pid: child.pid, pgid: child.pid, process_start_identity: processIdentity }, 500); + if (!listenerState || !listenerState.supported || !listenerState.owned) { + terminateProcessGroup(resource, 500); + releaseWorkerEnvironmentLease(resource.worker_environment_lease); + const stored = recordImmutable(root, environmentResourceKey(payload.cleanup_ref), resource); + if (!stored.written && !stored.replayed) fail('application resource binding differs'); fail(`application ownership could not be verified: ${listenerState && listenerState.reason || 'process-start-failed'}`); } - const resource = { - schema: 'generic-host.environment-resource.v1', kind: 'process', operation_id: runId, - effect_id: payload.effect_id, cleanup_ref: payload.cleanup_ref, workspace_ref: payload.workspace_ref, - workspace_path: workspace.path, workspace_identity: workspace.path_identity, - argv_sha256: sha256(stable(payload.argv)), ownership_token: crypto.randomBytes(16).toString('hex'), - runtime_ports: ports, pid: child.pid, pgid: child.pid, process_start_identity: processIdentity, - }; const stored = recordImmutable(root, environmentResourceKey(payload.cleanup_ref), resource); - if (!stored.written) { - terminateProcessGroup(resource, 500); - fail('application resource binding differs'); - } + if (!stored.written && !stored.replayed) fail('application resource binding differs'); return { status: 'running', cleanup_ref: payload.cleanup_ref, early_exit: false, runtime_ports: ports }; } @@ -465,6 +1103,11 @@ function inspectResources(projectRoot, payload) { || typeof process.ownership_token !== 'string' || process.ownership_token === '') { return { owned: false, reason: 'process-binding-changed' }; } + try { + verifyWorkerEnvironmentLease(process.worker_environment_lease); + } catch (_error) { + return { owned: false, reason: 'worker-environment-binding-changed' }; + } const group = processGroupState(process); if (!group.supported || !group.alive || group.foreign) return { owned: false, reason: 'process-group-not-owned' }; const listeners = listenersOwnedByProcessGroup(process.runtime_ports, process.pgid); @@ -487,9 +1130,28 @@ function releasedResources(projectRoot, payload) { process_group_absent: group.supported === true && group.alive === false, listeners_closed: listeners.supported === true && listeners.released === true, workspace_absent: !fs.existsSync(config.workspace_root), + worker_environment_absent: !process.worker_environment_lease + || !fs.existsSync(process.worker_environment_lease.home), }; } +function releaseOwnedProcessResource(resource, timeoutMs) { + const group = processGroupState(resource); + if (!group.supported || group.foreign) fail('process cleanup ownership cannot be verified'); + if (group.alive) { + const owned = listenersOwnedByProcessGroup(resource.runtime_ports, resource.pgid); + if (!owned.supported || !owned.owned) fail(`process listener ownership cannot be verified: ${owned.reason}`); + const stopped = terminateProcessGroup(resource, timeoutMs); + if (!stopped.released) fail(`process cleanup failed: ${stopped.reason}`); + } + const listeners = listenersReleased(resource.runtime_ports); + if (!listeners.supported || !listeners.released) fail('process listeners remain after cleanup'); + if (!releaseWorkerEnvironmentLease(resource.worker_environment_lease)) { + fail('process worker environment cleanup failed'); + } + return true; +} + function cleanupResource(projectRoot, payload) { const runId = runIdFor(payload); const root = runRoot(runId); @@ -506,14 +1168,7 @@ function cleanupResource(projectRoot, payload) { if (!workspaceState.owned || !samePathIdentity(resource.workspace_identity, workspace.path_identity)) { fail('process cleanup workspace ownership differs'); } - const group = processGroupState(resource); - if (!group.supported || !group.alive || group.foreign) fail('process cleanup ownership cannot be verified'); - const owned = listenersOwnedByProcessGroup(resource.runtime_ports, resource.pgid); - if (!owned.supported || !owned.owned) fail(`process listener ownership cannot be verified: ${owned.reason}`); - const stopped = terminateProcessGroup(resource, Math.max(1, Number(payload.timeout_seconds) || 5) * 1000); - if (!stopped.released) fail(`process cleanup failed: ${stopped.reason}`); - const listeners = listenersReleased(resource.runtime_ports); - if (!listeners.supported || !listeners.released) fail('process listeners remain after cleanup'); + releaseOwnedProcessResource(resource, Math.max(1, Number(payload.timeout_seconds) || 5) * 1000); } else if (resource.kind === 'workspace') { const process = resourceRecord(root, { kind: 'process-cleanup', ref: `${runId}-application` }); const group = processGroupState(process); @@ -598,7 +1253,8 @@ function inventoryAcceptanceReport(projectRoot, config, terminal) { const completed = recordList(runRoot(config.run_id), 'testing-runner/replay') .filter((entry) => entry.value && entry.value.status === 'completed'); if (completed.length !== 1) fail('inventory completed replay is unavailable'); - const artifacts = structuredExecutionArtifacts(projectRoot, completed[0].value.result_ref); + const artifacts = structuredExecutionArtifacts(projectRoot, completed[0].value.result_ref, + completed[0].value.result_sha256); if (artifacts.execution.digest !== completed[0].value.result_sha256) { fail('inventory completed replay result differs'); } @@ -773,6 +1429,109 @@ function argvAllowed(argv, capabilities) { && capability.argv_prefix.every((item, index) => item === argv[index])); } +const httpMethods = new Set(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE']); + +function validHttpCapabilities(value) { + return Array.isArray(value) && value.length <= 64 && value.every((capability) => + exactKeys(capability, ['origin', 'methods', 'path_prefixes']) + && boundedString(capability.origin, 512) + && /^https?:\/\/[^\/@]+$/.test(capability.origin) + && Array.isArray(capability.methods) && capability.methods.length > 0 + && capability.methods.length <= 8 && capability.methods.every((method) => httpMethods.has(method)) + && Array.isArray(capability.path_prefixes) && capability.path_prefixes.length > 0 + && capability.path_prefixes.length <= 16 + && capability.path_prefixes.every((prefix) => boundedString(prefix, 512) + && prefix.startsWith('/') && !prefix.includes('?') && !prefix.includes('#'))); +} + +function splitHttpUrl(value) { + if (!boundedString(value, 2048) || value.includes('?') || value.includes('#')) return null; + const match = /^(https?:\/\/[^/]+)(\/.*)?$/.exec(value); + if (!match || match[1].includes('@')) return null; + return { origin: match[1], path: match[2] || '/' }; +} + +function httpAllowed(request, capabilities) { + const target = request && splitHttpUrl(request.url); + return target !== null && httpMethods.has(request.method) && validHttpCapabilities(capabilities) + && capabilities.some((capability) => capability.origin === target.origin + && capability.methods.includes(request.method) + && capability.path_prefixes.some((prefix) => target.path.startsWith(prefix))); +} + +function planWithinCapabilities(plan, capabilities) { + if (!plan || plan.schema !== 'testing-structured-plan.v2' + || plan.execution_mode !== 'structured-api-cli' + || !exactKeys(capabilities, ['cli', 'http']) + || !validCliCapabilities(capabilities.cli) || !validHttpCapabilities(capabilities.http) + || !Array.isArray(plan.cases) || plan.cases.length === 0 || plan.cases.length > 64) return false; + return plan.cases.every((planned) => planned && ( + typeof planned.skip_reason === 'string' && boundedString(planned.skip_reason, 512) + || planned.kind === 'cli' && argvAllowed(planned.argv, capabilities.cli) + || planned.kind === 'http' && httpAllowed(planned.request, capabilities.http) + )); +} + +function hostStructuredGrantValues(config) { + return { + grant_id: `${config.run_id}-grant`, + evidence_ref: { kind: 'signed-attestation', ref: `${config.run_id}-execution-grant` }, + issued_at: '2026-07-22T00:15:00Z', + expires_at: '2026-07-22T00:45:00Z', + now: '2026-07-22T00:20:00Z', + }; +} + +function assertStructuredGrantDerivation(config, request, preauthorization, plan, environment, grant) { + const authorization = preauthorization && preauthorization.value; + const structuredPlan = plan && plan.value; + const readyEnvironment = environment && environment.value; + const persistedGrant = grant && grant.value; + const values = hostStructuredGrantValues(config); + if (config.execution_authorization_now !== undefined + && config.execution_authorization_now !== values.now + || !authorization || authorization.schema !== 'testing-structured-execution-authorization.v1' + || authorization.max_uses !== 1 || !validWindow(authorization, values.now) + || !sameRepository(authorization.repository, request.repository) + || !structuredPlan || !sameRepository(structuredPlan.repository, request.repository) + || authorization.case_catalog_sha256 !== structuredPlan.case_catalog_sha256 + || structuredPlan.environment_receipt_sha256 !== environment.digest + || authorization.trace_id !== request.trace_id || authorization.dedup_key !== request.dedup_key + || structuredPlan.trace_id !== request.trace_id || structuredPlan.dedup_key !== request.dedup_key + || !readyEnvironment || readyEnvironment.status !== 'ready' + || !sameRepository(readyEnvironment.repository, request.repository) + || readyEnvironment.trace_id !== request.trace_id || readyEnvironment.dedup_key !== request.dedup_key + || !planWithinCapabilities(structuredPlan, authorization.capabilities)) { + fail('grant derivation source differs'); + } + const expected = { + schema: 'testing-structured-execution-grant.v1', + grant_id: values.grant_id, + parent_authorization_sha256: preauthorization.digest, + plan_sha256: plan.digest, + environment_receipt_sha256: environment.digest, + repository: structuredPlan.repository, + cli_capabilities: authorization.capabilities.cli, + http_capabilities: authorization.capabilities.http, + authority: authorization.authority, + policy_revision: authorization.policy_revision, + evidence_ref: values.evidence_ref, + issued_at: values.issued_at, + expires_at: values.expires_at, + max_uses: 1, + trace_id: request.trace_id, + dedup_key: request.dedup_key, + }; + if (!persistedGrant || stable(persistedGrant) !== stable(expected) + || request.grant !== undefined && stable(request.grant) !== stable(persistedGrant) + || request.grant_raw !== undefined && (typeof request.grant_raw !== 'string' + || sha256(request.grant_raw) !== grant.digest) + || request.now !== undefined && request.now !== values.now) { + fail('grant differs from authenticated derivation'); + } + return expected; +} + function validWindow(value, now) { const issued = Date.parse(value && value.issued_at); const expires = Date.parse(value && value.expires_at); @@ -834,7 +1593,7 @@ function activeStructuredRequest(root, runId) { } function structuredCaseSequence(projectRoot, request, caseId) { - const plan = request && artifactRead(projectRoot, request.test_plan_ref); + const plan = request && artifactRead(projectRoot, request.test_plan_ref, request.test_plan_sha256); const cases = plan && plan.value && plan.value.cases; if (!Array.isArray(cases)) fail('structured test plan is unavailable'); const index = cases.findIndex((value) => value && value.case_id === caseId); @@ -895,12 +1654,15 @@ function authorizeCliEffect(projectRoot, payload) { if (!validCliEnvelope(envelope)) return deny('malformed-envelope'); const request = activeStructuredRequest(root, runId); if (!envelopeMatchesRequest(envelope, request, payload)) return deny('foreign-binding'); - const profile = artifactRead(projectRoot, envelope.profile_ref); - const validation = artifactRead(projectRoot, envelope.validation_receipt_ref); - const preauthorization = artifactRead(projectRoot, envelope.preauthorization_ref); - const environment = artifactRead(projectRoot, envelope.environment_receipt_ref); - const plan = artifactRead(projectRoot, envelope.plan_ref); - const grant = artifactRead(projectRoot, envelope.grant_ref); + const profile = artifactRead(projectRoot, envelope.profile_ref, envelope.profile_artifact_sha256); + const validation = artifactRead(projectRoot, envelope.validation_receipt_ref, + envelope.validation_receipt_sha256); + const preauthorization = artifactRead(projectRoot, envelope.preauthorization_ref, + envelope.preauthorization_sha256); + const environment = artifactRead(projectRoot, envelope.environment_receipt_ref, + envelope.environment_receipt_sha256); + const plan = artifactRead(projectRoot, envelope.plan_ref, envelope.plan_sha256); + const grant = artifactRead(projectRoot, envelope.grant_ref, envelope.grant_sha256); const inputs = { profile: profile && profile.digest || empty.profile, validation_receipt: validation && validation.digest || empty.validation_receipt, @@ -978,7 +1740,7 @@ function authorizeCliEffect(projectRoot, payload) { || !argvAllowed(envelope.case.argv, grant.value.cli_capabilities)) { return deny('scope-denied', inputs); } - const replayOwned = replay && replay.status === 'claimed' && replay.claim_id === envelope.fence_id + const replayOwned = replay && replay.status === 'claimed' && replay.fence_id === envelope.fence_id && replayBinding && replayBinding.grant_id === grant.value.grant_id && replayBinding.grant_sha256 === grant.digest && replayBinding.plan_sha256 === plan.digest @@ -1047,12 +1809,22 @@ function dispatch(name, payload, projectRoot) { } return saved; } - case 'artifact-load': - return artifactRead(projectRoot, payload.path); + case 'artifact-load': { + if (payload.expected_digest !== undefined && payload.expected_digest !== null + && !validDigest(payload.expected_digest)) fail('artifact expected digest is invalid'); + if (payload.durable_only !== undefined && typeof payload.durable_only !== 'boolean') { + fail('artifact durable-only option is invalid'); + } + return artifactRead(projectRoot, payload.path, payload.expected_digest, { + durableOnly: payload.durable_only === true, + }); + } case 'artifact-write': return artifactWrite(projectRoot, payload.path, payload.value); case 'artifact-digest': { - const artifact = artifactRead(projectRoot, payload.path); + const artifact = artifactRead(projectRoot, payload.path, undefined, { + allowGeneratedDigestImport: true, + }); return { digest: artifact && artifact.digest || null }; } case 'publication-load-ledger': { @@ -1089,22 +1861,42 @@ function dispatch(name, payload, projectRoot) { } case 'host-claim-preauthorization': { const runId = runIdFor(payload); - loadConfig(projectRoot, runId); - const claimed = recordClaim(runRoot(runId), `generic-host/preauthorization/${sha256(stable(payload.authorization_id))}`, { - binding: payload, claim_id: `${runId}-preauthorization`, + const config = loadConfig(projectRoot, runId); + const root = runRoot(runId); + const key = `generic-host/preauthorization/${sha256(stable(payload.authorization_id))}`; + const existing = recordRead(root, key); + const claimed = existing ? { + claimed: preauthorizationBindingMatches(existing.binding, payload, trustedPreauthorizationRefs(config)), + replayed: true, + value: existing, + } : recordClaim(root, key, { + binding: canonicalPreauthorizationBinding(payload), + claim_id: `private-claim-${crypto.randomBytes(32).toString('hex')}`, + claimed_at: config.execution_authorization_now, }); if (!claimed.claimed) return { status: 'blocked' }; + writePreauthorizationClaimReceipt(projectRoot, config, payload, claimed.value); return { status: 'claimed', claim_id: claimed.value.claim_id, replayed: claimed.replayed === true }; } + case 'host-reconcile-preauthorization-claim': { + const runId = runIdFor(payload); + const config = loadConfig(projectRoot, runId); + const preauthorization = boundLineageArtifact(projectRoot, payload.preauthorization_ref, + payload.preauthorization_sha256, 'preauthorization'); + const request = { ...payload, authorization_id: preauthorization.value.authorization_id }; + const claimed = recordRead(runRoot(runId), + `generic-host/preauthorization/${sha256(stable(request.authorization_id))}`); + if (!claimed || !preauthorizationBindingMatches( + claimed.binding, request, trustedPreauthorizationRefs(config), + )) { + return { reconciled: false }; + } + writePreauthorizationClaimReceipt(projectRoot, config, request, claimed); + return { reconciled: true }; + } case 'host-grant-values': { const runId = runIdFor(payload.request || {}); - loadConfig(projectRoot, runId); - return { - grant_id: `${runId}-grant`, - evidence_ref: { kind: 'signed-attestation', ref: `${runId}-execution-grant` }, - issued_at: '2026-07-22T00:15:00Z', expires_at: '2026-07-22T00:45:00Z', - now: '2026-07-22T00:20:00Z', - }; + return hostStructuredGrantValues(loadConfig(projectRoot, runId)); } case 'host-record-terminal': { const runId = runIdFor(payload); @@ -1176,7 +1968,8 @@ function dispatch(name, payload, projectRoot) { case 'authorize-claim-ports': { const runId = runIdFor(payload); const root = runRoot(runId); - loadConfig(projectRoot, runId); + const config = loadConfig(projectRoot, runId); + const expectedReplayBinding = expectedProfileReplayBinding(config); const key = `environment-factory/effects/${sha256(stable(payload.effect_id))}`; const existing = recordRead(root, key); const binding = payload.lookup_binding || {}; @@ -1187,17 +1980,28 @@ function dispatch(name, payload, projectRoot) { || stable(existing.result.profile_snapshot) !== stable(payload.profile_snapshot)) { fail('environment authorization claim replay differs'); } - return existing.result; + const profileClaim = recordRead(root, `generic-host/profile-approval/${runId}`); + if (!profileClaim || stable(profileClaim.binding) !== stable(expectedReplayBinding) + || typeof profileClaim.claim_id !== 'string') { + fail('environment authorization approval claim is unavailable'); + } + writeProfileClaimReceipt(projectRoot, config, profileClaim); + return { ...existing.result, claim_id: profileClaim.claim_id }; } - if (!payload.replay_claim || stable(payload.profile_snapshot) !== stable(loadConfig(projectRoot, runId).profile)) { + if (stable(payload.replay_claim) !== stable(expectedReplayBinding) + || stable(payload.profile_snapshot) !== stable(config.profile)) { fail('environment authorization claim replay differs'); } const approvalClaim = recordClaim(root, `generic-host/profile-approval/${runId}`, { - binding: payload.replay_claim, claim_id: `${runId}-profile-claim`, + binding: payload.replay_claim, + claim_id: `private-claim-${crypto.randomBytes(32).toString('hex')}`, + claimed_at: config.authorization_now, }); - if (!approvalClaim.claimed || !approvalClaim.value || approvalClaim.value.claim_id !== `${runId}-profile-claim`) { + if (!approvalClaim.claimed || !approvalClaim.value + || typeof approvalClaim.value.claim_id !== 'string') { fail('environment authorization approval claim was not acquired'); } + writeProfileClaimReceipt(projectRoot, config, approvalClaim.value); const runtimePorts = exactRuntimePorts(payload.runtime_ports); if (!Array.isArray(payload.listener_claimed_ports) || payload.listener_claimed_ports.length !== 0 || !samePorts(exactRuntimePorts(payload.listener_already_owned_ports), runtimePorts)) { @@ -1338,27 +2142,35 @@ function dispatch(name, payload, projectRoot) { case 'now': loadConfig(projectRoot, runIdFor(payload)); return { now: '2026-07-22T00:20:00Z' }; - case 'verify-grant': + case 'verify-grant': { + const runId = runIdFor(payload); + const config = loadConfig(projectRoot, runId); + writeGrantVerificationReceipt(projectRoot, config, payload); return { grant_sha256: payload.grant_sha256, authority: payload.grant.authority, policy_revision: payload.grant.policy_revision, evidence_ref: payload.grant.evidence_ref, }; + } case 'replay-guard': { const runId = runIdFor(payload); - loadConfig(projectRoot, runId); - const claimId = `${runId}-execution-claim`; + const config = loadConfig(projectRoot, runId); + const claimId = `private-claim-${crypto.randomBytes(32).toString('hex')}`; + const fenceId = claimFingerprint(config, 'structured-execution-fence', claimId); const claimed = recordClaim(runRoot(runId), structuredReplayKey(payload.grant_id), { - status: 'claimed', claim_id: claimId, binding: payload, + status: 'claimed', claim_id: claimId, fence_id: fenceId, binding: payload, + claimed_at: config.execution_authorization_now, }); if (!claimed.claimed) return null; + writeExecutionClaimReceipt(projectRoot, config, payload, claimed.value); if (claimed.value.status === 'completed') { + writeExecutionCompletionReceipt(projectRoot, config, claimed.value); return { status: 'completed', result_ref: claimed.value.result_ref, result_sha256: claimed.value.result_sha256 }; } if (claimed.replayed) return { status: 'in-progress' }; - return { status: 'claimed', claim_id: claimId }; + return { status: 'claimed', claim_id: fenceId }; } case 'authorize-cli-effect': return authorizeCliEffect(projectRoot, payload); @@ -1389,14 +2201,15 @@ function dispatch(name, payload, projectRoot) { && Date.parse(receipt.expires_at) > Date.parse('2026-07-22T00:20:00Z'); const authorization = validReceipt ? recordRead(root, structuredAuthorizationKey(receipt.receipt_id)) : null; - const grant = validReceipt ? artifactRead(projectRoot, envelope.grant_ref) : null; + const grant = validReceipt + ? artifactRead(projectRoot, envelope.grant_ref, envelope.grant_sha256) : null; const replay = grant && grant.value ? recordRead(root, structuredReplayKey(grant.value.grant_id)) : null; if (!validReceipt || !authorization || !grant || !grant.value || typeof grant.value !== 'object' || stable(authorization.receipt) !== stable(receipt) || authorization.grant_id !== grant.value.grant_id || authorization.fence_id !== envelope.fence_id || grant.digest !== envelope.grant_sha256 || !replay || replay.status !== 'claimed' - || replay.claim_id !== envelope.fence_id) { + || replay.fence_id !== envelope.fence_id) { fail('durable structured CLI authorization receipt is unavailable'); } if (envelope.operation_id !== runId || envelope.workspace_ref.ref !== `${runId}-workspace` @@ -1410,7 +2223,10 @@ function dispatch(name, payload, projectRoot) { if (!consumed.claimed || consumed.replayed) fail('durable structured CLI authorization receipt is replayed'); artifactWrite(projectRoot, `${payload.artifact_root}/authorization/${envelope.case.case_id}-consumption.json`, { schema: 'generic-host.cli-effect-consumption.v1', case_id: envelope.case.case_id, - receipt_id: receipt.receipt_id, grant_id: grant.value.grant_id, fence_id: envelope.fence_id, + receipt_id: receipt.receipt_id, grant_id: grant.value.grant_id, + consumption_fingerprint_sha256: claimFingerprint( + config, 'structured-execution-consumption', envelope.fence_id, + ), }); const result = directExec(envelope.case.argv, workspace.path, envelope.case.timeout_seconds, envelope.resource_bounds.output_bytes); @@ -1440,11 +2256,14 @@ function dispatch(name, payload, projectRoot) { const runId = runIdFor(payload); const config = loadConfig(projectRoot, runId); if (!validDigest(payload.result_sha256)) fail('completed execution result digest is required'); - const artifacts = structuredExecutionArtifacts(projectRoot, payload.result_ref); + const artifacts = structuredExecutionArtifacts(projectRoot, payload.result_ref, + payload.result_sha256); if (artifacts.execution.digest !== payload.result_sha256) { fail('completed execution result digest differs'); } const value = artifacts.execution.value; + const durableClaim = durableExecutionClaim(config); + assertExecutionMatchesClaim(value, durableClaim.binding); if (payload.result_ref !== `${payload.artifact_root}/execution.json` || value.operation_id !== payload.operation_id || value.environment_receipt_sha256 !== payload.environment_receipt_sha256 @@ -1479,7 +2298,7 @@ function dispatch(name, payload, projectRoot) { loadConfig(projectRoot, runId); let current = null; for (const entry of recordList(root, 'testing-runner/replay')) { - if (entry.value && payload.claim && entry.value.claim_id === payload.claim.claim_id) { + if (entry.value && payload.claim && entry.value.fence_id === payload.claim.claim_id) { current = entry; break; } @@ -1494,6 +2313,7 @@ function dispatch(name, payload, projectRoot) { } const artifacts = structuredExecutionArtifacts(projectRoot, payload.result_ref); const execution = artifacts.execution.value; + assertExecutionMatchesClaim(execution, binding); if (payload.result_ref !== `${binding.artifact_root}/execution.json` || execution.operation_id !== binding.operation_id || execution.environment_receipt_sha256 !== binding.environment_receipt_sha256 @@ -1501,12 +2321,15 @@ function dispatch(name, payload, projectRoot) { || execution.trace_id !== binding.trace_id || execution.dedup_key !== binding.dedup_key) { return { completed: false }; } - const completion = { ...payload, result_sha256: artifacts.execution.digest }; + const config = loadConfig(projectRoot, runId); + const completion = { ...payload, result_sha256: artifacts.execution.digest, + completed_at: config.execution_authorization_now }; delete completion.claim; const completed = storeExecute({ root, operation: 'replay-complete', key: current.key, - claim_id: payload.claim.claim_id, completion }); + claim_id: current.value.claim_id, completion }); if (!completed.completed) return completed; - const verified = structuredExecutionArtifacts(projectRoot, payload.result_ref); + const verified = structuredExecutionArtifacts(projectRoot, payload.result_ref, + artifacts.execution.digest); const canonicalChanged = Boolean(verified.caseResultSet) !== Boolean(artifacts.caseResultSet) || (artifacts.caseResultSet && (verified.caseResultSet.digest !== artifacts.caseResultSet.digest || verified.evidenceManifest.digest !== artifacts.evidenceManifest.digest)); @@ -1518,7 +2341,7 @@ function dispatch(name, payload, projectRoot) { || completed.value.result_sha256 !== artifacts.execution.digest) { fail('completed replay result artifact is unavailable or changed'); } - const config = loadConfig(projectRoot, runId); + writeExecutionCompletionReceipt(projectRoot, config, completed.value); const arm = config.completed_replay_failpoint; if (completed.replayed !== true && arm && arm.name === 'post-completed-replay' && typeof arm.token === 'string' @@ -1603,4 +2426,16 @@ function main() { } } -main(); +if (require.main === module) main(); + +module.exports = { + assertStructuredGrantDerivation, + assertExecutionMatchesClaim, + childProcessEnvironment, + hostStructuredGrantValues, + materializeImmutableNoReplace, + preauthorizationBindingMatches, + releaseOwnedProcessResource, + trustedPreauthorizationRefs, + verifyMaterializedImmutable, +}; diff --git a/examples/generic-host/bin/structured-execution-artifacts.js b/examples/generic-host/bin/structured-execution-artifacts.js index 3324610b..5ba37af2 100644 --- a/examples/generic-host/bin/structured-execution-artifacts.js +++ b/examples/generic-host/bin/structured-execution-artifacts.js @@ -86,8 +86,8 @@ function create(options) { } } - function structuredExecutionArtifacts(projectRoot, resultRef) { - const execution = artifactRead(projectRoot, resultRef); + function structuredExecutionArtifacts(projectRoot, resultRef, resultDigest) { + const execution = artifactRead(projectRoot, resultRef, resultDigest); const value = execution && execution.value; const executionRoot = path.posix.dirname(resultRef); if (!value || value.schema !== 'testing-structured-execution.v1' @@ -102,7 +102,7 @@ function create(options) { if (value.case_results_path === undefined && !group) { fail('structured execution result binding is invalid'); } - const testPlan = artifactRead(projectRoot, value.test_plan_path); + const testPlan = artifactRead(projectRoot, value.test_plan_path, value.plan_sha256); const caseResults = value.case_results_path === undefined ? undefined : artifactRead(projectRoot, value.case_results_path); if (!testPlan || testPlan.digest !== value.plan_sha256 @@ -113,8 +113,10 @@ function create(options) { } if (!group) return { execution, testPlan, caseResults }; - const caseResultSet = artifactRead(projectRoot, group.caseResultSetPath); - const evidenceManifest = artifactRead(projectRoot, group.evidenceManifestPath); + const caseResultSet = artifactRead(projectRoot, group.caseResultSetPath, + group.caseResultSetDigest); + const evidenceManifest = artifactRead(projectRoot, group.evidenceManifestPath, + group.evidenceManifestDigest); if (!caseResultSet || caseResultSet.digest !== group.caseResultSetDigest) { fail('structured execution case result set artifact digest differs'); } diff --git a/examples/generic-host/host_canonical_workflow_qa.lua b/examples/generic-host/host_canonical_workflow_qa.lua index 5ead3a51..44c15daf 100644 --- a/examples/generic-host/host_canonical_workflow_qa.lua +++ b/examples/generic-host/host_canonical_workflow_qa.lua @@ -528,10 +528,16 @@ function Context:_structured_runtime() end return { status = "in-progress" } end - claim = { claim_id = context.run_id .. "-execution-claim", binding = copy(request) } + local private_claim_id = context.run_id .. "-execution-claim" + claim = { + claim_id = private_claim_id, + fence_id = sha256_bytes(context.lineage_projection_secret + .. "\0structured-execution-fence\0" .. private_claim_id), + binding = copy(request), + } claims[request.grant_id] = claim context.execution_claims = context.execution_claims + 1 - return { status = "claimed", claim_id = claim.claim_id } + return { status = "claimed", claim_id = claim.fence_id } end, authorize_cli_effect = function(request) local envelope = request.action_envelope @@ -574,6 +580,7 @@ function Context:_structured_runtime() for _, item in ipairs(evaluated_plan.cases or {}) do if item.case_id == envelope.case.case_id then planned_case = item end end + local replay = claims[grant.value.grant_id] if not valid or profile.digest ~= envelope.profile_artifact_sha256 or project_profile.profile_sha256(profile.value, sha256_bytes) ~= envelope.profile_sha256 or validation.digest ~= envelope.validation_receipt_sha256 @@ -589,7 +596,8 @@ function Context:_structured_runtime() or not equal(environment.value.workspace_ref, envelope.workspace_ref) or not equal(planned_case, envelope.case) or not argv_allowed(envelope.case.argv, preauthorization.value.capabilities.cli) - or not argv_allowed(envelope.case.argv, grant.value.cli_capabilities) then + or not argv_allowed(envelope.case.argv, grant.value.cli_capabilities) + or type(replay) ~= "table" or replay.fence_id ~= envelope.fence_id then return decision(envelope, "deny", "foreign-binding", inputs) end return decision(envelope, "allow", "authorized", inputs) @@ -655,7 +663,7 @@ function Context:_structured_runtime() end, complete_replay = function(request) for _, stored in pairs(claims) do - if stored.claim_id == request.claim.claim_id then + if stored.fence_id == request.claim.claim_id then local artifact, canonical = structured_execution_artifacts(context, request.result_ref) local value = artifact and artifact.value or nil if value == nil or value.operation_id ~= request.operation_id @@ -911,6 +919,13 @@ function Context:_generic_host_runtime() context.preauthorization_claims = context.preauthorization_claims + 1 return { status = "claimed", claim_id = preauthorization_claim.claim_id } end, + reconcile_preauthorization_claim = function(value) + if preauthorization_claim == nil then return false end + for key, item in pairs(value) do + if not equal(preauthorization_claim.value[key], item) then return false end + end + return true + end, grant_values = function(_, materials) if context.browser_walking_skeleton then local correlation = materials.environment.browser_readiness.correlation @@ -1584,6 +1599,10 @@ function M.new(options) } workflow_qa.validate_request(request) + local lineage_projection_secret = require_exec({ + "node", "-e", "process.stdout.write(require('crypto').randomBytes(32).toString('hex'))", + }) + local context = setmetatable({ project_root = supervisor_project_root, port = port, @@ -1635,6 +1654,7 @@ function M.new(options) browser_clock = 0, browser_failpoint = options.browser_failpoint, browser_failpoint_fired = false, + lineage_projection_secret = lineage_projection_secret, browser_crash = options.browser_crash == true, cleanup_effects = 0, publication_ack_loss = options.publication_ack_loss == true, @@ -1651,16 +1671,22 @@ function M.new(options) local durable_root = options.durable_root if durable_root == nil and options.durable == true then durable_root = temp_root .. "/framework-durable" end if durable_root ~= nil then - local durable = require("host_durable_workflow_qa") - durable.initialize(context, durable_root) - context.runtime_config_ref = ".testing/generic-host-runtime.json" - write_file(context.project_root .. "/" .. context.runtime_config_ref, json_codec.encode({ - schema = "generic-host.runtime-config.v1", - project_root = context.project_root, - }) .. "\n") - if options.prepare_execution_grant_pending ~= false then - local prepared_context = durable.load(context.project_root, durable_root, context.run_id) - require("test_support.host_workflow_qa_supervisor").prepare(prepared_context, context.project_root) + local ok, failure = pcall(function() + local durable = require("host_durable_workflow_qa") + durable.initialize(context, durable_root) + context.runtime_config_ref = ".testing/generic-host-runtime.json" + write_file(context.project_root .. "/" .. context.runtime_config_ref, json_codec.encode({ + schema = "generic-host.runtime-config.v1", + project_root = context.project_root, + }) .. "\n") + if options.prepare_execution_grant_pending ~= false then + local prepared_context = durable.load(context.project_root, durable_root, context.run_id) + require("test_support.host_workflow_qa_supervisor").prepare(prepared_context, context.project_root) + end + end) + if not ok then + pcall(function() context:cleanup() end) + error(failure, 0) end end return context diff --git a/examples/generic-host/host_durable_workflow_qa.lua b/examples/generic-host/host_durable_workflow_qa.lua index ef15534b..926bdd67 100644 --- a/examples/generic-host/host_durable_workflow_qa.lua +++ b/examples/generic-host/host_durable_workflow_qa.lua @@ -4,6 +4,7 @@ local environment_factory = require("contract.environment_factory") local json_codec = require("testing_runtime.json") local project_profile = require("contract.project_profile") local ai_design_loop = require("testing_ai.module_ai_design_loop") +local lineage_projection = require("testing_runtime.authorization_lineage_projection") local Store = require("host_durable_store") local M = {} @@ -246,6 +247,88 @@ end local Context = {} Context.__index = Context +local function bound_artifact(store, path, expected_digest, label) + local artifact = store:load(path) + if artifact == nil or type(artifact.value) ~= "table" + or (expected_digest ~= nil and artifact.digest ~= expected_digest) then + error("generic-host durable " .. label .. " artifact binding differs") + end + return artifact +end + +local lineage_envelope_fields = { + repository = true, + run_id = true, + trace_id = true, + dedup_key = true, +} + +local function receipt_fields(source) + local fields = {} + for key, value in pairs(source) do + if lineage_envelope_fields[key] ~= true then fields[key] = copy(value) end + end + return fields +end + +local function expected_profile_replay_binding(config) + return { + approval_id = config.approval.approval_id, + approval_sha256 = config.validation_receipt.approval_sha256, + profile_sha256 = config.validation_receipt.profile_sha256, + repository = copy(config.profile.repository), + trace_id = config.validation_receipt.trace_id, + dedup_key = config.validation_receipt.dedup_key, + max_uses = config.approval.max_uses, + } +end + +local function profile_claim_receipt(config, store, projector, durable_claim) + local claimed_at = type(durable_claim) == "table" + and (durable_claim.claimed_at or config.authorization_now) or nil + if type(durable_claim) ~= "table" + or not equal(durable_claim.binding, expected_profile_replay_binding(config)) + or type(durable_claim.claim_id) ~= "string" or durable_claim.claim_id == "" + or type(claimed_at) ~= "string" or claimed_at == "" then + error("generic-host durable environment authorization approval claim is unavailable") + end + local start = config.request.environment_start + local profile = bound_artifact(store, start.profile_ref.ref, nil, "profile") + local approval = bound_artifact(store, start.approval_ref.ref, nil, "profile approval") + local validation = bound_artifact(store, start.validation_receipt_ref.ref, nil, "profile validation") + if profile.value.revision ~= config.validation_receipt.profile_revision + or approval.value.approval_id ~= config.validation_receipt.approval_id + or validation.value.profile_sha256 ~= config.validation_receipt.profile_sha256 + or validation.value.approval_sha256 ~= config.validation_receipt.approval_sha256 then + error("generic-host durable profile claim source artifacts differ") + end + local fingerprint = projector:fingerprint("project-profile-approval-claim", durable_claim.claim_id) + local source = { + repository = { url = config.repository.url, commit_sha = config.repository.commit_sha }, + run_id = config.run_id, + trace_id = config.request.trace_id, + dedup_key = config.request.dedup_key, + profile_source_ref = copy(config.profile_source_ref), + profile_artifact_ref = start.profile_ref.ref, + profile_artifact_sha256 = profile.digest, + profile_sha256 = validation.value.profile_sha256, + profile_revision = profile.value.revision, + approval_artifact_ref = start.approval_ref.ref, + approval_artifact_sha256 = approval.digest, + approval_id = approval.value.approval_id, + approval_sha256 = validation.value.approval_sha256, + approval_authority = copy(approval.value.authority), + policy_revision = approval.value.policy_revision, + evidence_ref = copy(approval.value.evidence_ref), + validation_receipt_ref = start.validation_receipt_ref.ref, + validation_receipt_sha256 = validation.digest, + claim_fingerprint_sha256 = fingerprint, + claimed_at = claimed_at, + } + return projector:write_receipt("profile_claim", "profile-claim-" .. fingerprint:sub(1, 32), + claimed_at, receipt_fields(source), source) +end + local canonical_execution_fields = { "case_result_set_path", "case_result_set_artifact_sha256", "evidence_manifest_path", "evidence_manifest_artifact_sha256", @@ -299,6 +382,24 @@ local function structured_execution_artifacts(context, result_ref) } end +local function durable_execution_claim(context, required_status) + local entries = context.records:list("testing-runner/replay") + if #entries ~= 1 or type(entries[1].value) ~= "table" + or type(entries[1].value.binding) ~= "table" + or (required_status ~= nil and entries[1].value.status ~= required_status) then + error("generic-host durable execution claim is unavailable") + end + return entries[1].value +end + +local function execution_plan_matches_claim(execution_artifact, durable_claim) + local value = execution_artifact and execution_artifact.value or nil + local binding = durable_claim and durable_claim.binding or nil + return type(value) == "table" and type(binding) == "table" + and valid_digest(value.plan_sha256) and valid_digest(binding.plan_sha256) + and value.plan_sha256 == binding.plan_sha256 +end + function Context:_key(value) return self.records:digest(json_codec.encode(value)) end @@ -635,6 +736,400 @@ function Context:_testing_design_runtime() } end +local function lineage_source(context, fields) + local source = { + repository = { url = context.repository.url, commit_sha = context.repository.commit_sha }, + run_id = context.run_id, + trace_id = context.request.trace_id, + dedup_key = context.request.dedup_key, + } + for key, value in pairs(fields) do source[key] = copy(value) end + return source +end + +local function legacy_preauthorization_binding(request) + return { + authorization_id = request.authorization_id, + preauthorization_sha256 = request.preauthorization_sha256, + repository = copy(request.repository), + plan_sha256 = request.plan_sha256, + environment_receipt_sha256 = request.environment_receipt_sha256, + trace_id = request.trace_id, + dedup_key = request.dedup_key, + } +end + +local function canonical_preauthorization_binding(request) + return { + authorization_id = request.authorization_id, + preauthorization_ref = request.preauthorization_ref, + preauthorization_sha256 = request.preauthorization_sha256, + repository = copy(request.repository), + plan_ref = request.plan_ref, + plan_sha256 = request.plan_sha256, + environment_receipt_ref = request.environment_receipt_ref, + environment_receipt_sha256 = request.environment_receipt_sha256, + trace_id = request.trace_id, + dedup_key = request.dedup_key, + } +end + +local function compatible_stored_preauthorization_binding(stored) + if type(stored) ~= "table" or stored.runtime_config_ref == nil then return stored end + if not equal(stored.runtime_config_ref, { + kind = "artifact", ref = ".testing/generic-host-runtime.json", + }) then return stored end + local normalized = copy(stored) + normalized.runtime_config_ref = nil + return normalized +end + +local function trusted_preauthorization_refs(context) + return { + preauthorization_ref = context.request.structured_execution.preauthorization_ref, + plan_ref = context.request.structured_execution.structured_plan_ref, + environment_receipt_ref = context.request.environment_start.artifact_root + .. "/environment-receipt-ready.json", + } +end + +local function preauthorization_binding_matches(stored, request, trusted_refs) + if type(trusted_refs) ~= "table" + or request.preauthorization_ref ~= trusted_refs.preauthorization_ref + or request.plan_ref ~= trusted_refs.plan_ref + or request.environment_receipt_ref ~= trusted_refs.environment_receipt_ref then + return false + end + local normalized = compatible_stored_preauthorization_binding(stored) + return equal(normalized, canonical_preauthorization_binding(request)) + or equal(normalized, legacy_preauthorization_binding(request)) +end + +local function preauthorization_request(preauthorization, request) + return { + authorization_id = preauthorization.value.authorization_id, + preauthorization_ref = request.preauthorization_ref, + preauthorization_sha256 = preauthorization.digest, + repository = copy(request.repository), + plan_ref = request.plan_ref, + plan_sha256 = request.plan_sha256, + environment_receipt_ref = request.environment_receipt_ref, + environment_receipt_sha256 = request.environment_receipt_sha256, + trace_id = request.trace_id, + dedup_key = request.dedup_key, + } +end + +function Context:_private_claim_id(domain) + local nonce = next_nonce() + return "private-claim-" .. self.records:digest( + self.lineage_projection_secret .. "\0" .. domain .. "\0" .. nonce) +end + +function Context:_persist_profile_claim() + local durable_claim = self.records:read("generic-host/profile-approval/" .. self.run_id) + if type(durable_claim) ~= "table" then + error("generic-host durable Profile claim is unavailable") + end + return profile_claim_receipt(self, self.store, self.lineage, durable_claim) +end + +function Context:_persist_preauthorization_claim(request, durable_claim) + if type(durable_claim) ~= "table" then + error("generic-host durable preauthorization claim binding differs") + end + local claimed_at = durable_claim.claimed_at or self.execution_authorization_now + if not preauthorization_binding_matches( + durable_claim.binding, request, trusted_preauthorization_refs(self)) + or type(durable_claim.claim_id) ~= "string" or type(claimed_at) ~= "string" then + error("generic-host durable preauthorization claim binding differs") + end + local profile_receipt = self:_persist_profile_claim() + local preauthorization = bound_artifact(self.store, request.preauthorization_ref, + request.preauthorization_sha256, "preauthorization") + local catalog_ref = self.request.structured_execution.case_catalog_ref + local catalog = bound_artifact(self.store, catalog_ref, + self.request.structured_execution.case_catalog_sha256, "case catalog") + local plan = bound_artifact(self.store, request.plan_ref, request.plan_sha256, "structured plan") + local environment = bound_artifact(self.store, request.environment_receipt_ref, + request.environment_receipt_sha256, "environment receipt") + if preauthorization.value.authorization_id ~= request.authorization_id + or preauthorization.value.profile_sha256 ~= profile_receipt.value.profile_sha256 + or preauthorization.value.case_catalog_sha256 ~= catalog.digest + or plan.value.environment_receipt_sha256 ~= environment.digest then + error("generic-host durable preauthorization lineage source differs") + end + local fingerprint = self.lineage:fingerprint("structured-preauthorization-claim", durable_claim.claim_id) + local source = lineage_source(self, { + profile_claim_receipt_ref = profile_receipt.ref, + profile_claim_receipt_sha256 = profile_receipt.sha256, + preauthorization_ref = request.preauthorization_ref, + preauthorization_sha256 = preauthorization.digest, + authorization_id = preauthorization.value.authorization_id, + profile_sha256 = preauthorization.value.profile_sha256, + case_catalog_ref = catalog_ref, + case_catalog_sha256 = catalog.digest, + plan_ref = request.plan_ref, + plan_sha256 = plan.digest, + environment_receipt_ref = request.environment_receipt_ref, + environment_receipt_sha256 = environment.digest, + authority = copy(preauthorization.value.authority), + policy_revision = preauthorization.value.policy_revision, + evidence_ref = copy(preauthorization.value.evidence_ref), + claim_fingerprint_sha256 = fingerprint, + claimed_at = claimed_at, + }) + return self.lineage:write_receipt("preauthorization_claim", + "preauthorization-claim-" .. fingerprint:sub(1, 32), claimed_at, + receipt_fields(source), source) +end + +local function execution_grant_values(context) + return { + grant_id = context.run_id .. "-grant", + evidence_ref = { kind = "signed-attestation", ref = context.run_id .. "-execution-grant" }, + issued_at = "2026-07-22T00:15:00Z", + expires_at = "2026-07-22T00:45:00Z", + now = "2026-07-22T00:20:00Z", + } +end + +function Context:_persist_grant_verification(request) + local preauthorization = bound_artifact(self.store, request.preauthorization_ref, + request.preauthorization_sha256, "preauthorization") + local durable_claim = self.records:read( + "generic-host/preauthorization/" .. self:_key(preauthorization.value.authorization_id)) + if type(durable_claim) ~= "table" then + error("generic-host durable Preauthorization claim is unavailable") + end + local grant = bound_artifact(self.store, request.grant_ref, request.grant_sha256, "execution grant") + local plan = bound_artifact(self.store, request.plan_ref, request.plan_sha256, "structured plan") + local environment = bound_artifact(self.store, request.environment_receipt_ref, + request.environment_receipt_sha256, "environment receipt") + if grant.value.parent_authorization_sha256 ~= preauthorization.digest + or grant.value.plan_sha256 ~= plan.digest + or grant.value.environment_receipt_sha256 ~= environment.digest + or not execution.same_repository(grant.value.repository, request.repository) then + error("generic-host durable grant verification source differs") + end + local derivation_request = { + schema = execution.schemas.grant_request, + execution_mode = plan.value.execution_mode, + repository = copy(request.repository), + preauthorization_ref = request.preauthorization_ref, + preauthorization_sha256 = preauthorization.digest, + plan_ref = request.plan_ref, + plan_sha256 = plan.digest, + environment_receipt_ref = request.environment_receipt_ref, + environment_receipt_sha256 = environment.digest, + grant_ref = request.grant_ref, + trace_id = request.trace_id, + dedup_key = request.dedup_key, + source_ref = { kind = "workflow-qa", ref = self.run_id }, + } + local expected_grant = execution.derive_grant( + preauthorization.value, preauthorization.digest, + plan.value, plan.digest, environment.digest, derivation_request, + execution_grant_values(self)) + if not equal(grant.value, expected_grant) then + error("generic-host durable Grant differs from authenticated derivation") + end + local complete_preauthorization_request = preauthorization_request(preauthorization, request) + local preauthorization_receipt = self:_persist_preauthorization_claim( + complete_preauthorization_request, durable_claim) + local verification_id = "grant-verification-" .. grant.digest:sub(1, 32) + local durable = self.records:immutable( + "testing-runner/grant-verifications/" .. self.records:digest(grant.digest), { + binding = { + grant_ref = request.grant_ref, + grant_sha256 = grant.digest, + preauthorization_ref = request.preauthorization_ref, + preauthorization_sha256 = preauthorization.digest, + plan_ref = request.plan_ref, + plan_sha256 = plan.digest, + environment_receipt_ref = request.environment_receipt_ref, + environment_receipt_sha256 = environment.digest, + repository = copy(request.repository), + trace_id = request.trace_id, + dedup_key = request.dedup_key, + }, + verification_id = verification_id, + verified_at = self.execution_authorization_now, + }) + if durable.written ~= true and durable.replayed ~= true then + error("generic-host durable grant verification record differs") + end + local value = durable.value + local source = lineage_source(self, { + preauthorization_claim_receipt_ref = preauthorization_receipt.ref, + preauthorization_claim_receipt_sha256 = preauthorization_receipt.sha256, + grant_ref = request.grant_ref, + grant_sha256 = grant.digest, + grant_id = grant.value.grant_id, + parent_authorization_ref = request.preauthorization_ref, + parent_authorization_sha256 = preauthorization.digest, + plan_ref = request.plan_ref, + plan_sha256 = plan.digest, + environment_receipt_ref = request.environment_receipt_ref, + environment_receipt_sha256 = environment.digest, + authority = copy(grant.value.authority), + policy_revision = grant.value.policy_revision, + evidence_ref = copy(grant.value.evidence_ref), + verifier_ref = copy(self.grant_verifier_ref), + verification_id = verification_id, + verified_at = value.verified_at, + }) + return self.lineage:write_receipt("grant_verification", verification_id, + value.verified_at, receipt_fields(source), source) +end + +function Context:_persist_execution_claim(request, durable_claim) + if type(durable_claim) ~= "table" then + error("generic-host durable execution claim binding differs") + end + local claimed_at = durable_claim.claimed_at or self.execution_authorization_now + if type(durable_claim.claim_id) ~= "string" + or type(claimed_at) ~= "string" or not equal(durable_claim.binding, request) + or durable_claim.fence_id ~= self.lineage:fingerprint( + "structured-execution-fence", durable_claim.claim_id) then + error("generic-host durable execution claim binding differs") + end + local grant_record = self.records:read( + "testing-runner/grant-verifications/" .. self.records:digest(request.grant_sha256)) + if type(grant_record) ~= "table" then + error("generic-host durable Grant verification is unavailable") + end + local grant_request = copy(grant_record.binding) + grant_request.grant = bound_artifact(self.store, grant_request.grant_ref, + grant_request.grant_sha256, "execution grant").value + local grant_receipt = self:_persist_grant_verification(grant_request) + local preauthorization = bound_artifact(self.store, request.preauthorization_ref, + request.preauthorization_sha256, "preauthorization") + local preauthorization_claim = self.records:read( + "generic-host/preauthorization/" .. self:_key(preauthorization.value.authorization_id)) + local complete_preauthorization_request = preauthorization_request( + preauthorization, grant_record.binding) + local preauthorization_receipt = self:_persist_preauthorization_claim( + complete_preauthorization_request, preauthorization_claim) + local fingerprint = self.lineage:fingerprint("structured-execution-claim", durable_claim.claim_id) + local source = lineage_source(self, { + grant_verification_receipt_ref = grant_receipt.ref, + grant_verification_receipt_sha256 = grant_receipt.sha256, + preauthorization_claim_receipt_ref = preauthorization_receipt.ref, + preauthorization_claim_receipt_sha256 = preauthorization_receipt.sha256, + grant_ref = request.grant_ref, + grant_sha256 = request.grant_sha256, + grant_id = request.grant_id, + plan_ref = request.plan_ref, + plan_sha256 = request.plan_sha256, + environment_receipt_ref = request.environment_receipt_ref, + environment_receipt_sha256 = request.environment_receipt_sha256, + artifact_root = request.artifact_root, + operation_id = request.operation_id, + claim_fingerprint_sha256 = fingerprint, + claimed_at = claimed_at, + }) + return self.lineage:write_receipt("execution_claim", + "execution-claim-" .. fingerprint:sub(1, 32), claimed_at, + receipt_fields(source), source) +end + +function Context:_authorization_lineage_sources(completion_receipt, completion_expected) + local profile, profile_expected = self:_persist_profile_claim() + local preauthorization_ref = self.request.structured_execution.preauthorization_ref + local preauthorization = bound_artifact(self.store, preauthorization_ref, + self.request.structured_execution.preauthorization_sha256, "preauthorization") + local preauthorization_claim = self.records:read( + "generic-host/preauthorization/" .. self:_key(preauthorization.value.authorization_id)) + if type(preauthorization_claim) ~= "table" then + error("generic-host durable Preauthorization claim is unavailable") + end + local grant_ref = self.request.structured_execution.grant_ref + local grant = bound_artifact(self.store, grant_ref, nil, "execution grant") + local grant_record = self.records:read( + "testing-runner/grant-verifications/" .. self.records:digest(grant.digest)) + if type(grant_record) ~= "table" then + error("generic-host durable Grant verification is unavailable") + end + local complete_preauthorization_request = preauthorization_request( + preauthorization, grant_record.binding) + local preauthorization_receipt, preauthorization_expected = self:_persist_preauthorization_claim( + complete_preauthorization_request, preauthorization_claim) + local grant_receipt, grant_expected = self:_persist_grant_verification(grant_record.binding) + local replay_entries = self.records:list("testing-runner/replay") + if #replay_entries ~= 1 or type(replay_entries[1].value) ~= "table" then + error("generic-host durable execution claim is unavailable") + end + local execution_receipt, execution_expected = self:_persist_execution_claim( + replay_entries[1].value.binding, replay_entries[1].value) + return { + profile_claim = profile, + preauthorization_claim = preauthorization_receipt, + grant_verification = grant_receipt, + execution_claim = execution_receipt, + execution_completion = completion_receipt, + }, { + profile_claim = profile_expected, + preauthorization_claim = preauthorization_expected, + grant_verification = grant_expected, + execution_claim = execution_expected, + execution_completion = completion_expected, + } +end + +function Context:_execution_completion_source(durable_claim) + local completed_at = type(durable_claim) == "table" and type(durable_claim.completion) == "table" + and (durable_claim.completion.completed_at or self.execution_authorization_now) or nil + if type(durable_claim) ~= "table" or durable_claim.status ~= "completed" + or type(durable_claim.completion) ~= "table" + or type(completed_at) ~= "string" then + error("generic-host durable execution completion binding differs") + end + local completion = durable_claim.completion + local execution_claim = self:_persist_execution_claim(durable_claim.binding, durable_claim) + local execution_artifact, canonical = structured_execution_artifacts(self, completion.result_ref) + if execution_artifact == nil or canonical == nil + or execution_artifact.digest ~= completion.result_sha256 then + error("generic-host durable canonical completion artifacts are unavailable") + end + if not execution_plan_matches_claim(execution_artifact, durable_claim) then + error("generic-host durable completion Plan binding differs") + end + local source = lineage_source(self, { + execution_claim_receipt_ref = execution_claim.ref, + execution_claim_receipt_sha256 = execution_claim.sha256, + result_ref = completion.result_ref, + result_sha256 = execution_artifact.digest, + case_result_set_ref = execution_artifact.value.case_result_set_path, + case_result_set_artifact_sha256 = canonical.case_result_set.digest, + evidence_manifest_ref = execution_artifact.value.evidence_manifest_path, + evidence_manifest_artifact_sha256 = canonical.evidence_manifest.digest, + completed_at = completed_at, + }) + return source, completed_at, execution_artifact +end + +function Context:_persist_execution_completion(durable_claim) + local source, completed_at, execution_artifact = self:_execution_completion_source(durable_claim) + local receipt, completion_expected = self.lineage:write_receipt("execution_completion", + "execution-completion-" .. execution_artifact.digest:sub(1, 32), completed_at, + receipt_fields(source), source) + local artifacts, expected = self:_authorization_lineage_sources(receipt, completion_expected) + self.lineage:write_index(completed_at, artifacts, expected) + return receipt +end + +function Context:authorization_lineage_evidence() + local replay_entries = self.records:list("testing-runner/replay") + if #replay_entries ~= 1 or type(replay_entries[1].value) ~= "table" + or replay_entries[1].value.status ~= "completed" then + error("generic-host durable completed execution claim is unavailable") + end + local source = self:_execution_completion_source(replay_entries[1].value) + local completion = self.lineage:load_receipt("execution_completion", source) + return self:_authorization_lineage_sources(completion, source) +end + function Context:_structured_runtime() local context = self local function replay_key(grant_id) return "testing-runner/replay/" .. context:_key(grant_id) end @@ -687,6 +1182,7 @@ function Context:_structured_runtime() end, verify_grant = function(request) local grant = request.grant + context:_persist_grant_verification(request) return { grant_sha256 = request.grant_sha256, authority = copy(grant.authority), policy_revision = grant.policy_revision, evidence_ref = copy(grant.evidence_ref), @@ -694,17 +1190,21 @@ function Context:_structured_runtime() end, replay_guard = function(request) local key = replay_key(request.grant_id) - local claim_id = context.run_id .. "-execution-claim" + local claim_id = context:_private_claim_id("structured-execution") + local fence_id = context.lineage:fingerprint("structured-execution-fence", claim_id) local claimed = context.records:claim(key, { - status = "claimed", claim_id = claim_id, binding = copy(request), + status = "claimed", claim_id = claim_id, fence_id = fence_id, binding = copy(request), + claimed_at = context.execution_authorization_now, }) if claimed.claimed ~= true then return nil end local value = claimed.value + context:_persist_execution_claim(request, value) if value.status == "completed" then + context:_persist_execution_completion(value) return { status = "completed", result_ref = value.result_ref, result_sha256 = value.result_sha256 } end if claimed.replayed == true then return { status = "in-progress" } end - return { status = "claimed", claim_id = claim_id } + return { status = "claimed", claim_id = value.fence_id } end, authorize_cli_effect = function(request) local envelope = request.action_envelope @@ -721,6 +1221,8 @@ function Context:_structured_runtime() local environment = context.store:load(envelope.environment_receipt_ref) local plan = context.store:load(envelope.plan_ref) local grant = context.store:load(envelope.grant_ref) + local replay = type(grant) == "table" and type(grant.value) == "table" + and context.records:read(replay_key(grant.value.grant_id)) or nil local inputs = { profile = profile and profile.digest or empty.profile, validation_receipt = validation and validation.digest or empty.validation_receipt, @@ -757,6 +1259,8 @@ function Context:_structured_runtime() or grant.value.plan_sha256 ~= plan.digest or grant.value.environment_receipt_sha256 ~= environment.digest or not equal(environment.value.workspace_ref, envelope.workspace_ref) + or type(replay) ~= "table" or replay.status ~= "claimed" + or replay.fence_id ~= envelope.fence_id or not equal(planned_case, envelope.case) or not argv_allowed(envelope.case.argv, preauthorization.value.capabilities.cli) or not argv_allowed(envelope.case.argv, grant.value.cli_capabilities) then @@ -809,9 +1313,14 @@ function Context:_structured_runtime() write_artifact = function(path, value) return context.store:write(path, value) end, load_result = function(request) local artifact = structured_execution_artifacts(context, request.result_ref) + local durable_claim = durable_execution_claim(context, "completed") if artifact == nil or (request.result_sha256 ~= nil and artifact.digest ~= request.result_sha256) then return nil end local value = artifact.value - if value.operation_id ~= request.operation_id or value.environment_receipt_sha256 ~= request.environment_receipt_sha256 + if not execution_plan_matches_claim(artifact, durable_claim) + or durable_claim.result_ref ~= request.result_ref + or durable_claim.result_sha256 ~= request.result_sha256 + or value.operation_id ~= request.operation_id + or value.environment_receipt_sha256 ~= request.environment_receipt_sha256 or not execution.same_repository(value.repository, request.repository) or value.trace_id ~= request.trace_id or value.dedup_key ~= request.dedup_key then return nil end local summary = { @@ -834,7 +1343,7 @@ function Context:_structured_runtime() complete_replay = function(request) local current for _, entry in ipairs(context.records:list("testing-runner/replay")) do - if type(entry.value) == "table" and entry.value.claim_id == request.claim.claim_id then + if type(entry.value) == "table" and entry.value.fence_id == request.claim.claim_id then current = entry break end @@ -859,7 +1368,8 @@ function Context:_structured_runtime() local completion = copy(request) completion.claim = nil completion.result_sha256 = artifact.digest - local completed = context.records:complete_replay(current.key, request.claim.claim_id, completion) + completion.completed_at = context.execution_authorization_now + local completed = context.records:complete_replay(current.key, current.value.claim_id, completion) if completed.completed ~= true then return false end local verified, verified_canonical = structured_execution_artifacts(context, request.result_ref) if verified.digest ~= artifact.digest @@ -868,6 +1378,7 @@ function Context:_structured_runtime() or verified_canonical.evidence_manifest.digest ~= canonical.evidence_manifest.digest)) then error("generic-host durable canonical execution artifacts changed during replay completion") end + context:_persist_execution_completion(completed.value) return true end, } @@ -952,22 +1463,42 @@ function Context:_generic_host_runtime() } end, claim_preauthorization = function(value) - local claimed = context.records:claim("generic-host/preauthorization/" .. context:_key(value.authorization_id), { - binding = copy(value), claim_id = context.run_id .. "-preauthorization", - }) + local key = "generic-host/preauthorization/" .. context:_key(value.authorization_id) + local existing = context.records:read(key) + local claimed + if existing ~= nil then + claimed = { + claimed = preauthorization_binding_matches( + existing.binding, value, trusted_preauthorization_refs(context)), + replayed = true, + value = existing, + } + else + claimed = context.records:claim(key, { + binding = canonical_preauthorization_binding(value), + claim_id = context:_private_claim_id("structured-preauthorization"), + claimed_at = context.execution_authorization_now, + }) + end if claimed.claimed ~= true then return { status = "blocked" } end + context:_persist_preauthorization_claim(value, claimed.value) return { status = "claimed", claim_id = claimed.value.claim_id, replayed = claimed.replayed == true, } end, - grant_values = function() - return { - grant_id = context.run_id .. "-grant", - evidence_ref = { kind = "signed-attestation", ref = context.run_id .. "-execution-grant" }, - issued_at = "2026-07-22T00:15:00Z", expires_at = "2026-07-22T00:45:00Z", - now = "2026-07-22T00:20:00Z", - } + reconcile_preauthorization_claim = function(value) + local preauthorization = bound_artifact(context.store, value.preauthorization_ref, + value.preauthorization_sha256, "preauthorization") + local request = copy(value) + request.authorization_id = preauthorization.value.authorization_id + local claimed = context.records:read( + "generic-host/preauthorization/" .. context:_key(request.authorization_id)) + if claimed == nil or not preauthorization_binding_matches( + claimed.binding, request, trusted_preauthorization_refs(context)) then return false end + context:_persist_preauthorization_claim(request, claimed) + return true end, + grant_values = function() return execution_grant_values(context) end, record_terminal = function(value) local result = context.records:immutable("generic-host/terminal/" .. context.run_id, copy(value)) return result.written == true or result.replayed == true @@ -1030,7 +1561,7 @@ function Context:terminal_record() return self.records:read("generic-host/terminal/" .. self.run_id) end -local function authorization_context(config, records) +local function authorization_context(config, records, store, projector) local authority = copy(config.approval.authority) local policy_revision = config.approval.policy_revision local evidence_ref = copy(config.approval.evidence_ref) @@ -1053,9 +1584,16 @@ local function authorization_context(config, records) trusted_authorities = { trusted }, approval_ref = copy(config.authorization_approval_ref), replay_guard = function(value) + local nonce = next_nonce() local claimed = records:claim("generic-host/profile-approval/" .. config.run_id, { - binding = copy(value), claim_id = config.run_id .. "-profile-claim", + binding = copy(value), + claim_id = "private-claim-" .. records:digest( + config.lineage_projection_secret .. "\0project-profile\0" .. nonce), + claimed_at = config.authorization_now, }) + if claimed.claimed == true then + profile_claim_receipt(config, store, projector, claimed.value) + end return { claimed = claimed.claimed == true, claim_id = claimed.value and claimed.value.claim_id } end, } @@ -1069,7 +1607,20 @@ local function build_context(config, durable_root) context.durable_root = durable_root context.records = records context.store = ArtifactStore.new(records) - context.authorization_context = authorization_context(config, records) + context.profile_source_ref = copy(config.profile_source_ref) + context.grant_verifier_ref = copy(config.grant_verifier_ref) + context.execution_authorization_now = config.execution_authorization_now + context.lineage = lineage_projection.new({ + store = context.store, + sha256 = function(body) return records:digest(body) end, + repository = { url = config.repository.url, commit_sha = config.repository.commit_sha }, + run_id = config.run_id, + trace_id = config.request.trace_id, + dedup_key = config.request.dedup_key, + artifact_root = config.artifact_root, + fingerprint_secret = config.lineage_projection_secret, + }) + context.authorization_context = authorization_context(config, records, context.store, context.lineage) context.environment_runtime = context:_environment_runtime() context.workflow_runtime = context:_workflow_runtime() context.module_loop_runtime = context:_module_loop_runtime() @@ -1100,10 +1651,30 @@ function M.initialize(context, durable_root) host_root = context.host_root, commit_sha = context.commit_sha, repository = copy(context.repository), + target_execution_boundary = { + schema = "testing-host.target-execution-boundary.v1", + mode = "trusted-fixture-exact", + target_class = "host-owned-exact-trusted-fixture", + repository = copy(context.profile.repository), + authority = { + kind = "host-policy", ref = "fixtures/" .. context.fixture_name .. "-target-execution-boundary", + }, + policy_revision = "generic-host-trusted-fixture-exact-v1", + authorization_capability = false, + execution_authorized = false, + }, profile = copy(context.profile), approval = copy(context.approval), validation_receipt = copy(context.validation_receipt), + profile_source_ref = { + kind = "host-profile-policy", ref = "fixtures/" .. context.fixture_name .. "-profile", + }, + grant_verifier_ref = { + kind = "host-verifier", ref = "fixtures/" .. context.fixture_name .. "-grant-verifier", + }, + lineage_projection_secret = context.lineage_projection_secret, authorization_now = context.authorization_context.now, + execution_authorization_now = "2026-07-22T00:20:00Z", authorization_approval_ref = copy(context.authorization_context.approval_ref), completed_replay_failpoint = copy(context.completed_replay_failpoint), crash_barrier = copy(context.crash_barrier), diff --git a/examples/generic-host/test_support/host_workflow_qa_supervisor.lua b/examples/generic-host/test_support/host_workflow_qa_supervisor.lua index ccb53669..2eb03056 100644 --- a/examples/generic-host/test_support/host_workflow_qa_supervisor.lua +++ b/examples/generic-host/test_support/host_workflow_qa_supervisor.lua @@ -202,6 +202,12 @@ function M.run(context, project_root, options) stopped = prepared("environment-pending", actions) if stopped ~= nil then return stopped end local environment_pending = environment.start(actions[1].payload, context.environment_runtime) + if type(environment_pending) ~= "table" or type(environment_pending.readiness_check) ~= "table" then + error("canonical lifecycle environment start blocked: status=" + .. tostring(type(environment_pending) == "table" and environment_pending.status) + .. " class=" .. tostring(type(environment_pending) == "table" and environment_pending.failure_class) + .. " message=" .. tostring(type(environment_pending) == "table" and environment_pending.message)) + end local environment_ready = environment.handle_browser_readiness( readiness.result(environment_pending.readiness_check), context.environment_runtime).result actions = workflow.handle_environment_result(environment_ready, context.request, context.workflow_runtime) diff --git a/examples/generic-host/tests/authorization_lineage_node_validator_test.js b/examples/generic-host/tests/authorization_lineage_node_validator_test.js new file mode 100644 index 00000000..5491213a --- /dev/null +++ b/examples/generic-host/tests/authorization_lineage_node_validator_test.js @@ -0,0 +1,374 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const { stable } = require(path.resolve(__dirname, '../bin/durable-host-store')); +const lineage = require(path.resolve(__dirname, '../bin/authorization-lineage')); +const runtime = require(path.resolve(__dirname, '../bin/generic-host-runtime')); +const { + releaseWorkerEnvironment, + verifyWorkerEnvironment, + workerEnvironmentLease, +} = require(path.resolve(__dirname, '../../../packages/environment-factory/bin/runtime/common')); + +const runId = 'node-lineage-validator'; +const repository = { url: 'https://example.invalid/testing/fixture.git', commit_sha: '1'.repeat(40) }; +const ref = (suffix) => `.testing/runs/${runId}/${suffix}`; +const digest = (value) => crypto.createHash('sha256').update(stable(value)).digest('hex'); +const fakeDigest = (value) => String(value).repeat(64).slice(0, 64); +const copy = (value) => JSON.parse(JSON.stringify(value)); +const common = (value) => ({ + repository: copy(repository), run_id: runId, trace_id: 'trace-node-lineage', dedup_key: runId, + recorded_at: '2026-09-10T00:10:00Z', source_max_uses: 1, evidence_role: 'audit-only', + authorization_capability: false, reusable: false, ...value, +}); + +const envelopeFields = new Set([ + 'schema', 'status', 'receipt_id', 'recorded_at', 'source_max_uses', + 'evidence_role', 'authorization_capability', 'reusable', +]); +const expected = (value) => Object.fromEntries( + Object.entries(copy(value)).filter(([key]) => !envelopeFields.has(key)), +); + +function fixture() { + const values = { + profile_claim: common({ + schema: lineage.schemas.profile_claim, status: 'claimed', receipt_id: 'profile-claim-1', + profile_source_ref: { kind: 'host-profile-policy', ref: 'policies/profile-v1' }, + profile_artifact_ref: ref('authorization/project-profile.json'), + profile_artifact_sha256: fakeDigest('1'), profile_sha256: fakeDigest('2'), + profile_revision: 'profile-v1', approval_artifact_ref: ref('authorization/profile-approval.json'), + approval_artifact_sha256: fakeDigest('3'), approval_id: 'profile-approval-1', + approval_sha256: fakeDigest('4'), + approval_authority: { kind: 'host-policy', ref: 'policies/profile-approval-v1' }, + policy_revision: 'profile-policy-v1', + evidence_ref: { kind: 'signed-attestation', ref: 'attestations/profile-approval-1' }, + validation_receipt_ref: ref('authorization/profile-validation.json'), + validation_receipt_sha256: fakeDigest('5'), claim_fingerprint_sha256: fakeDigest('6'), + claimed_at: '2026-09-10T00:01:00Z', + }), + preauthorization_claim: common({ + schema: lineage.schemas.preauthorization_claim, status: 'claimed', + receipt_id: 'preauthorization-claim-1', + profile_claim_receipt_ref: ref(lineage.paths.profile_claim), + profile_claim_receipt_sha256: fakeDigest('7'), + preauthorization_ref: ref('execution/preauthorization.json'), + preauthorization_sha256: fakeDigest('8'), authorization_id: 'preauthorization-1', + profile_sha256: fakeDigest('2'), case_catalog_ref: ref('execution/case-catalog.json'), + case_catalog_sha256: fakeDigest('9'), plan_ref: ref('execution/structured-plan.json'), + plan_sha256: fakeDigest('a'), environment_receipt_ref: ref('environment/ready.json'), + environment_receipt_sha256: fakeDigest('b'), + authority: { kind: 'host-policy', ref: 'policies/execution-v1' }, + policy_revision: 'execution-v1', + evidence_ref: { kind: 'signed-attestation', ref: 'attestations/preauthorization-1' }, + claim_fingerprint_sha256: fakeDigest('c'), claimed_at: '2026-09-10T00:02:00Z', + }), + grant_verification: common({ + schema: lineage.schemas.grant_verification, status: 'authenticated', + receipt_id: 'grant-verification-1', + preauthorization_claim_receipt_ref: ref(lineage.paths.preauthorization_claim), + preauthorization_claim_receipt_sha256: fakeDigest('d'), + grant_ref: ref('execution/execution-grant.json'), grant_sha256: fakeDigest('e'), + grant_id: 'grant-1', parent_authorization_ref: ref('execution/preauthorization.json'), + parent_authorization_sha256: fakeDigest('8'), plan_ref: ref('execution/structured-plan.json'), + plan_sha256: fakeDigest('a'), environment_receipt_ref: ref('environment/ready.json'), + environment_receipt_sha256: fakeDigest('b'), + authority: { kind: 'host-policy', ref: 'policies/execution-v1' }, + policy_revision: 'execution-v1', + evidence_ref: { kind: 'signed-attestation', ref: 'attestations/grant-1' }, + verifier_ref: { kind: 'host-verifier', ref: 'verifiers/execution-v1' }, + verification_id: 'verification-1', verified_at: '2026-09-10T00:03:00Z', + }), + execution_claim: common({ + schema: lineage.schemas.execution_claim, status: 'claimed', receipt_id: 'execution-claim-1', + grant_verification_receipt_ref: ref(lineage.paths.grant_verification), + grant_verification_receipt_sha256: fakeDigest('f'), + preauthorization_claim_receipt_ref: ref(lineage.paths.preauthorization_claim), + preauthorization_claim_receipt_sha256: fakeDigest('d'), + grant_ref: ref('execution/execution-grant.json'), grant_sha256: fakeDigest('e'), + grant_id: 'grant-1', plan_ref: ref('execution/structured-plan.json'), plan_sha256: fakeDigest('a'), + environment_receipt_ref: ref('environment/ready.json'), environment_receipt_sha256: fakeDigest('b'), + artifact_root: ref('execution'), operation_id: runId, + claim_fingerprint_sha256: fakeDigest('0'), claimed_at: '2026-09-10T00:04:00Z', + }), + execution_completion: common({ + schema: lineage.schemas.execution_completion, status: 'completed', + receipt_id: 'execution-completion-1', + execution_claim_receipt_ref: ref(lineage.paths.execution_claim), + execution_claim_receipt_sha256: fakeDigest('1'), result_ref: ref('execution/execution.json'), + result_sha256: fakeDigest('2'), case_result_set_ref: ref('execution/case-result-set.json'), + case_result_set_artifact_sha256: fakeDigest('3'), + evidence_manifest_ref: ref('execution/evidence-manifest.json'), + evidence_manifest_artifact_sha256: fakeDigest('4'), completed_at: '2026-09-10T00:05:00Z', + }), + }; + const state = { values }; + reseal(state); + return state; +} + +function reseal(state) { + const { values } = state; + const artifacts = {}; + const bind = (name) => { + const sha256 = digest(values[name]); + artifacts[name] = { ref: ref(lineage.paths[name]), sha256, value: values[name] }; + return sha256; + }; + values.preauthorization_claim.profile_claim_receipt_sha256 = bind('profile_claim'); + const preauthorizationDigest = bind('preauthorization_claim'); + values.grant_verification.preauthorization_claim_receipt_sha256 = preauthorizationDigest; + values.execution_claim.preauthorization_claim_receipt_sha256 = preauthorizationDigest; + values.execution_claim.grant_verification_receipt_sha256 = bind('grant_verification'); + values.execution_completion.execution_claim_receipt_sha256 = bind('execution_claim'); + bind('execution_completion'); + state.artifacts = artifacts; + state.expected = Object.fromEntries(Object.entries(values).map(([name, value]) => [name, expected(value)])); + state.index = { + schema: lineage.schemas.lineage_index, status: 'complete', repository: copy(repository), + run_id: runId, trace_id: 'trace-node-lineage', dedup_key: runId, + recorded_at: '2026-09-10T00:10:00Z', + receipts: Object.fromEntries(Object.entries(artifacts).map(([name, artifact]) => [ + name, { ref: artifact.ref, sha256: artifact.sha256 }, + ])), + lineage_complete: true, source_max_uses: 1, evidence_role: 'audit-only', + authorization_capability: false, reusable: false, + }; +} + +function rejects(mutator) { + const value = fixture(); + mutator(value.values); + reseal(value); + assert.throws(() => lineage.validateLineageIndex(value.index, value.artifacts, value.expected)); +} + +const valid = fixture(); +assert.equal(lineage.validateLineageIndex(valid.index, valid.artifacts, valid.expected), valid.index); + +rejects((values) => { values.grant_verification.authority.ref = 'policies/foreign-execution'; }); +rejects((values) => { values.grant_verification.policy_revision = 'execution-v2'; }); +rejects((values) => { + values.grant_verification.plan_ref = ref('execution/foreign-plan.json'); + values.grant_verification.plan_sha256 = fakeDigest('5'); + values.execution_claim.plan_ref = values.grant_verification.plan_ref; + values.execution_claim.plan_sha256 = values.grant_verification.plan_sha256; +}); +rejects((values) => { + values.grant_verification.environment_receipt_ref = ref('environment/foreign-ready.json'); + values.grant_verification.environment_receipt_sha256 = fakeDigest('6'); + values.execution_claim.environment_receipt_ref = values.grant_verification.environment_receipt_ref; + values.execution_claim.environment_receipt_sha256 = values.grant_verification.environment_receipt_sha256; +}); +rejects((values) => { values.execution_claim.grant_id = 'foreign-grant'; }); +rejects((values) => { values.preauthorization_claim.claimed_at = '2026-09-09T23:59:59Z'; }); +rejects((values) => { values.execution_claim.authorization_capability = true; }); +for (const invalidTimestamp of [ + '2026-02-30T00:00:00Z', + '2026-09-10T24:00:00Z', + '2026-09-10T00:00:00.123Z', +]) { + rejects((values) => { values.execution_claim.claimed_at = invalidTimestamp; }); +} +rejects((values) => { values.execution_claim.receipt_id = '\u00e9'.repeat(91); }); + +const substituted = fixture(); +const trusted = copy(substituted.expected); +substituted.values.profile_claim.profile_sha256 = fakeDigest('f'); +reseal(substituted); +substituted.expected = trusted; +assert.throws(() => lineage.validateLineageIndex( + substituted.index, substituted.artifacts, substituted.expected, +)); + +const preauthorizationRequest = { + authorization_id: 'preauthorization-1', + preauthorization_ref: ref('execution/preauthorization.json'), + preauthorization_sha256: fakeDigest('8'), + repository: copy(repository), + plan_ref: ref('execution/structured-plan.json'), + plan_sha256: fakeDigest('a'), + environment_receipt_ref: ref('environment/ready.json'), + environment_receipt_sha256: fakeDigest('b'), + trace_id: 'trace-node-lineage', + dedup_key: runId, +}; +const legacyPreauthorizationBinding = { + authorization_id: preauthorizationRequest.authorization_id, + preauthorization_sha256: preauthorizationRequest.preauthorization_sha256, + repository: copy(repository), + plan_sha256: preauthorizationRequest.plan_sha256, + environment_receipt_sha256: preauthorizationRequest.environment_receipt_sha256, + trace_id: preauthorizationRequest.trace_id, + dedup_key: preauthorizationRequest.dedup_key, +}; +const runtimeConfigRef = { + kind: 'artifact', ref: '.testing/generic-host-runtime.json', +}; +const trustedPreauthorizationRefs = { + preauthorization_ref: preauthorizationRequest.preauthorization_ref, + plan_ref: preauthorizationRequest.plan_ref, + environment_receipt_ref: preauthorizationRequest.environment_receipt_ref, +}; + +assert.equal(runtime.preauthorizationBindingMatches( + copy(preauthorizationRequest), preauthorizationRequest, trustedPreauthorizationRefs), true); +assert.equal(runtime.preauthorizationBindingMatches( + copy(legacyPreauthorizationBinding), preauthorizationRequest, trustedPreauthorizationRefs), true); +assert.equal(runtime.preauthorizationBindingMatches({ + ...copy(preauthorizationRequest), runtime_config_ref: copy(runtimeConfigRef), +}, preauthorizationRequest, trustedPreauthorizationRefs), true); +assert.equal(runtime.preauthorizationBindingMatches({ + ...copy(legacyPreauthorizationBinding), runtime_config_ref: copy(runtimeConfigRef), +}, preauthorizationRequest, trustedPreauthorizationRefs), true); +assert.equal(runtime.preauthorizationBindingMatches({ + ...copy(preauthorizationRequest), + runtime_config_ref: { kind: 'artifact', ref: '.testing/foreign-runtime.json' }, +}, preauthorizationRequest, trustedPreauthorizationRefs), false); +assert.equal(runtime.preauthorizationBindingMatches({ + ...copy(preauthorizationRequest), unexpected: true, +}, preauthorizationRequest, trustedPreauthorizationRefs), false); +assert.equal(runtime.preauthorizationBindingMatches( + copy(legacyPreauthorizationBinding), { + ...copy(preauthorizationRequest), plan_ref: ref('execution/foreign-plan.json'), + }, trustedPreauthorizationRefs), false); + +const structuredConfig = { run_id: runId }; +const structuredRequest = { + repository: copy(repository), preauthorization_ref: ref('execution/preauthorization.json'), + preauthorization_sha256: fakeDigest('8'), plan_ref: ref('execution/structured-plan.json'), + plan_sha256: fakeDigest('a'), environment_receipt_ref: ref('environment/ready.json'), + environment_receipt_sha256: fakeDigest('b'), grant_ref: ref('execution/execution-grant.json'), + grant_sha256: fakeDigest('e'), now: '2026-07-22T00:20:00Z', + trace_id: 'trace-node-lineage', dedup_key: runId, +}; +const structuredPreauthorization = { digest: structuredRequest.preauthorization_sha256, value: { + schema: 'testing-structured-execution-authorization.v1', authorization_id: 'preauthorization-1', + repository: copy(repository), profile_sha256: fakeDigest('2'), case_catalog_sha256: fakeDigest('9'), + capabilities: { + cli: [], + http: [{ origin: 'http://127.0.0.1:4173', methods: ['GET'], path_prefixes: ['/health'] }], + }, + authority: { kind: 'host-policy', ref: 'policies/execution-v1' }, policy_revision: 'execution-v1', + evidence_ref: { kind: 'signed-attestation', ref: 'attestations/preauthorization-1' }, + issued_at: '2026-07-22T00:00:00Z', expires_at: '2026-07-22T01:00:00Z', max_uses: 1, + trace_id: structuredRequest.trace_id, dedup_key: structuredRequest.dedup_key, +} }; +const structuredPlan = { digest: structuredRequest.plan_sha256, value: { + schema: 'testing-structured-plan.v2', execution_mode: 'structured-api-cli', repository: copy(repository), + environment_receipt_sha256: structuredRequest.environment_receipt_sha256, + case_catalog_sha256: structuredPreauthorization.value.case_catalog_sha256, + cases: [{ + case_id: 'health', kind: 'http', timeout_seconds: 10, + request: { method: 'GET', url: 'http://127.0.0.1:4173/health', headers: [] }, + assertions: [{ type: 'status-code', expected: 200 }], + }], + residual_risk_case_ids: [], browser_readiness_sha256: fakeDigest('c'), + module_plan_sha256: fakeDigest('d'), trace_id: structuredRequest.trace_id, + dedup_key: structuredRequest.dedup_key, +} }; +const structuredEnvironment = { digest: structuredRequest.environment_receipt_sha256, value: { + schema: 'environment-factory.receipt.v2', status: 'ready', repository: copy(repository), + trace_id: structuredRequest.trace_id, dedup_key: structuredRequest.dedup_key, +} }; +const structuredGrant = { digest: structuredRequest.grant_sha256, value: { + schema: 'testing-structured-execution-grant.v1', grant_id: `${runId}-grant`, + parent_authorization_sha256: structuredRequest.preauthorization_sha256, + plan_sha256: structuredRequest.plan_sha256, + environment_receipt_sha256: structuredRequest.environment_receipt_sha256, + repository: copy(repository), cli_capabilities: [], + http_capabilities: copy(structuredPreauthorization.value.capabilities.http), + authority: copy(structuredPreauthorization.value.authority), policy_revision: 'execution-v1', + evidence_ref: { kind: 'signed-attestation', ref: `${runId}-execution-grant` }, + issued_at: '2026-07-22T00:15:00Z', expires_at: '2026-07-22T00:45:00Z', max_uses: 1, + trace_id: structuredRequest.trace_id, dedup_key: structuredRequest.dedup_key, +} }; +assert.deepEqual(runtime.assertStructuredGrantDerivation(structuredConfig, structuredRequest, + structuredPreauthorization, structuredPlan, structuredEnvironment, structuredGrant), structuredGrant.value); +assert.equal(runtime.assertExecutionMatchesClaim( + { plan_sha256: structuredRequest.plan_sha256 }, + { plan_sha256: structuredRequest.plan_sha256 }, +), true); +assert.throws(() => runtime.assertExecutionMatchesClaim( + { plan_sha256: fakeDigest('f') }, + { plan_sha256: structuredRequest.plan_sha256 }, +), /Plan binding differs/); + +for (const mutate of [ + (state) => { state.grant.value.authority.ref = 'policies/foreign'; }, + (state) => { state.grant.value.policy_revision = 'execution-v2'; }, + (state) => { state.grant.value.evidence_ref.ref = 'foreign-evidence'; }, + (state) => { state.grant.value.http_capabilities[0].methods.push('POST'); }, + (state) => { state.grant.value.http_capabilities[0].path_prefixes.push('/admin'); }, + (state) => { state.plan.value.cases[0].request.url = 'http://127.0.0.1:4173/admin'; }, +]) { + const state = { + preauthorization: copy(structuredPreauthorization), plan: copy(structuredPlan), + environment: copy(structuredEnvironment), grant: copy(structuredGrant), + }; + mutate(state); + assert.throws(() => runtime.assertStructuredGrantDerivation(structuredConfig, structuredRequest, + state.preauthorization, state.plan, state.environment, state.grant)); +} + +const cliState = { + preauthorization: copy(structuredPreauthorization), plan: copy(structuredPlan), + environment: copy(structuredEnvironment), grant: copy(structuredGrant), +}; +cliState.preauthorization.value.capabilities = { + cli: [{ argv_prefix: ['fixture-cli', 'health'] }], http: [], +}; +cliState.plan.value.cases = [{ + case_id: 'health-cli', kind: 'cli', argv: ['fixture-cli', 'health'], + timeout_seconds: 10, assertions: [{ type: 'exit-code', expected: 0 }], +}]; +cliState.grant.value.cli_capabilities = copy(cliState.preauthorization.value.capabilities.cli); +cliState.grant.value.http_capabilities = []; +assert.doesNotThrow(() => runtime.assertStructuredGrantDerivation(structuredConfig, structuredRequest, + cliState.preauthorization, cliState.plan, cliState.environment, cliState.grant)); +cliState.grant.value.cli_capabilities[0].argv_prefix = ['fixture-cli']; +assert.throws(() => runtime.assertStructuredGrantDerivation(structuredConfig, structuredRequest, + cliState.preauthorization, cliState.plan, cliState.environment, cliState.grant)); + +for (const key of [ + 'GH_TOKEN', 'GITHUB_TOKEN', 'SSH_AUTH_SOCK', 'GIT_ASKPASS', 'SSH_ASKPASS', + 'FKST_GENERIC_HOST_DURABLE_ROOT', 'FKST_GENERIC_HOST_PROJECT_ROOT', + 'FKST_STRUCTURED_EXECUTION_RUNTIME_CLI', 'FKST_STRUCTURED_EXECUTION_RUNTIME_CONFIG_REF', + 'FKST_DURABLE_COMPLETED_REPLAY_FAILPOINT', 'FKST_GENERIC_HOST_FIXTURE_CLI_DENY_TOKEN', +]) { + process.env[key] = 'must-not-be-inherited'; +} +const childEnvironment = runtime.childProcessEnvironment(process.cwd()); +for (const key of [ + 'GH_TOKEN', 'GITHUB_TOKEN', 'SSH_AUTH_SOCK', 'GIT_ASKPASS', 'SSH_ASKPASS', + 'FKST_GENERIC_HOST_DURABLE_ROOT', 'FKST_GENERIC_HOST_PROJECT_ROOT', + 'FKST_STRUCTURED_EXECUTION_RUNTIME_CLI', 'FKST_STRUCTURED_EXECUTION_RUNTIME_CONFIG_REF', + 'FKST_DURABLE_COMPLETED_REPLAY_FAILPOINT', 'FKST_GENERIC_HOST_FIXTURE_CLI_DENY_TOKEN', +]) { + assert.equal(Object.prototype.hasOwnProperty.call(childEnvironment, key), false); +} +assert.equal(childEnvironment.GIT_TERMINAL_PROMPT, '0'); +assert.equal(childEnvironment.GIT_CONFIG_NOSYSTEM, '1'); +assert.equal(childEnvironment.GIT_CONFIG_COUNT, '4'); +assert.equal(childEnvironment.GIT_CONFIG_KEY_2, 'core.fsmonitor'); +assert.equal(childEnvironment.GIT_CONFIG_VALUE_2, 'false'); +assert.equal(childEnvironment.GIT_CONFIG_KEY_3, 'core.hooksPath'); +assert.equal(verifyWorkerEnvironment(childEnvironment), true); +const childHome = childEnvironment.HOME; +assert.equal(releaseWorkerEnvironment(childEnvironment), true); +assert.equal(fs.existsSync(childHome), false); + +const exitedEnvironment = runtime.childProcessEnvironment(process.cwd()); +const exitedLease = workerEnvironmentLease(exitedEnvironment); +assert.equal(runtime.releaseOwnedProcessResource({ + pid: 2147483647, + pgid: 2147483647, + process_start_identity: 'naturally-exited-process', + runtime_ports: [], + worker_environment_lease: exitedLease, +}, 100), true); +assert.equal(fs.existsSync(exitedLease.home), false); +assert.equal(releaseWorkerEnvironment(exitedEnvironment), true); diff --git a/examples/generic-host/tests/authorization_lineage_node_validator_test.lua b/examples/generic-host/tests/authorization_lineage_node_validator_test.lua new file mode 100644 index 00000000..6ec55169 --- /dev/null +++ b/examples/generic-host/tests/authorization_lineage_node_validator_test.lua @@ -0,0 +1,18 @@ +local process = require("test_support.durable_workflow_qa_process") +local t = fkst.test + +return { + test_node_authorization_lineage_validator_matches_fail_closed_contract = function() + local candidates = { + "examples/generic-host/tests/authorization_lineage_node_validator_test.js", + "packages/generic-host/tests/authorization_lineage_node_validator_test.js", + } + local script + for _, candidate in ipairs(candidates) do + if process.read_file(candidate) ~= nil then script = candidate break end + end + t.is_true(type(script) == "string") + local result = process.exec({ "node", script }) + t.eq(result.exit_code, 0) + end, +} diff --git a/examples/generic-host/tests/downstream_local_qa_acceptance_e2e_test.lua b/examples/generic-host/tests/downstream_local_qa_acceptance_e2e_test.lua index 77cb1a61..941cf80d 100644 --- a/examples/generic-host/tests/downstream_local_qa_acceptance_e2e_test.lua +++ b/examples/generic-host/tests/downstream_local_qa_acceptance_e2e_test.lua @@ -91,7 +91,7 @@ return { t.eq(context.profile.mutation_policy.allowed_operations[1], "update") for _, name in ipairs({ "load_artifact", "write_artifact", "artifact_digest", "claim_preauthorization", - "grant_values", "record_terminal", + "reconcile_preauthorization_claim", "grant_values", "record_terminal", }) do t.eq(type(context.generic_host_runtime[name]), "function") end @@ -145,6 +145,9 @@ return { .. "/authorization/" .. expected .. "-consumption.json") t.eq(consumption.schema, "generic-host.cli-effect-consumption.v1") t.eq(consumption.case_id, expected) + t.eq(consumption.fence_id, nil) + t.is_true(type(consumption.consumption_fingerprint_sha256) == "string") + t.eq(#consumption.consumption_fingerprint_sha256, 64) end t.eq(at_barrier:terminal_record(), nil) local ownership = at_barrier:_fixture_effect("fixture-resource-status", { @@ -233,6 +236,7 @@ return { t.eq(released.process_group_absent, true) t.eq(released.listeners_closed, true) t.eq(released.workspace_absent, true) + t.eq(released.worker_environment_absent, true) t.eq(support.read_file(context.workspace_root .. "/state/inventory.json"), nil) local before = counts(recovered) diff --git a/examples/generic-host/tests/durable_host_store_test.lua b/examples/generic-host/tests/durable_host_store_test.lua index 9171d7e6..1e853f0d 100644 --- a/examples/generic-host/tests/durable_host_store_test.lua +++ b/examples/generic-host/tests/durable_host_store_test.lua @@ -79,6 +79,78 @@ local function generic_runtime_cli() end return { + test_materialized_immutable_artifact_publish_is_atomic_no_clobber = function() + local root = os.tmpname() .. "-generic-host-no-clobber" + cleanup(root) + local target = root .. "/receipt.json" + local script = table.concat({ + "const fs=require('fs'),path=require('path'),runtime=require(path.resolve(process.argv[1]));", + "const target=process.argv[2],sentinel='external-sentinel\\n',original=fs.linkSync;", + "fs.linkSync=(source,destination)=>{fs.writeFileSync(destination,sentinel,{flag:'wx'});", + "return original(source,destination)};let blocked=false;", + "try{runtime.materializeImmutableNoReplace(target,'trusted-receipt')}catch(error){", + "blocked=String(error&&error.message||error).includes('materialized artifact differs')}", + "if(!blocked||fs.readFileSync(target,'utf8')!==sentinel)process.exit(41);", + "if(fs.readdirSync(path.dirname(target)).some(name=>name.includes('.tmp-')))process.exit(42);", + }) + local result = direct_exec({ "node", "-e", script, generic_runtime_cli(), target }) + t.eq(result.exit_code, 0) + t.eq(read_file(target), "external-sentinel\n") + cleanup(root) + end, + + test_materialized_immutable_artifact_rejects_symlink_parent = function() + local root = os.tmpname() .. "-generic-host-symlink-parent" + local outside = os.tmpname() .. "-generic-host-symlink-outside" + cleanup(root) + cleanup(outside) + local target = root .. "/linked/receipt.json" + local script = table.concat({ + "const fs=require('fs'),path=require('path'),runtime=require(path.resolve(process.argv[1]));", + "const root=process.argv[2],outside=process.argv[3],target=root+'/linked/receipt.json';", + "fs.mkdirSync(root,{recursive:true});fs.mkdirSync(outside,{recursive:true});", + "fs.symlinkSync(outside,root+'/linked','dir');let blocked=false;", + "try{runtime.materializeImmutableNoReplace(target,'trusted-receipt',root)}catch(error){", + "blocked=String(error&&error.message||error).includes('physical directory')}", + "if(!blocked||fs.existsSync(outside+'/receipt.json'))process.exit(43);", + }) + local result = direct_exec({ "node", "-e", script, generic_runtime_cli(), root, outside }) + t.eq(result.exit_code, 0) + t.eq(read_file(outside .. "/receipt.json"), nil) + cleanup(root) + cleanup(outside) + end, + + test_materialized_immutable_artifact_publish_is_anchored_during_parent_swap = function() + local root = os.tmpname() .. "-generic-host-parent-swap" + local parked = root .. "-parked" + local outside = root .. "-outside" + cleanup(root) + cleanup(parked) + cleanup(outside) + local target = root .. "/receipt.json" + local script = table.concat({ + "const fs=require('fs'),path=require('path'),runtime=require(path.resolve(process.argv[1]));", + "const root=process.argv[2],parked=process.argv[3],outside=process.argv[4];", + "fs.mkdirSync(root,{recursive:true});fs.mkdirSync(outside,{recursive:true});", + "const original=fs.linkSync;fs.linkSync=(source,destination)=>{", + "fs.renameSync(root,parked);fs.renameSync(outside,root);", + "try{return original(source,destination)}finally{fs.renameSync(root,outside);fs.renameSync(parked,root)}};", + "runtime.materializeImmutableNoReplace(root+'/receipt.json','trusted-receipt',root);", + "if(fs.readFileSync(root+'/receipt.json','utf8')!=='trusted-receipt')process.exit(44);", + "if(fs.existsSync(outside+'/receipt.json'))process.exit(45);", + }) + local result = direct_exec({ + "node", "-e", script, generic_runtime_cli(), root, parked, outside, + }) + t.eq(result.exit_code, 0) + t.eq(read_file(target), "trusted-receipt") + t.eq(read_file(outside .. "/receipt.json"), nil) + cleanup(root) + cleanup(parked) + cleanup(outside) + end, + test_generic_host_runtime_echoes_exact_request_id_for_success_and_error = function() local root = os.tmpname() .. "-generic-host-runtime-correlation" cleanup(root) @@ -335,7 +407,8 @@ return { t.eq(consumption.grant_id, grant.value.grant_id) t.eq(consumption.binding.fence_id, envelope.fence_id) t.eq(consumption.binding.envelope_sha256, receipt.envelope_sha256) - t.eq(replay.claim_id, envelope.fence_id) + t.eq(replay.fence_id, envelope.fence_id) + t.is_true(replay.claim_id ~= envelope.fence_id) t.eq(replay.binding.grant_id, grant.value.grant_id) t.eq(replay.binding.grant_sha256, envelope.grant_sha256) t.eq(replay.binding.plan_sha256, envelope.plan_sha256) diff --git a/examples/generic-host/tests/durable_workflow_qa_crash_matrix_test.lua b/examples/generic-host/tests/durable_workflow_qa_crash_matrix_test.lua index 40372fb4..63a6799b 100644 --- a/examples/generic-host/tests/durable_workflow_qa_crash_matrix_test.lua +++ b/examples/generic-host/tests/durable_workflow_qa_crash_matrix_test.lua @@ -130,6 +130,7 @@ local function assert_barrier(context, case) t.eq(released.process_group_absent, true) t.eq(released.listeners_closed, true) t.eq(released.workspace_absent, true) + t.eq(released.worker_environment_absent, true) elseif case.name == "publication-after-effect" then local effect = matching_effect(recovered, "test-publication/effects", aggregate_effect) t.eq(effect.value.result.status, "materialized") diff --git a/examples/generic-host/tests/durable_workflow_qa_recovery_test.lua b/examples/generic-host/tests/durable_workflow_qa_recovery_test.lua index d3bb6da3..aeb4157e 100644 --- a/examples/generic-host/tests/durable_workflow_qa_recovery_test.lua +++ b/examples/generic-host/tests/durable_workflow_qa_recovery_test.lua @@ -1,4 +1,5 @@ local durable = require("host_durable_workflow_qa") +local lineage = require("contract.execution_authorization_lineage") local support = require("host_canonical_workflow_qa") local process = require("test_support.durable_workflow_qa_process") local supervisor_support = require("test_support.host_workflow_qa_supervisor") @@ -32,6 +33,57 @@ local function assert_counts_equal(left, right) for key, value in pairs(left) do t.eq(right[key], value) end end +local function assert_authorization_lineage(context, recovered) + local artifacts, expected = recovered:authorization_lineage_evidence() + for name, suffix in pairs(lineage.paths) do + local ref = context.artifact_root .. "/" .. suffix + local artifact = recovered.store:load(ref) + t.is_true(type(artifact) == "table" and type(artifact.value) == "table") + t.eq(artifacts[name].ref, ref) + t.eq(artifacts[name].sha256, artifact.digest) + t.eq(artifact.raw:sub(-1), "}") + t.is_true(artifact.raw:find('"claim_id"', 1, true) == nil) + t.is_true(artifact.raw:find('"fence_id"', 1, true) == nil) + t.is_true(artifact.raw:find('"mac"', 1, true) == nil) + t.is_true(artifact.raw:find(context.workspace_root, 1, true) == nil) + t.is_true(artifact.value.authorization_capability == false) + t.is_true(artifact.value.reusable == false) + t.eq(artifact.value.source_max_uses, 1) + end + local index_ref = context.artifact_root .. "/authorization-lineage/index.json" + local index = recovered.store:load(index_ref) + t.is_true(type(index) == "table" and type(index.value) == "table") + t.eq(lineage.validate_lineage_index(index.value, artifacts, expected), index.value) + t.eq(index.raw:sub(-1), "}") + t.eq(index.value.lineage_complete, true) + t.eq(#recovered.records:list("generic-host/profile-approval"), 1) + t.eq(#recovered.records:list("generic-host/preauthorization"), 1) + t.eq(#recovered.records:list("testing-runner/grant-verifications"), 1) + t.eq(#recovered.records:list("testing-runner/replay"), 1) + local profile_claim = recovered.records:list("generic-host/profile-approval")[1].value + local preauthorization_claim = recovered.records:list("generic-host/preauthorization")[1].value + local replay = recovered.records:list("testing-runner/replay")[1] + for _, private_id in ipairs({ + profile_claim.claim_id, preauthorization_claim.claim_id, replay.value.claim_id, + }) do + t.is_true(type(private_id) == "string" and private_id:sub(1, 14) == "private-claim-") + t.is_true(private_id ~= context.run_id .. "-profile-claim") + t.is_true(private_id ~= context.run_id .. "-preauthorization") + t.is_true(private_id ~= context.run_id .. "-execution-claim") + for _, artifact in pairs(artifacts) do + t.is_true(recovered.store:load(artifact.ref).raw:find(private_id, 1, true) == nil) + end + end + local public_guess = artifacts.execution_claim.value.receipt_id + local rejected = recovered.records:complete_replay( + replay.key, public_guess, replay.value.completion) + t.eq(rejected.completed, false) + rejected = recovered.records:complete_replay( + replay.key, artifacts.execution_claim.value.claim_fingerprint_sha256, + replay.value.completion) + t.eq(rejected.completed, false) +end + local function state_diagnostic(context) local ok, recovered = pcall(durable.load, context.project_root, context.durable_root, context.run_id) if not ok then return "state_load_error=" .. tostring(recovered) end @@ -131,7 +183,9 @@ local function assert_terminal(context) t.eq(released.process_group_absent, true) t.eq(released.listeners_closed, true) t.eq(released.workspace_absent, true) + t.eq(released.worker_environment_absent, true) t.eq(#recovered.records:list("generic-host/terminal"), 1) + assert_authorization_lineage(context, recovered) return recovered end @@ -140,6 +194,7 @@ local function mutate_record(path, mode) "const fs=require('fs'),path=process.argv[1],mode=process.argv[2],value=JSON.parse(fs.readFileSync(path,'utf8'));", "if(mode==='request')value.issue.number+=1;", "else if(mode==='authorization')value.authorization.profile_sha256='0'.repeat(64);", + "else if(mode==='profile-claim')value.binding.profile_sha256='0'.repeat(64);", "else if(mode==='execution')value.binding.trace_id+='-mutated';", "else process.exit(51);", "const next=path+'.mutation-'+process.pid;fs.writeFileSync(next,JSON.stringify(value)+'\\n',{flag:'wx'});fs.renameSync(next,path);", @@ -154,6 +209,36 @@ local function replay_record_path(context, recovered) return context.durable_run_root .. "/records/" .. entries[1].key .. ".json" end +local function substitute_completed_execution_plan(context, recovered) + local replay = recovered.records:list("testing-runner/replay")[1].value + local script = table.concat({ + "const crypto=require('crypto'),fs=require('fs'),path=require('path');", + "const root=process.argv[1],replayPath=process.argv[2],resultRef=process.argv[3];", + "const sha=v=>crypto.createHash('sha256').update(v).digest('hex');", + "const file=ref=>path.join(root,'artifacts',sha(ref)+'.json');", + "const read=ref=>JSON.parse(fs.readFileSync(file(ref),'utf8'));", + "const write=(ref,value)=>{const target=file(ref),record=read(ref),body=JSON.stringify(value)+'\\n';record.body=body;record.digest=sha(body);fs.writeFileSync(target,JSON.stringify(record)+'\\n');return record.digest};", + "const executionRecord=read(resultRef),execution=JSON.parse(executionRecord.body);", + "const planRecord=read(execution.test_plan_path),plan=JSON.parse(planRecord.body);plan.foreign_plan_identity='substituted';", + "const planDigest=write(execution.test_plan_path,plan);", + "const manifestRecord=read(execution.evidence_manifest_path),manifest=JSON.parse(manifestRecord.body);manifest.plan_sha256=planDigest;", + "const manifestDigest=write(execution.evidence_manifest_path,manifest);", + "const setRecord=read(execution.case_result_set_path),set=JSON.parse(setRecord.body);set.plan_sha256=planDigest;", + "for(const item of set.cases||[])item.plan_sha256=planDigest;", + "set.evidence_manifest_ref.sha256=manifestDigest;set.evidence_manifest_artifact_sha256=manifestDigest;", + "const setDigest=write(execution.case_result_set_path,set);", + "execution.plan_sha256=planDigest;execution.case_result_set_artifact_sha256=setDigest;execution.evidence_manifest_artifact_sha256=manifestDigest;", + "const resultDigest=write(resultRef,execution);", + "const claim=JSON.parse(fs.readFileSync(replayPath,'utf8'));claim.result_sha256=resultDigest;claim.completion.result_sha256=resultDigest;", + "fs.writeFileSync(replayPath,JSON.stringify(claim)+'\\n');", + }) + local result = process.exec({ + "node", "-e", script, context.durable_run_root, + replay_record_path(context, recovered), replay.result_ref, + }) + if result.exit_code ~= 0 then error("generic-host recovery test: execution Plan substitution failed") end +end + local function mutation_case(mode, expected_fragment, expected_stream) with_context({ cli_only = true, @@ -167,6 +252,9 @@ local function mutation_case(mode, expected_fragment, expected_stream) mutate_record(context.durable_run_root .. "/records/workflow-qa/requests/" .. context.run_id .. ".json", mode) elseif mode == "authorization" then mutate_record(context.durable_run_root .. "/records/workflow-qa/state/" .. context.run_id .. ".json", mode) + elseif mode == "profile-claim" then + mutate_record(context.durable_run_root .. "/records/generic-host/profile-approval/" + .. context.run_id .. ".json", mode) else local state = recovered.workflow_runtime.load_state(recovered.request.state_ref) execution_request = support.copy(state.pending_actions[1].payload) @@ -176,10 +264,17 @@ local function mutation_case(mode, expected_fragment, expected_stream) local label = mode .. "-second" local pid, stdout_path, stderr_path = start_supervisor(context, label, false, live_pids) local observed_path = expected_stream == "stdout" and stdout_path or stderr_path - local observed = mode == "execution" - and wait_for_child_text(context.host_root .. "/framework-runtime-" .. label .. "/logs/framework-child", - "testing-runner.run_structured_execution-", expected_fragment, 45) - or wait_for_text(observed_path, expected_fragment, 45) + local child_logs = context.host_root .. "/framework-runtime-" .. label .. "/logs/framework-child" + local observed + if mode == "execution" then + observed = wait_for_child_text( + child_logs, "testing-runner.run_structured_execution-", expected_fragment, 45) + elseif mode == "profile-claim" then + observed = wait_for_child_text( + child_logs, "environment-factory.finalize-", expected_fragment, 45) + else + observed = wait_for_text(observed_path, expected_fragment, 45) + end if not observed then error("generic-host recovery test: mutation did not fail closed: " .. mode .. "\nstdout=" .. tostring(read_file(stdout_path)) .. "\nstderr=" .. tostring(read_file(stderr_path))) @@ -248,6 +343,35 @@ return { end) end, + test_durable_workflow_qa_rejects_foreign_plan_during_result_recovery_and_lineage_projection = function() + with_context({ + cli_only = true, + count_effect = true, + durable = true, + arm_completed_replay_failpoint = true, + }, function(context, live_pids) + local recovered = run_to_barrier(context, live_pids, "foreign-plan-first") + substitute_completed_execution_plan(context, recovered) + recovered = durable.load(context.project_root, context.durable_root, context.run_id) + local claim = recovered.records:list("testing-runner/replay")[1].value + local binding = claim.binding + local summary = recovered.structured_runtime.load_result({ + artifact_root = binding.artifact_root, + result_ref = claim.result_ref, + result_sha256 = claim.result_sha256, + operation_id = binding.operation_id, + repository = support.copy(binding.repository), + environment_receipt_sha256 = binding.environment_receipt_sha256, + trace_id = binding.trace_id, + dedup_key = binding.dedup_key, + }) + t.eq(summary, nil) + local ok, failure = pcall(function() recovered:authorization_lineage_evidence() end) + t.eq(ok, false) + t.is_true(tostring(failure):find("completion Plan binding differs", 1, true) ~= nil) + end) + end, + test_durable_workflow_qa_unarmed_supervisor_reaches_terminal_without_pausing = function() with_context({ cli_only = true, count_effect = true, durable = true }, function(context, live_pids) local pid, stdout_path, stderr_path = start_supervisor(context, "unarmed", false, live_pids) @@ -262,6 +386,74 @@ return { end) end, + test_durable_workflow_qa_node_runtime_recovers_legacy_preauthorization_claim = function() + with_context({ + cli_only = true, + count_effect = true, + durable = true, + prepare_execution_grant_pending = true, + }, function(context, live_pids) + local recovered = durable.load(context.project_root, context.durable_root, context.run_id) + local state = recovered.workflow_runtime.load_state(recovered.request.state_ref) + t.eq(state.phase, "execution-grant-pending") + t.eq(state.pending_actions[1].queue, "workflow_qa_execution_grant_request") + local request = support.copy(state.pending_actions[1].payload) + local preauthorization = recovered.store:load(request.preauthorization_ref) + local claim_request = { + authorization_id = preauthorization.value.authorization_id, + preauthorization_ref = request.preauthorization_ref, + preauthorization_sha256 = request.preauthorization_sha256, + repository = support.copy(request.repository), + plan_ref = request.plan_ref, + plan_sha256 = request.plan_sha256, + environment_receipt_ref = request.environment_receipt_ref, + environment_receipt_sha256 = request.environment_receipt_sha256, + trace_id = request.trace_id, + dedup_key = request.dedup_key, + } + local private_claim_id = recovered:_private_claim_id("legacy-preauthorization") + local legacy_binding = support.copy(claim_request) + legacy_binding.preauthorization_ref = nil + legacy_binding.plan_ref = nil + legacy_binding.environment_receipt_ref = nil + local key = "generic-host/preauthorization/" .. recovered:_key(claim_request.authorization_id) + local stored = recovered.records:claim(key, { + binding = legacy_binding, + claim_id = private_claim_id, + claimed_at = recovered.execution_authorization_now, + }) + t.eq(stored.claimed, true) + t.eq(#recovered.records:list("testing-runner/grant-verifications"), 0) + + local replayed = recovered:_fixture_effect("host-claim-preauthorization", claim_request) + t.eq(replayed.status, "claimed") + t.eq(replayed.replayed, true) + t.eq(replayed.claim_id, private_claim_id) + t.eq(#recovered.records:list("generic-host/preauthorization"), 1) + t.eq(#recovered.records:list("testing-runner/grant-verifications"), 0) + + local substituted = support.copy(claim_request) + substituted.plan_ref = context.artifact_root .. "/execution/foreign-plan.json" + local blocked = recovered:_fixture_effect("host-claim-preauthorization", substituted) + t.eq(blocked.status, "blocked") + t.eq(#recovered.records:list("generic-host/preauthorization"), 1) + + local pid, stdout_path, stderr_path = start_supervisor( + context, "legacy-preauthorization-recovery", false, live_pids) + if not wait_for_terminal(context) then + error("generic-host recovery test: legacy Preauthorization recovery did not reach terminal " + .. state_diagnostic(context) .. "\nstdout=" .. tostring(read_file(stdout_path)) + .. "\nstderr=" .. tostring(read_file(stderr_path))) + end + stop_live(pid, live_pids) + local terminal = assert_terminal(context) + t.eq(#terminal.records:list("generic-host/preauthorization"), 1) + t.eq(#terminal.records:list("testing-runner/grant-verifications"), 1) + t.eq(terminal.records:list("generic-host/preauthorization")[1].value.claim_id, + private_claim_id) + end) + end, + test_durable_workflow_qa_production_pep_denial_reaches_blocked_terminal_without_cli_effect = function() with_context({ cli_only = true, @@ -334,6 +526,10 @@ return { mutation_case("authorization", "authorization-binding-changed", "stdout") end, + test_durable_workflow_qa_profile_claim_binding_mutation_fails_closed = function() + mutation_case("profile-claim", "environment authorization approval claim is unavailable", "stderr") + end, + test_durable_workflow_qa_execution_binding_mutation_fails_closed = function() mutation_case("execution", "testing-runner dept=run_structured_execution tag=BLOCKED", "stdout") end, diff --git a/examples/generic-host/tests/workflow_qa_adapter_test.lua b/examples/generic-host/tests/workflow_qa_adapter_test.lua index 317e5d2a..4f5ebcd5 100644 --- a/examples/generic-host/tests/workflow_qa_adapter_test.lua +++ b/examples/generic-host/tests/workflow_qa_adapter_test.lua @@ -199,6 +199,13 @@ local function runtime(request, materials, mutate_artifacts) claim = claim or { value = value, claim_id = "controlled-preauthorization-claim" } return { status = "claimed", claim_id = claim.claim_id, replayed = claim_calls > 1 } end, + reconcile_preauthorization_claim = function(value) + if claim == nil then return false end + for key, item in pairs(value) do + if not execution.equal(claim.value[key], item) then return false end + end + return true + end, grant_values = function() return values() end, record_terminal = function(value) if terminal ~= nil and not workflow_qa.same_request(terminal, value) then return false end diff --git a/libraries/testing_runtime/authorization_lineage_projection.lua b/libraries/testing_runtime/authorization_lineage_projection.lua new file mode 100644 index 00000000..f2e940a3 --- /dev/null +++ b/libraries/testing_runtime/authorization_lineage_projection.lua @@ -0,0 +1,168 @@ +local canonical_json = require("contract.canonical_json") +local lineage = require("contract.execution_authorization_lineage") + +local M = {} + +local Projector = {} +Projector.__index = Projector + +local validators = { + profile_claim = lineage.validate_profile_claim_receipt, + preauthorization_claim = lineage.validate_preauthorization_claim_receipt, + grant_verification = lineage.validate_grant_verification_receipt, + execution_claim = lineage.validate_execution_claim_receipt, + execution_completion = lineage.validate_execution_completion_receipt, +} + +local statuses = { + profile_claim = "claimed", + preauthorization_claim = "claimed", + grant_verification = "authenticated", + execution_claim = "claimed", + execution_completion = "completed", +} + +local function copy(value) + if type(value) ~= "table" then return value end + local result = {} + for key, item in pairs(value) do result[copy(key)] = copy(item) end + return result +end + +local function fail(message) + error("testing-runtime: authorization-lineage-projection: " .. message, 0) +end + +local function require_identity(value, label) + if type(value) ~= "string" or value == "" or #value > 180 + or value:find("%s") ~= nil or value:find("[%z\1-\31\127]") ~= nil then + fail(label .. " is invalid") + end + return value +end + +function M.new(options) + if type(options) ~= "table" or type(options.store) ~= "table" + or type(options.store.load) ~= "function" or type(options.store.write_raw) ~= "function" + or type(options.sha256) ~= "function" or type(options.fingerprint_secret) ~= "string" + or #options.fingerprint_secret < 32 then + fail("store, sha256, and a private fingerprint secret are required") + end + require_identity(options.run_id, "run_id") + require_identity(options.trace_id, "trace_id") + require_identity(options.dedup_key, "dedup_key") + local root = ".testing/runs/" .. options.run_id + if options.artifact_root ~= root then fail("artifact_root must be the canonical run root") end + return setmetatable({ + store = options.store, + sha256 = options.sha256, + fingerprint_secret = options.fingerprint_secret, + repository = copy(options.repository), + run_id = options.run_id, + trace_id = options.trace_id, + dedup_key = options.dedup_key, + artifact_root = root, + }, Projector) +end + +function Projector:path(name) + local suffix = lineage.paths[name] + if suffix == nil then fail("unsupported receipt name " .. tostring(name)) end + return self.artifact_root .. "/" .. suffix +end + +function Projector:fingerprint(domain, private_claim_id) + require_identity(domain, "fingerprint domain") + require_identity(private_claim_id, "private claim id") + local inner = self.sha256(self.fingerprint_secret .. "\0" .. domain .. "\0" .. private_claim_id) + return self.sha256("fkst-authorization-lineage.v1\0" .. domain .. "\0" .. inner) +end + +function Projector:_persist(path, value) + local body = canonical_json.encode(value) + local digest = self.sha256(body) + local existing = self.store:load(path) + if existing == nil then + if self.store:write_raw(path, body) ~= true then fail("immutable receipt write failed") end + existing = self.store:load(path) + end + if type(existing) ~= "table" or existing.raw ~= body or existing.digest ~= digest then + fail("immutable receipt differs at " .. path) + end + return { ref = path, sha256 = digest, value = copy(value) } +end + +function Projector:write_receipt(name, receipt_id, recorded_at, fields, expected) + local validator = validators[name] + if validator == nil or type(fields) ~= "table" or type(expected) ~= "table" then + fail("receipt fields and independent source bindings are required") + end + local value = { + schema = lineage.schemas[name], + status = statuses[name], + receipt_id = require_identity(receipt_id, "receipt_id"), + repository = copy(self.repository), + run_id = self.run_id, + trace_id = self.trace_id, + dedup_key = self.dedup_key, + recorded_at = recorded_at, + source_max_uses = 1, + evidence_role = "audit-only", + authorization_capability = false, + reusable = false, + } + for key, item in pairs(fields) do + if value[key] ~= nil then fail("receipt field shadows envelope: " .. tostring(key)) end + value[key] = copy(item) + end + validator(value, expected) + return self:_persist(self:path(name), value), expected +end + +function Projector:load_receipt(name, expected) + if type(expected) ~= "table" then fail("independent source bindings are required") end + local artifact = self.store:load(self:path(name)) + if artifact == nil or type(artifact.value) ~= "table" then + fail("required predecessor receipt is unavailable: " .. tostring(name)) + end + local canonical = canonical_json.encode(artifact.value) + local digest = self.sha256(canonical) + if artifact.raw ~= canonical or artifact.digest ~= digest then + fail("predecessor receipt is not canonical: " .. tostring(name)) + end + validators[name](artifact.value, expected) + return { ref = self:path(name), sha256 = digest, value = copy(artifact.value) } +end + +function Projector:write_index(recorded_at, artifacts, expected) + if type(artifacts) ~= "table" or type(expected) ~= "table" then + fail("complete trusted receipt artifacts and source bindings are required") + end + local receipts = {} + for _, name in ipairs({ + "profile_claim", "preauthorization_claim", "grant_verification", + "execution_claim", "execution_completion", + }) do + artifacts[name] = self:load_receipt(name, expected[name]) + receipts[name] = { ref = artifacts[name].ref, sha256 = artifacts[name].sha256 } + end + local value = { + schema = lineage.schemas.lineage_index, + status = "complete", + repository = copy(self.repository), + run_id = self.run_id, + trace_id = self.trace_id, + dedup_key = self.dedup_key, + recorded_at = recorded_at, + receipts = receipts, + lineage_complete = true, + source_max_uses = 1, + evidence_role = "audit-only", + authorization_capability = false, + reusable = false, + } + lineage.validate_lineage_index(value, artifacts, expected) + return self:_persist(self.artifact_root .. "/authorization-lineage/index.json", value) +end + +return M diff --git a/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js b/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js index 029fe0cc..77a67396 100644 --- a/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js +++ b/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js @@ -13,12 +13,17 @@ const { minimalEnvironment, parseArgs, readJson, + releaseWorkerEnvironment, sha256, stableStringify, validateArgv, + verifyWorkerEnvironment, writeJsonAtomic, writeJsonImmutable, } = require('../../../packages/environment-factory/bin/runtime/common'); +const { + validateTargetExecutionBoundary, +} = require('../../../packages/environment-factory/bin/runtime/target-execution-boundary'); const { runMeasuredCommand } = require('../../../packages/environment-factory/bin/runtime/measured-command'); const { processGroupUsage } = require('../../../packages/environment-factory/bin/runtime/platform'); const { resolveWorkspace } = require('../../../packages/environment-factory/bin/runtime/workspace'); @@ -40,6 +45,16 @@ function runtimeConfig(payload) { || config.state_mac_generation.length > 180) { throw new Error('structured execution config requires state_mac_generation'); } + validateTargetExecutionBoundary(config.target_execution_boundary); + return config; +} + +function targetExecutionConfig(payload, repository) { + const config = runtimeConfig(payload); + validateTargetExecutionBoundary(config.target_execution_boundary, repository, { + runtimeConfigRef: payload.runtime_config_ref, + artifactRoot: payload.artifact_root, + }); return config; } @@ -138,6 +153,36 @@ function writeReplay(config, grantId, value) { }); } +function completedReplayForResult(config, payload) { + const directory = path.join(durableRoot(), 'testing-runner', 'structured-execution'); + if (!fs.existsSync(directory)) throw new Error('authenticated completed replay claim is unavailable'); + const matches = []; + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + if (!entry.isFile() || !/^[0-9a-f]{64}\.json$/.test(entry.name)) continue; + const envelope = readJson(path.join(directory, entry.name)); + if (!envelope || envelope.schema !== 'testing-runtime.structured-execution-replay.v1' + || envelope.mac !== replayMac(config, envelope.value)) { + throw new Error('structured execution replay state authentication failed'); + } + const value = envelope.value; + const binding = value && value.binding; + if (!binding || entry.name !== `${sha256(binding.grant_id)}.json`) { + throw new Error('structured execution replay state identity differs'); + } + if (value.status === 'completed' && value.result_ref === payload.result_ref + && value.result_sha256 === payload.result_sha256 + && binding.artifact_root === payload.artifact_root + && binding.operation_id === payload.operation_id + && binding.environment_receipt_sha256 === payload.environment_receipt_sha256 + && sameRepository(binding.repository, payload.repository) + && binding.trace_id === payload.trace_id && binding.dedup_key === payload.dedup_key) { + matches.push(value); + } + } + if (matches.length !== 1) throw new Error('authenticated completed replay claim is unavailable or ambiguous'); + return matches[0]; +} + function sameRepository(left, right) { return Boolean(left && right && left.url === right.url && left.commit_sha === right.commit_sha); } @@ -455,6 +500,8 @@ function validateExecutionArtifact(payload, binding, expectedDigest) { || !statuses.has(value.status) || typeof value.classification !== 'string' || !sameRepository(value.repository, binding.repository) || value.environment_receipt_sha256 !== binding.environment_receipt_sha256 + || !/^[0-9a-f]{64}$/.test(String(value.plan_sha256 || '')) + || value.plan_sha256 !== binding.plan_sha256 || value.trace_id !== binding.trace_id || value.dedup_key !== binding.dedup_key || value.test_plan_path !== `${binding.artifact_root}/test-plan.json` || value.case_results_path !== `${binding.artifact_root}/case-results.json` @@ -699,8 +746,8 @@ function evaluateCliEnvelope(config, envelope, now) { } function authorizeCliEffect(payload) { - const config = runtimeConfig(payload); const envelope = payload.action_envelope || {}; + const config = runtimeConfig(payload); const now = new Date(); let inputs = { profile: '0'.repeat(64), validation_receipt: '0'.repeat(64), @@ -708,6 +755,10 @@ function authorizeCliEffect(payload) { plan: '0'.repeat(64), grant: '0'.repeat(64), }; try { + validateTargetExecutionBoundary(config.target_execution_boundary, envelope.repository, { + runtimeConfigRef: payload.runtime_config_ref, + artifactRoot: payload.artifact_root, + }); inputs = evaluateCliEnvelope(config, envelope, now); const receipt = authorizationReceipt(config, envelope, 'allow', 'authorized', inputs, now); const target = authorizationPath(receipt.receipt_id); @@ -743,8 +794,8 @@ function authorizeCliEffect(payload) { } async function execArgv(payload) { - const config = runtimeConfig(payload); const envelope = validateEnvelope(payload.action_envelope); + const config = targetExecutionConfig(payload, envelope.repository); const receipt = payload.authorization_receipt; exactKeys(receipt, [ 'schema', 'decision', 'reason_code', 'receipt_id', 'envelope_sha256', @@ -779,22 +830,36 @@ async function execArgv(payload) { environment_receipt_sha256: envelope.environment_receipt_sha256, workspace_ref: envelope.workspace_ref, require_clean: true, }); - const result = await runMeasuredCommand(validateArgv(envelope.case.argv), { - cwd: workspace.cwd, - env: minimalEnvironment(config.command_environment || {}), - timeoutMs: envelope.case.timeout_seconds * 1000, - outputBytes: Math.min(boundedOutput(config), envelope.resource_bounds.output_bytes), + const environment = minimalEnvironment(config.command_environment || {}, { + schema: 'testing-runtime.structured-cli-isolation.v1', + operation_id: envelope.operation_id, + run_id: envelope.run_id, + repository: envelope.repository, + case_id: envelope.case.case_id, + attempt: envelope.attempt, + purpose: 'structured-cli', }); - if (result.timedOut) throw new Error('structured CLI effect timed out'); - if (result.outputExceeded) throw new Error('structured CLI effect exceeded output bound'); - if (result.error) throw result.error; - const remaining = processGroupUsage([result.pgid]); - if (!remaining.supported) throw new Error('structured CLI process cleanup verification is unavailable'); - if (remaining.processes > 0) { - try { process.kill(-result.pgid, 'SIGKILL'); } catch (_error) {} - throw new Error('structured CLI effect left a surviving process group'); + try { + verifyWorkerEnvironment(environment); + const result = await runMeasuredCommand(validateArgv(envelope.case.argv), { + cwd: workspace.cwd, + env: environment, + timeoutMs: envelope.case.timeout_seconds * 1000, + outputBytes: Math.min(boundedOutput(config), envelope.resource_bounds.output_bytes), + }); + if (result.timedOut) throw new Error('structured CLI effect timed out'); + if (result.outputExceeded) throw new Error('structured CLI effect exceeded output bound'); + if (result.error) throw result.error; + const remaining = processGroupUsage([result.pgid]); + if (!remaining.supported) throw new Error('structured CLI process cleanup verification is unavailable'); + if (remaining.processes > 0) { + try { process.kill(-result.pgid, 'SIGKILL'); } catch (_error) {} + throw new Error('structured CLI effect left a surviving process group'); + } + return { exit_code: result.exitCode, stdout: result.stdout, stderr: result.stderr }; + } finally { + releaseWorkerEnvironment(environment); } - return { exit_code: result.exitCode, stdout: result.stdout, stderr: result.stderr }; } finally { release(); } } @@ -808,7 +873,7 @@ function localOrigin(value) { } function httpRequest(payload) { - const config = runtimeConfig(payload); + const config = targetExecutionConfig(payload, payload.repository); const allowedMethods = new Set(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE']); if (!payload.request || !allowedMethods.has(payload.request.method) || !Array.isArray(payload.request.headers) || payload.request.headers.length !== 0) { @@ -852,15 +917,9 @@ function loadResult(payload) { if (!/^[0-9a-f]{64}$/.test(String(payload.result_sha256 || ''))) { throw new Error('completed execution result digest is required'); } - const binding = { - artifact_root: payload.artifact_root, - operation_id: payload.operation_id, - repository: payload.repository, - environment_receipt_sha256: payload.environment_receipt_sha256, - trace_id: payload.trace_id, - dedup_key: payload.dedup_key, - }; - const artifact = validateExecutionArtifact(payload, binding, payload.result_sha256); + const config = runtimeConfig(payload); + const replay = completedReplayForResult(config, payload); + const artifact = validateExecutionArtifact(payload, replay.binding, payload.result_sha256); const execution = artifact.value; const result = { schema: 'testing-runner.structured-execution-summary.v1', diff --git a/libraries/testing_runtime/generic_host_workflow_qa.lua b/libraries/testing_runtime/generic_host_workflow_qa.lua index d5d7a577..0b6d4dbb 100644 --- a/libraries/testing_runtime/generic_host_workflow_qa.lua +++ b/libraries/testing_runtime/generic_host_workflow_qa.lua @@ -22,8 +22,12 @@ function M.configured(options) return client(options).configured() end function M.production(options) local cli = client(options) return { - load_artifact = function(path) - return cli.call("artifact-load", { path = path }, run_root(path), path, 15) + load_artifact = function(path, expected_digest, options) + return cli.call("artifact-load", { + path = path, + expected_digest = expected_digest, + durable_only = type(options) == "table" and options.durable_only == true or nil, + }, run_root(path), path, 15) end, write_artifact = function(path, value) local result = cli.call("artifact-write", { path = path, value = value }, run_root(path), path, 15) @@ -39,6 +43,10 @@ function M.production(options) claim_preauthorization = function(value) return cli.call("host-claim-preauthorization", value, nil, value.dedup_key, 15) end, + reconcile_preauthorization_claim = function(value) + local result = cli.call("host-reconcile-preauthorization-claim", value, nil, value.dedup_key, 15) + return type(result) == "table" and result.reconciled == true + end, grant_values = function(request, materials) return cli.call("host-grant-values", { request = request, materials = materials }, nil, request.dedup_key, 15) diff --git a/libraries/testing_runtime/tests/structured_execution_runtime_test.js b/libraries/testing_runtime/tests/structured_execution_runtime_test.js index ddd772c6..51ea88ef 100644 --- a/libraries/testing_runtime/tests/structured_execution_runtime_test.js +++ b/libraries/testing_runtime/tests/structured_execution_runtime_test.js @@ -1,6 +1,7 @@ 'use strict'; const assert = require('assert'); +const crypto = require('crypto'); const fs = require('fs'); const http = require('http'); const os = require('os'); @@ -48,6 +49,11 @@ async function main() { const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'structured-runtime-test-')); const previousDurable = process.env.FKST_DURABLE_ROOT; const previousRuntime = process.env.FKST_RUNTIME_ROOT; + const ambientKeys = [ + 'HOME', 'GH_TOKEN', 'GITHUB_TOKEN', 'SSH_AUTH_SOCK', 'GIT_ASKPASS', 'SSH_ASKPASS', + 'GIT_CONFIG_GLOBAL', 'GIT_CONFIG_COUNT', 'GIT_CONFIG_KEY_0', 'GIT_CONFIG_VALUE_0', + ]; + const previousAmbient = Object.fromEntries(ambientKeys.map((key) => [key, process.env[key]])); process.env.FKST_DURABLE_ROOT = path.join(temp, 'durable'); process.env.FKST_RUNTIME_ROOT = path.join(temp, 'runtime'); const runId = `structured-runtime-${process.pid}`; @@ -79,13 +85,43 @@ async function main() { run(['git', 'commit', '--quiet', '-m', 'fixture'], source); repository.commit_sha = run(['git', 'rev-parse', 'HEAD'], source); + const ambientHome = path.join(temp, 'ambient-home'); + fs.mkdirSync(ambientHome); + fs.writeFileSync(path.join(ambientHome, '.gitconfig'), [ + '[url "file:///definitely-missing-worker-credential-isolation/"]', + `\tinsteadOf = ${source}`, + '', + ].join('\n')); + Object.assign(process.env, { + HOME: ambientHome, + GH_TOKEN: 'ambient-gh-token-canary', + GITHUB_TOKEN: 'ambient-github-token-canary', + SSH_AUTH_SOCK: path.join(temp, 'ambient-ssh-agent.sock'), + GIT_ASKPASS: path.join(temp, 'ambient-git-askpass'), + SSH_ASKPASS: path.join(temp, 'ambient-ssh-askpass'), + GIT_CONFIG_GLOBAL: path.join(ambientHome, '.gitconfig'), + GIT_CONFIG_COUNT: '1', + GIT_CONFIG_KEY_0: 'credential.helper', + GIT_CONFIG_VALUE_0: 'ambient-helper', + }); + fs.mkdirSync(path.dirname(environmentConfigRef), { recursive: true }); fs.writeFileSync(environmentConfigRef, `${stableStringify({ schema: 'environment-factory.runtime-config.v1', state_auth_key: 'environment-runtime-test-state-key-00000000000000000000', state_mac_generation: 'environment-runtime-test-v1', repository_mirrors: { [repository.url]: source }, - command_environment: {}, + command_environment: { FKST_RUNTIME_SAFE_MARKER: 'present' }, + target_execution_boundary: { + schema: 'testing-host.target-execution-boundary.v1', + mode: 'trusted-fixture-exact', + target_class: 'host-owned-exact-trusted-fixture', + repository, + authority: { kind: 'host-policy', ref: 'fixtures/structured-runtime-target-boundary' }, + policy_revision: 'structured-runtime-target-boundary-v1', + authorization_capability: false, + execution_authorized: false, + }, })}\n`); checkout = await environmentDispatch('checkout', { effect_id: `${operationId}/checkout`, @@ -105,6 +141,27 @@ async function main() { workspaceRef = checkout.workspace_ref; workspace = JSON.parse(fs.readFileSync(resourcePath(workspaceRef.ref), 'utf8')).path; + const fsmonitorCanary = path.join(temp, 'ambient-fsmonitor-canary.json'); + const fsmonitorHook = path.join(temp, 'ambient-fsmonitor-hook.js'); + fs.writeFileSync(fsmonitorHook, [ + '#!/usr/bin/env node', + "'use strict';", + "const fs=require('fs');", + `fs.writeFileSync(${JSON.stringify(fsmonitorCanary)}, JSON.stringify({`, + " gh:process.env.GH_TOKEN||null,github:process.env.GITHUB_TOKEN||null,", + " ssh:process.env.SSH_AUTH_SOCK||null,gitAskpass:process.env.GIT_ASKPASS||null,", + " sshAskpass:process.env.SSH_ASKPASS||null,home:process.env.HOME||null,", + '}));', + "process.stdout.write('\\n');", + '', + ].join('\n'), { mode: 0o700 }); + fs.writeFileSync(path.join(ambientHome, '.gitconfig'), [ + '[core]', + `\tfsmonitor = ${fsmonitorHook}`, + '', + ].join('\n')); + run(['git', 'config', 'core.fsmonitor', fsmonitorHook], workspace); + const traceId = `${runId}-trace`; const dedupKey = `${runId}-dedup`; const expiresAt = new Date(Date.now() + 60 * 60 * 1000).toISOString(); @@ -137,7 +194,15 @@ async function main() { }); const cliCase = { case_id: 'cli-version', kind: 'cli', - argv: [process.execPath, '-e', 'process.stdout.write(process.cwd())'], timeout_seconds: 10, + argv: [process.execPath, '-e', [ + "const path=require('path')", + "for(const key of ['GH_TOKEN','GITHUB_TOKEN','SSH_AUTH_SOCK','GIT_ASKPASS','SSH_ASKPASS'])if(process.env[key])process.exit(70)", + "const home=process.env.HOME||''", + "const nullDevice=process.platform==='win32'?'NUL':'/dev/null'", + "if(home===process.argv[1]||path.basename(path.dirname(home))!=='worker-homes')process.exit(71)", + "if(process.env.GIT_CONFIG_NOSYSTEM!=='1'||process.env.GIT_CONFIG_GLOBAL!==nullDevice||process.env.GIT_CONFIG_COUNT!=='4'||process.env.GIT_CONFIG_KEY_0!=='credential.helper'||process.env.GIT_CONFIG_VALUE_0!==''||process.env.GIT_CONFIG_KEY_1!=='core.askPass'||process.env.GIT_CONFIG_VALUE_1!==''||process.env.GIT_CONFIG_KEY_2!=='core.fsmonitor'||process.env.GIT_CONFIG_VALUE_2!=='false'||process.env.GIT_CONFIG_KEY_3!=='core.hooksPath'||process.env.GIT_CONFIG_VALUE_3!==nullDevice||process.env.GIT_TERMINAL_PROMPT!=='0'||process.env.FKST_RUNTIME_SAFE_MARKER!=='present')process.exit(72)", + 'process.stdout.write(process.cwd())', + ].join(';'), ambientHome], timeout_seconds: 10, assertions: [{ type: 'exit-code', expected: 0 }], }; const plan = writeAuthority('plan', { @@ -159,8 +224,19 @@ async function main() { fs.writeFileSync(configRef, `${stableStringify({ schema: 'testing-runtime.structured-execution-config.v1', state_auth_key: 'structured-runtime-test-state-key-00000000000000000000', - state_mac_generation: 'runtime-test-v1', command_environment: {}, + state_mac_generation: 'runtime-test-v1', + command_environment: { FKST_RUNTIME_SAFE_MARKER: 'present' }, output_bytes: 65536, http_response_bytes: 65536, + target_execution_boundary: { + schema: 'testing-host.target-execution-boundary.v1', + mode: 'trusted-fixture-exact', + target_class: 'host-owned-exact-trusted-fixture', + repository, + authority: { kind: 'host-policy', ref: 'fixtures/structured-runtime-target-boundary' }, + policy_revision: 'structured-runtime-target-boundary-v1', + authorization_capability: false, + execution_authorized: false, + }, grant_attestations: [{ grant_sha256: grantSha256, authority, policy_revision: 'runtime-test-policy-v1', evidence_ref: evidenceRef }], })}\n`); @@ -223,6 +299,7 @@ async function main() { }); assert.strictEqual(cli.exit_code, 0); assert.strictEqual(cli.stdout, fs.realpathSync(workspace)); + assert.strictEqual(fs.existsSync(fsmonitorCanary), false); await assert.rejects(() => dispatch('exec-argv', { ...common, action_envelope: actionEnvelope, authorization_receipt: authorization, }), /replayed or is unavailable/); @@ -266,6 +343,7 @@ async function main() { schema: 'testing-structured-execution.v1', operation_id: operationId, status: 'passed', classification: 'passed', repository, environment_receipt_sha256: common.environment_receipt_sha256, + plan_sha256: 'd'.repeat(64), trace_id: common.trace_id, dedup_key: common.dedup_key, case_count: 1, passed_count: 1, failed_count: 0, skipped_count: 0, error_count: 0, test_plan_path: `${artifactRoot}/test-plan.json`, @@ -293,6 +371,11 @@ async function main() { await assert.rejects(() => dispatch('replay-guard', { ...historicalClaimRequest, artifact_root: `.testing/runs/${runId}-foreign/execution`, }), /replay binding differs/); + persistJson(resultRef, { ...historicalExecution, plan_sha256: 'e'.repeat(64) }); + await assert.rejects(() => dispatch('complete-replay', { + ...common, claim: historicalClaim, result_ref: resultRef, + }), /execution result binding is invalid/); + persistJson(resultRef, historicalExecution); const historicalCompletion = await dispatch('complete-replay', { ...common, claim: historicalClaim, result_ref: resultRef, }); @@ -539,6 +622,25 @@ async function main() { assert.strictEqual(fs.readFileSync(caseResultSetPath, 'utf8'), validResultSetArtifact.raw); assert.strictEqual(fs.readFileSync(evidenceManifestPath, 'utf8'), validManifestArtifact.raw); + const replayPath = path.join(process.env.FKST_DURABLE_ROOT, 'testing-runner', + 'structured-execution', `${sha256(`${runId}-canonical-grant`)}.json`); + const replayEnvelopeRaw = fs.readFileSync(replayPath, 'utf8'); + const replayEnvelope = JSON.parse(replayEnvelopeRaw); + const foreignPlanExecutionArtifact = persistJson(resultRef, { + ...canonicalExecution, plan_sha256: 'f'.repeat(64), + }); + replayEnvelope.value.result_sha256 = foreignPlanExecutionArtifact.digest; + replayEnvelope.mac = crypto.createHmac('sha256', + 'structured-runtime-test-state-key-00000000000000000000') + .update(`runtime-test-v1\0${stableStringify(replayEnvelope.value)}`).digest('hex'); + fs.writeFileSync(replayPath, `${stableStringify(replayEnvelope)}\n`); + await assert.rejects(() => dispatch('load-result', { + ...common, result_ref: resultRef, result_sha256: foreignPlanExecutionArtifact.digest, + plan_sha256: 'f'.repeat(64), + }), /execution result binding is invalid/); + fs.writeFileSync(replayPath, replayEnvelopeRaw); + persistJson(resultRef, canonicalExecution); + persistJson(caseResultSetPath, { ...resultSet, set_id: 'tampered-after-completion' }); await assert.rejects(() => dispatch('load-result', { ...common, result_ref: resultRef, result_sha256: replay.result_sha256, @@ -575,6 +677,10 @@ async function main() { else process.env.FKST_DURABLE_ROOT = previousDurable; if (previousRuntime === undefined) delete process.env.FKST_RUNTIME_ROOT; else process.env.FKST_RUNTIME_ROOT = previousRuntime; + for (const [key, value] of Object.entries(previousAmbient)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } fs.rmSync(`.testing/runs/${runId}`, { recursive: true, force: true }); fs.rmSync(linkPath, { force: true }); fs.rmSync(configRef, { force: true }); diff --git a/libraries/testing_runtime/workflow_qa.lua b/libraries/testing_runtime/workflow_qa.lua index c639098d..faaeb28e 100644 --- a/libraries/testing_runtime/workflow_qa.lua +++ b/libraries/testing_runtime/workflow_qa.lua @@ -38,8 +38,12 @@ function M.production(options) }, run_root(path), path, 15) return type(result) == "table" and result.saved == true end, - load_artifact = function(path) - return cli.call("artifact-load", { path = path }, run_root(path), path, 15) + load_artifact = function(path, expected_digest, options) + return cli.call("artifact-load", { + path = path, + expected_digest = expected_digest, + durable_only = type(options) == "table" and options.durable_only == true or nil, + }, run_root(path), path, 15) end, write_artifact = function(path, value) local result = cli.call("artifact-write", { path = path, value = value }, run_root(path), path, 15) diff --git a/libraries/testing_runtime/workflow_qa_host_adapter.lua b/libraries/testing_runtime/workflow_qa_host_adapter.lua index dda68392..cafe421f 100644 --- a/libraries/testing_runtime/workflow_qa_host_adapter.lua +++ b/libraries/testing_runtime/workflow_qa_host_adapter.lua @@ -7,7 +7,7 @@ local M = {} local required_ports = { "load_artifact", "write_artifact", "artifact_digest", "claim_preauthorization", - "grant_values", "record_terminal", + "reconcile_preauthorization_claim", "grant_values", "record_terminal", } local function copy(value) @@ -52,7 +52,7 @@ function M.new(options) end local function load_bound(ports, ref, expected_digest, label) - local artifact = ports.load_artifact(ref) + local artifact = ports.load_artifact(ref, expected_digest) if type(artifact) ~= "table" or type(artifact.value) ~= "table" or artifact.digest ~= expected_digest then fail(label .. " immutable binding failed") @@ -159,32 +159,52 @@ function M.new(options) function adapter.handle_execution_grant(request, supplied_ports) structured_execution.validate_grant_request(request) local ports = resolve_ports(supplied_ports) - local environment = validate_environment_binding(request, load_bound(ports, - request.environment_receipt_ref, request.environment_receipt_sha256, "environment receipt")) - local existing = ports.load_artifact(request.grant_ref) - if existing ~= nil then - if type(existing) ~= "table" or type(existing.value) ~= "table" then - fail("replayed grant artifact is malformed") - end - local existing_digest = digest(existing.digest, "existing grant digest") - validate_grant_binding(request, existing.value, environment) - return adapter.execution_grant_result_event(request, existing_digest) - end - local materials = { preauthorization = load_bound(ports, request.preauthorization_ref, request.preauthorization_sha256, "preauthorization"), preauthorization_sha256 = request.preauthorization_sha256, plan = load_bound(ports, request.plan_ref, request.plan_sha256, "plan"), plan_sha256 = request.plan_sha256, - environment = environment, + environment = validate_environment_binding(request, load_bound(ports, + request.environment_receipt_ref, request.environment_receipt_sha256, "environment receipt")), environment_receipt_sha256 = request.environment_receipt_sha256, } + local existing = ports.load_artifact(request.grant_ref, nil, { durable_only = true }) + if existing ~= nil then + if ports.reconcile_preauthorization_claim({ + preauthorization_ref = request.preauthorization_ref, + preauthorization_sha256 = request.preauthorization_sha256, + plan_ref = request.plan_ref, + plan_sha256 = request.plan_sha256, + environment_receipt_ref = request.environment_receipt_ref, + environment_receipt_sha256 = request.environment_receipt_sha256, + repository = copy(request.repository), + trace_id = request.trace_id, + dedup_key = request.dedup_key, + }) ~= true then + fail("persisted grant has no authenticated preauthorization claim") + end + if type(existing) ~= "table" or type(existing.value) ~= "table" then + fail("replayed grant artifact is malformed") + end + local expected = adapter.derive_execution_grant(request, materials, + ports.grant_values(copy(request), copy(materials))) + if not structured_execution.equal(existing.value, expected) then + fail("persisted grant differs from authenticated derivation") + end + local existing_digest = digest(existing.digest, "existing grant digest") + validate_grant_binding(request, existing.value, materials.environment) + return adapter.execution_grant_result_event(request, existing_digest) + end + local claim = ports.claim_preauthorization({ authorization_id = materials.preauthorization.authorization_id, + preauthorization_ref = request.preauthorization_ref, preauthorization_sha256 = request.preauthorization_sha256, repository = copy(request.repository), + plan_ref = request.plan_ref, plan_sha256 = request.plan_sha256, + environment_receipt_ref = request.environment_receipt_ref, environment_receipt_sha256 = request.environment_receipt_sha256, trace_id = request.trace_id, dedup_key = request.dedup_key, @@ -200,7 +220,7 @@ function M.new(options) end local grant_sha256 = digest(ports.artifact_digest(request.grant_ref), "grant_sha256") local persisted = load_bound(ports, request.grant_ref, grant_sha256, "persisted grant") - validate_grant_binding(request, persisted, environment) + validate_grant_binding(request, persisted, materials.environment) return adapter.execution_grant_result_event(request, grant_sha256) end diff --git a/packages/environment-factory/bin/environment-factory-runtime.js b/packages/environment-factory/bin/environment-factory-runtime.js index b4b3c1fd..df4b3e9a 100644 --- a/packages/environment-factory/bin/environment-factory-runtime.js +++ b/packages/environment-factory/bin/environment-factory-runtime.js @@ -6,25 +6,46 @@ const http = require('http'); const https = require('https'); const net = require('net'); const path = require('path'); -const { spawn } = require('child_process'); const { loadAuthorizationBundle } = require('./runtime/authorization'); const { createBudgetRuntime } = require('./runtime/budgets'); const { DEFAULT_OUTPUT_BYTES, acquireLock, artifactPath, boundedText, commandResult, - isSafeArtifactPath, minimalEnvironment, parseArgs, readJson, runtimeConfig, sameArray, sha256, - stableStringify, validateArgv, writeJsonAtomic, writeJsonImmutable, + isSafeArtifactPath, minimalEnvironment, parseArgs, readJson, releaseWorkerEnvironment, + releaseWorkerEnvironmentLease, processStartIdentity, runtimeConfig, sameArray, sha256, stableStringify, validateArgv, + verifyWorkerEnvironment, verifyWorkerEnvironmentLease, writeJsonAtomic, writeJsonImmutable, } = require('./runtime/common'); +const { validateTargetExecutionBoundary } = require('./runtime/target-execution-boundary'); const { createListenerClaims } = require('./runtime/listener-claims'); const { runMeasuredCommand } = require('./runtime/measured-command'); const { listenersOwnedByProcessGroup, processGroupUsage } = require('./runtime/platform'); +const { startOrRecoverSupervisedProcess } = require('./runtime/supervised-process'); const { loadState, saveState } = require('./runtime/state'); -const { resolveWorkspace } = require('./runtime/workspace'); +const { readResource: readWorkspaceResource, resolveWorkspace } = require('./runtime/workspace'); function durableRoot() { return path.resolve(process.env.FKST_DURABLE_ROOT || path.join('.testing', 'durable')); } function executionRoot() { return path.resolve(process.env.FKST_RUNTIME_ROOT || path.join('.testing', 'runtime')); } +function workerIsolationIdentity(payload, purpose) { + const identity = { + schema: 'environment-factory.worker-isolation-identity.v1', + operation_id: payload.operation_id, + effect_id: payload.effect_id || purpose, + purpose, + }; + if (payload.repository) identity.repository = payload.repository; + return identity; +} + +function targetExecutionConfig(payload, repository = payload && payload.repository) { + const config = runtimeConfig(payload); + validateTargetExecutionBoundary(config.target_execution_boundary, repository, { + runtimeConfigRef: payload.runtime_config_ref, + artifactRoot: payload.artifact_root, + }); + return config; +} function ledgerPath(kind, id) { return path.join(durableRoot(), 'environment-factory', kind, `${sha256(String(id))}.json`); } @@ -179,6 +200,7 @@ const { durableRoot, ledgerPath, readIfExists, + verifyWorkerEnvironment, writeJsonAtomic, }); @@ -192,7 +214,7 @@ function remainingTimeoutMs(deadline) { async function checkout(payload) { return withEffect(payload, async () => { - const config = runtimeConfig(payload); + const config = targetExecutionConfig(payload); const deadline = effectDeadline(payload); const repository = payload.repository || {}; const source = config.repository_mirrors && config.repository_mirrors[repository.url]; @@ -218,50 +240,56 @@ async function checkout(payload) { working_directory: payload.working_directory, cleaned: false, }); - const commandEnvironment = minimalEnvironment(config.command_environment || {}); - const clone = await executeBudgetedCommand(payload, - ['git', 'clone', '--quiet', '--no-checkout', source, workspacePath], { + const commandEnvironment = minimalEnvironment( + config.command_environment || {}, workerIsolationIdentity(payload, 'checkout'), + ); + try { + const clone = await executeBudgetedCommand(payload, + ['git', 'clone', '--quiet', '--no-checkout', source, workspacePath], { + env: commandEnvironment, + timeoutMs: remainingTimeoutMs(deadline), + workspacePath, + }); + if (clone.reason !== null) { + return { + status: 'blocked', + workspace_ref: workspaceRef, + cleanup_ref: cleanupRef, + diagnostic_ref: writeDiagnostic(payload, 'checkout', { + status: 'blocked', reason: clone.reason, stderr: boundedText(clone.stderr, payload.output_bytes), + }), + }; + } + const checkoutResult = await executeBudgetedCommand(payload, + ['git', 'checkout', '--quiet', '--detach', repository.commit_sha], { + cwd: workspacePath, + env: commandEnvironment, + timeoutMs: remainingTimeoutMs(deadline), + workspacePath, + }); + const resolved = await executeBudgetedCommand(payload, ['git', 'rev-parse', 'HEAD'], { + cwd: workspacePath, env: commandEnvironment, timeoutMs: remainingTimeoutMs(deadline), workspacePath, }); - if (clone.reason !== null) { + const resolvedCommit = String(resolved.stdout || '').trim(); + const passed = checkoutResult.reason === null && resolved.reason === null + && resolvedCommit === repository.commit_sha; return { - status: 'blocked', + status: passed ? 'passed' : 'blocked', + resolved_commit: resolvedCommit || null, workspace_ref: workspaceRef, cleanup_ref: cleanupRef, diagnostic_ref: writeDiagnostic(payload, 'checkout', { - status: 'blocked', reason: clone.reason, stderr: boundedText(clone.stderr, payload.output_bytes), + status: passed ? 'passed' : 'blocked', + resolved_commit: resolvedCommit, + stderr: boundedText(`${checkoutResult.stderr} ${resolved.stderr}`, 1024), }), }; + } finally { + releaseWorkerEnvironment(commandEnvironment); } - const checkoutResult = await executeBudgetedCommand(payload, - ['git', 'checkout', '--quiet', '--detach', repository.commit_sha], { - cwd: workspacePath, - env: commandEnvironment, - timeoutMs: remainingTimeoutMs(deadline), - workspacePath, - }); - const resolved = await executeBudgetedCommand(payload, ['git', 'rev-parse', 'HEAD'], { - cwd: workspacePath, - env: commandEnvironment, - timeoutMs: remainingTimeoutMs(deadline), - workspacePath, - }); - const resolvedCommit = String(resolved.stdout || '').trim(); - const passed = checkoutResult.reason === null && resolved.reason === null - && resolvedCommit === repository.commit_sha; - return { - status: passed ? 'passed' : 'blocked', - resolved_commit: resolvedCommit || null, - workspace_ref: workspaceRef, - cleanup_ref: cleanupRef, - diagnostic_ref: writeDiagnostic(payload, 'checkout', { - status: passed ? 'passed' : 'blocked', - resolved_commit: resolvedCommit, - stderr: boundedText(`${checkoutResult.stderr} ${resolved.stderr}`, 1024), - }), - }; }); } @@ -374,16 +402,6 @@ async function createReadinessAttempt(payload) { }); } -function processStartIdentity(pid) { - if (!Number.isInteger(pid) || pid < 1) return null; - const result = commandResult(['ps', '-o', 'lstart=', '-p', String(pid)], { - timeoutMs: 1000, - outputBytes: 1024, - }); - const identity = result.exitCode === 0 ? result.stdout.trim() : ''; - return identity || null; -} - function inheritedListenerNames(payload) { if (payload.listener_mode !== 'fkst-inherited-listeners-v1') { throw new Error('supervised argv requires fkst-inherited-listeners-v1'); @@ -401,13 +419,22 @@ function inheritedListenerNames(payload) { async function runArgvEffect(payload) { return withEffect(payload, async () => { - const config = runtimeConfig(payload); + const workspaceResource = readWorkspaceResource(payload.workspace_ref); + const config = targetExecutionConfig(payload, workspaceResource.repository); const argv = validateArgv(payload.argv); const cwd = workspaceCwd(payload); const timeoutMs = Math.max(1, Number(payload.timeout_seconds) || 1) * 1000; const outputBytes = Number(payload.output_bytes) || DEFAULT_OUTPUT_BYTES; - const baseEnv = minimalEnvironment(config.command_environment || {}); + if (payload.mode !== 'oneshot' && payload.mode !== 'supervised') { + throw new Error('unsupported argv mode'); + } + let baseEnv = null; + let retainWorkerHome = false; + try { if (payload.mode === 'oneshot') { + baseEnv = minimalEnvironment( + config.command_environment || {}, workerIsolationIdentity(payload, 'run-argv'), + ); const frozen = payload.requires_frozen_dependencies === true; let frozenProof = null; if (frozen) { @@ -466,14 +493,6 @@ async function runArgvEffect(payload) { }), }; } - if (payload.mode !== 'supervised') throw new Error('unsupported argv mode'); - const listenerNames = inheritedListenerNames(payload); - const supervisedEnv = { - ...baseEnv, - FKST_LISTEN_FDS: String(listenerNames.length), - FKST_LISTEN_FDNAMES: listenerNames.join(':'), - }; - const inheritedStdio = listenerNames.map((_name, index) => 3 + index); resourceBudgets(payload); const workspacePath = resolveWorkspace(payload).workspaceRoot; const before = enforceCurrentBudgets(payload, workspacePath); @@ -489,6 +508,9 @@ async function runArgvEffect(payload) { } const resourceRef = `environment-factory-resource-${sha256(`process\0${payload.operation_id}\0${payload.effect_id}`).slice(0, 32)}`; const cleanupRef = { kind: 'resource-cleanup', ref: resourceRef }; + const processResourcePath = resourcePath(resourceRef); + const startupClaimPath = `${processResourcePath}.startup`; + const startupClaimExisted = fs.existsSync(startupClaimPath); const starting = { schema: 'environment-factory.resource.v1', kind: 'process', @@ -496,53 +518,81 @@ async function runArgvEffect(payload) { ref: resourceRef, effect_id: payload.effect_id, argv_sha256: sha256(stableStringify(argv)), - ownership_token: crypto.randomBytes(16).toString('hex'), + ownership_token: sha256(stableStringify({ + schema: 'environment-factory.process-ownership.v1', + operation_id: payload.operation_id, + effect_id: payload.effect_id, + ref: resourceRef, + })), runtime_ports: exactPortList(payload.runtime_ports), - pid: null, - pgid: null, - process_start_identity: null, + repository: workspaceResource.repository, cleaned: false, }; - writeJsonAtomic(resourcePath(resourceRef), starting); - let child; + const existing = readIfExists(processResourcePath); + if (existing) { + const volatile = new Set([ + 'startup_state', 'startup_token_sha256', 'pid', 'pgid', + 'process_start_identity', 'worker_environment_lease', + ]); + const existingBinding = Object.fromEntries( + Object.entries(existing).filter(([key]) => !volatile.has(key)), + ); + if (stableStringify(existingBinding) !== stableStringify(starting)) { + throw new Error('supervised process resource binding differs'); + } + verifyWorkerEnvironmentLease(existing.worker_environment_lease); + const state = processGroupState(existing); + const running = state.supported && state.alive && state.foreign !== true; + return { + status: running ? 'running' : 'blocked', + early_exit: !running, + runtime_ports: exactPortList(payload.runtime_ports), + cleanup_ref: cleanupRef, + diagnostic_ref: writeDiagnostic(payload, sha256(payload.effect_id).slice(0, 16), { + status: running ? 'running' : 'blocked', + reason: running ? null : 'supervised-process-exited', + replayed_startup_resource: true, + output_capture: 'discarded-by-bounded-runtime', + }), + }; + } + const listenerNames = inheritedListenerNames(payload); + const inheritedStdio = listenerNames.map((_name, index) => 3 + index); + const createSupervisedEnvironment = () => { + const environment = minimalEnvironment( + config.command_environment || {}, workerIsolationIdentity(payload, 'run-argv'), + ); + environment.FKST_LISTEN_FDS = String(listenerNames.length); + environment.FKST_LISTEN_FDNAMES = listenerNames.join(':'); + return environment; + }; try { - child = spawn(argv[0], argv.slice(1), { + const launch = startOrRecoverSupervisedProcess({ + claimPath: startupClaimPath, + argv, cwd, - env: supervisedEnv, - shell: false, - detached: true, - stdio: ['ignore', 'ignore', 'ignore', ...inheritedStdio], - }); - await new Promise((resolve, reject) => { - const timer = setTimeout(resolve, 100); - child.once('error', (error) => { clearTimeout(timer); reject(error); }); - }); - const earlyExit = child.exitCode !== null; - const startIdentity = earlyExit ? null : processStartIdentity(child.pid); - if (!earlyExit && startIdentity === null) throw new Error('supervised process identity is unavailable'); - writeJsonAtomic(resourcePath(resourceRef), { - ...starting, - pid: child.pid, - pgid: child.pid, - process_start_identity: startIdentity, - cleaned: false, + createEnvironment: createSupervisedEnvironment, + inheritedStdio, + binding: starting, }); - const measured = earlyExit ? { passed: false, reason: 'supervised-process-exited' } - : enforceCurrentBudgets(payload, workspacePath); - if (!measured.passed && !earlyExit) { - try { process.kill(-child.pid, 'SIGKILL'); } catch (_error) { - try { process.kill(child.pid, 'SIGKILL'); } catch (_ignored) {} + if (launch.interrupted || !launch.resource) { + throw new Error('supervised process startup was interrupted before registration'); + } + retainWorkerHome = launch.environment_retained === true; + const resource = launch.resource; + writeJsonAtomic(processResourcePath, resource); + const live = processGroupState(resource); + const running = launch.state === 'running' && live.supported && live.alive && live.foreign !== true; + const measured = running + ? enforceCurrentBudgets(payload, workspacePath) + : { passed: false, reason: launch.state === 'failed' + ? 'supervised-process-start-failed' : 'supervised-process-exited' }; + if (!measured.passed && running) { + try { process.kill(-resource.pgid, 'SIGKILL'); } catch (_error) { + try { process.kill(resource.pid, 'SIGKILL'); } catch (_ignored) {} } - writeJsonAtomic(resourcePath(resourceRef), { - ...starting, - pid: child.pid, - pgid: child.pid, - process_start_identity: startIdentity, - cleaned: false, - }); } - child.unref(); - const blocked = earlyExit || !measured.passed; + const blocked = !measured.passed; return { status: blocked ? 'blocked' : 'running', early_exit: blocked, @@ -555,24 +605,7 @@ async function runArgvEffect(payload) { }), }; } catch (error) { - let pid = null; - let cleaned = true; - if (child && Number.isInteger(child.pid)) { - pid = child.pid; - try { process.kill(-child.pid, 'SIGKILL'); } catch (_killError) { - try { process.kill(child.pid, 'SIGKILL'); } catch (_ignored) {} - } - const usage = processGroupUsage([child.pid]); - cleaned = usage.supported && usage.processes === 0; - child.unref(); - } - writeJsonAtomic(resourcePath(resourceRef), { - ...starting, - pid, - pgid: pid, - process_start_identity: pid === null ? null : processStartIdentity(pid), - cleaned, - }); + if (!startupClaimExisted && fs.existsSync(startupClaimPath)) retainWorkerHome = true; return { status: 'blocked', early_exit: true, @@ -583,6 +616,9 @@ async function runArgvEffect(payload) { }), }; } + } finally { + if (!retainWorkerHome) releaseWorkerEnvironment(baseEnv); + } }); } @@ -616,20 +652,27 @@ function checkHttp(check, timeoutMs) { }); } -async function readinessCheck(check, payload, deadline) { +async function readinessCheck(check, payload, deadline, config) { const remaining = deadline - Date.now(); if (remaining <= 0) return false; if (check.type === 'tcp') return checkTcp(check, Math.min(500, remaining)); if (check.type === 'http') return checkHttp(check, Math.min(500, remaining)); if (check.type === 'argv') { const workspace = resolveWorkspace(payload); - const result = await executeBudgetedCommand(payload, validateArgv(check.argv), { - cwd: workspace.cwd, - env: minimalEnvironment(runtimeConfig(payload).command_environment || {}), - timeoutMs: Math.min(2_000, remaining), - workspacePath: workspace.workspaceRoot, - }); - return result.reason === null && Date.now() <= deadline; + const environment = minimalEnvironment( + config.command_environment || {}, workerIsolationIdentity(payload, 'readiness-argv'), + ); + try { + const result = await executeBudgetedCommand(payload, validateArgv(check.argv), { + cwd: workspace.cwd, + env: environment, + timeoutMs: Math.min(2_000, remaining), + workspacePath: workspace.workspaceRoot, + }); + return result.reason === null && Date.now() <= deadline; + } finally { + releaseWorkerEnvironment(environment); + } } return false; } @@ -646,6 +689,8 @@ function initialReadinessState(budgets, checks) { async function waitReadiness(payload) { return withEffect(payload, async () => { + const workspaceResource = readWorkspaceResource(payload.workspace_ref); + const config = targetExecutionConfig(payload, workspaceResource.repository); const budgets = resourceBudgets(payload); const checks = Array.isArray(payload.checks) ? payload.checks : []; const ports = exactPortList(payload.runtime_ports); @@ -672,7 +717,7 @@ async function waitReadiness(payload) { } probes += 1; } - results.push(await readinessCheck(check, payload, deadline)); + results.push(await readinessCheck(check, payload, deadline, config)); if (Date.now() > deadline) { reason = 'readiness-timeout'; break; } } if (reason === 'network-request-budget-exceeded' || reason === 'readiness-timeout') break; @@ -751,7 +796,9 @@ async function stopProcess(resource, deadline) { function resourceIsReleased(resource) { if (resource.kind === 'process') { const state = processGroupState(resource); - return state.supported && (!state.alive || state.foreign === true); + const homeReleased = !resource.worker_environment_lease + || !fs.existsSync(resource.worker_environment_lease.home); + return state.supported && (!state.alive || state.foreign === true) && homeReleased; } if (resource.kind === 'workspace') return typeof resource.path === 'string' && !fs.existsSync(resource.path); if (resource.kind === 'ports') { @@ -798,17 +845,30 @@ async function cleanup(payload) { const deadline = effectDeadline(payload); let cleaned = true; if (resource.kind === 'process') { + const config = targetExecutionConfig(payload, resource.repository); if (Array.isArray(payload.argv) && payload.argv.length > 0) { - const config = runtimeConfig(payload); - const result = await runMeasuredCommand(validateArgv(payload.argv), { - cwd: workspaceCwd(payload), - env: minimalEnvironment(config.command_environment || {}), - timeoutMs: remainingTimeoutMs(deadline), - outputBytes: payload.output_bytes, - }); - cleaned = result.exitCode === 0 && result.timedOut !== true && result.outputExceeded !== true; + const environment = minimalEnvironment( + config.command_environment || {}, workerIsolationIdentity(payload, 'cleanup-argv'), + ); + try { + verifyWorkerEnvironment(environment); + const result = await runMeasuredCommand(validateArgv(payload.argv), { + cwd: workspaceCwd(payload), + env: environment, + timeoutMs: remainingTimeoutMs(deadline), + outputBytes: payload.output_bytes, + }); + cleaned = result.exitCode === 0 && result.timedOut !== true && result.outputExceeded !== true; + } finally { + releaseWorkerEnvironment(environment); + } } await stopProcess(resource, deadline); + const stopped = processGroupState(resource); + if (stopped.supported && (!stopped.alive || stopped.foreign === true) + && resource.worker_environment_lease) { + releaseWorkerEnvironmentLease(resource.worker_environment_lease); + } cleaned = cleaned && resourceIsReleased(resource); } else if (resource.kind === 'workspace') { fs.rmSync(resource.path, { recursive: true, force: true }); diff --git a/packages/environment-factory/bin/runtime/budgets.js b/packages/environment-factory/bin/runtime/budgets.js index e3e7bd33..d1997f44 100644 --- a/packages/environment-factory/bin/runtime/budgets.js +++ b/packages/environment-factory/bin/runtime/budgets.js @@ -11,6 +11,7 @@ function createBudgetRuntime(deps) { durableRoot, ledgerPath, readIfExists, + verifyWorkerEnvironment, writeJsonAtomic, } = deps; @@ -105,6 +106,9 @@ function createBudgetRuntime(deps) { const budgets = resourceBudgets(payload); const before = enforceCurrentBudgets(payload, options.workspacePath); if (!before.passed) return { reason: before.reason, exitCode: -1 }; + if (options.env && typeof verifyWorkerEnvironment === 'function') { + verifyWorkerEnvironment(options.env); + } const result = await runMeasuredCommand(argv, { cwd: options.cwd, env: options.env, diff --git a/packages/environment-factory/bin/runtime/common.js b/packages/environment-factory/bin/runtime/common.js index a52ce444..11de40e1 100644 --- a/packages/environment-factory/bin/runtime/common.js +++ b/packages/environment-factory/bin/runtime/common.js @@ -3,12 +3,13 @@ const crypto = require('crypto'); const fs = require('fs'); const path = require('path'); -const { spawnSync } = require('child_process'); +const { spawn, spawnSync } = require('child_process'); const MAX_JSON_BYTES = 2 * 1024 * 1024; const DEFAULT_OUTPUT_BYTES = 64 * 1024; const LOCK_TIMEOUT_MS = 10_000; const sleepCell = new Int32Array(new SharedArrayBuffer(4)); +const WORKER_ENVIRONMENT_LEASE = Symbol('fkst.worker-environment-lease'); function stableStringify(value) { const active = new Set(); @@ -90,14 +91,31 @@ function sleep(ms) { function processStartIdentity(pid) { if (!Number.isInteger(pid) || pid < 1) return null; - const result = spawnSync('ps', ['-o', 'lstart=', '-p', String(pid)], { + if (process.platform === 'linux') { + try { + const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8'); + const close = stat.lastIndexOf(')'); + if (close < 0) return null; + const fields = stat.slice(close + 2).trim().split(/\s+/); + if (fields[0] === 'Z') return null; + const startTicks = fields[19]; + const bootId = fs.readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim(); + if (!/^\d+$/.test(startTicks) || !/^[0-9a-f-]{36}$/.test(bootId)) return null; + return `linux-proc-v1:${bootId}:${startTicks}`; + } catch (_error) { + return null; + } + } + const result = spawnSync('ps', ['-o', 'stat=', '-o', 'lstart=', '-o', 'command=', '-p', String(pid)], { shell: false, encoding: 'utf8', timeout: 1_000, - maxBuffer: 1024, + maxBuffer: 64 * 1024, }); const identity = result.status === 0 ? String(result.stdout || '').trim() : ''; - return identity || null; + const parsed = identity.match(/^(\S+)\s+(.+)$/s); + if (!parsed || parsed[1].startsWith('Z')) return null; + return `${process.platform}-ps-command-v1:${sha256(parsed[2])}`; } function processAlive(pid) { @@ -137,8 +155,7 @@ function readLockOwner(lockPath) { try { return readJson(path.join(lockPath, 'owner.json')); } catch (_error) { return null; } } -function recordedOwnerIsStale(lockPath) { - const owner = readLockOwner(lockPath); +function lockOwnerIsStale(owner) { if (!owner || !Number.isInteger(owner.pid) || typeof owner.process_start_identity !== 'string' || typeof owner.token !== 'string') return false; if (!processAlive(owner.pid)) return true; @@ -146,40 +163,194 @@ function recordedOwnerIsStale(lockPath) { return current !== null && current !== owner.process_start_identity; } -function acquireLock(lockPath, timeoutMs = LOCK_TIMEOUT_MS) { +function sameLockOwner(left, right) { + return Boolean(left && right && left.pid === right.pid + && left.process_start_identity === right.process_start_identity && left.token === right.token); +} + +function createDirectoryLock(lockPath, identity) { + fs.mkdirSync(lockPath); + const directoryIdentity = pathIdentity(lockPath); + const owner = { + schema: 'environment-factory.lock-owner.v1', + pid: process.pid, + process_start_identity: identity, + token: crypto.randomBytes(16).toString('hex'), + }; + try { + fs.writeFileSync(path.join(lockPath, 'owner.json'), `${stableStringify(owner)}\n`, { flag: 'wx' }); + } catch (error) { + if (samePathIdentity(pathIdentity(lockPath), directoryIdentity)) { + fs.rmSync(directoryIdentity.realpath, { recursive: true, force: true }); + } + throw error; + } + let released = false; + return { + owner, + directoryIdentity, + release() { + if (released) return; + const recorded = readLockOwner(lockPath); + if (sameLockOwner(recorded, owner) + && samePathIdentity(pathIdentity(lockPath), directoryIdentity)) { + fs.rmSync(directoryIdentity.realpath, { recursive: true, force: true }); + } + released = true; + }, + }; +} + +function executablePath(candidates) { + for (const candidate of candidates) { + try { + fs.accessSync(candidate, fs.constants.X_OK); + return candidate; + } catch (_error) {} + } + return null; +} + +function acquireTakeoverGuard(lockPath, timeoutMs) { + if (process.platform !== 'linux' && process.platform !== 'darwin') { + throw new Error('lock stale takeover is unsupported on this platform'); + } + const holderPath = path.join(__dirname, 'lock-holder.js'); + const guardPath = `${lockPath}.takeover.lock`; + const markerPath = `${lockPath}.takeover.active`; + const token = crypto.randomBytes(16).toString('hex'); + const environment = Object.create(null); + for (const key of ['LANG', 'LC_ALL', 'PATH', 'SystemRoot', 'WINDIR']) { + if (typeof process.env[key] === 'string') environment[key] = process.env[key]; + } + let argv; + if (process.platform === 'linux') { + const flock = executablePath(['/usr/bin/flock', '/bin/flock']); + if (flock === null) throw new Error('lock stale takeover requires flock'); + argv = [flock, '-x', '-w', String(Math.max(1, Math.ceil(timeoutMs / 1000))), '-F', guardPath, + process.execPath, holderPath, 'linux', guardPath, markerPath, token, String(timeoutMs)]; + } else { + argv = [process.execPath, holderPath, 'darwin', guardPath, markerPath, token]; + } + const child = spawn(argv[0], argv.slice(1), { + env: environment, + shell: false, + stdio: ['pipe', 'ignore', 'ignore'], + }); + child.once('error', () => {}); + const deadline = Date.now() + timeoutMs; + let holderIdentity = null; + const held = () => { + const currentIdentity = processStartIdentity(child.pid); + if (currentIdentity === null) return false; + try { + const marker = JSON.parse(fs.readFileSync(markerPath, 'utf8')); + if (marker.pid !== child.pid || marker.token !== token) return false; + if (holderIdentity === null) holderIdentity = currentIdentity; + return currentIdentity === holderIdentity; + } catch (_error) { + return false; + } + }; + while (!held()) { + if (processStartIdentity(child.pid) === null) { + throw new Error(`lock takeover holder exited: ${lockPath}`); + } + if (Date.now() >= deadline) { + try { child.kill('SIGKILL'); } catch (_error) {} + throw new Error(`lock takeover timeout: ${lockPath}`); + } + sleep(10); + } + let released = false; + return { + markerPath, + assertHeld() { + if (released || !held()) throw new Error(`lock takeover ownership changed: ${lockPath}`); + }, + release() { + if (released) return; + released = true; + try { child.kill('SIGTERM'); } catch (_error) {} + const releaseDeadline = Date.now() + 2_000; + while (fs.existsSync(markerPath) && Date.now() < releaseDeadline) sleep(10); + if (fs.existsSync(markerPath)) { + try { child.kill('SIGKILL'); } catch (_error) {} + throw new Error(`lock takeover release could not be verified: ${lockPath}`); + } + }, + }; +} + +function acquireLock(lockPath, timeoutMs = LOCK_TIMEOUT_MS, options = {}) { fs.mkdirSync(path.dirname(lockPath), { recursive: true }); const deadline = Date.now() + timeoutMs; const identity = processStartIdentity(process.pid); if (identity === null) throw new Error('lock owner identity is unavailable'); + const takeoverMarkerPath = `${lockPath}.takeover.active`; while (true) { + if (fs.existsSync(takeoverMarkerPath)) { + const cleanup = acquireTakeoverGuard(lockPath, Math.max(1, deadline - Date.now())); + cleanup.assertHeld(); + cleanup.release(); + continue; + } try { - fs.mkdirSync(lockPath); - const owner = { - schema: 'environment-factory.lock-owner.v1', - pid: process.pid, - process_start_identity: identity, - token: crypto.randomBytes(16).toString('hex'), - }; - try { - fs.writeFileSync(path.join(lockPath, 'owner.json'), `${stableStringify(owner)}\n`, { flag: 'wx' }); - } catch (error) { - fs.rmSync(lockPath, { recursive: true, force: true }); - throw error; + const acquired = createDirectoryLock(lockPath, identity); + if (fs.existsSync(takeoverMarkerPath)) { + acquired.release(); + continue; } - return () => { - const recorded = readLockOwner(lockPath); - if (recorded && recorded.pid === owner.pid && recorded.process_start_identity === owner.process_start_identity - && recorded.token === owner.token) fs.rmSync(lockPath, { recursive: true, force: true }); - }; + return acquired.release; } catch (error) { if (error.code !== 'EEXIST') throw error; - if (recordedOwnerIsStale(lockPath)) { - fs.rmSync(lockPath, { recursive: true, force: true }); - continue; + } + const observedOwner = readLockOwner(lockPath); + if (lockOwnerIsStale(observedOwner)) { + let takeover; + try { + takeover = acquireTakeoverGuard(lockPath, Math.max(1, deadline - Date.now())); + } catch (error) { + if (Date.now() >= deadline) throw error; + } + if (takeover) { + try { + if (typeof options.afterTakeoverAcquired === 'function') options.afterTakeoverAcquired(takeover); + takeover.assertHeld(); + const currentOwner = readLockOwner(lockPath); + if (sameLockOwner(currentOwner, observedOwner) && lockOwnerIsStale(currentOwner)) { + const staleIdentity = pathIdentity(lockPath); + const confirmedOwner = readLockOwner(lockPath); + if (sameLockOwner(confirmedOwner, currentOwner) + && samePathIdentity(pathIdentity(lockPath), staleIdentity)) { + takeover.assertHeld(); + fs.rmSync(staleIdentity.realpath, { recursive: true, force: true }); + while (true) { + try { + const acquired = createDirectoryLock(lockPath, identity); + try { + takeover.assertHeld(); + } catch (error) { + acquired.release(); + throw error; + } + takeover.release(); + return acquired.release; + } catch (error) { + if (error.code !== 'EEXIST') throw error; + if (Date.now() >= deadline) throw new Error(`lock takeover timeout: ${lockPath}`); + sleep(10); + } + } + } + } + } finally { + takeover.release(); + } } - if (Date.now() >= deadline) throw new Error(`lock timeout: ${lockPath}`); - sleep(10); } + if (Date.now() >= deadline) throw new Error(`lock timeout: ${lockPath}`); + sleep(10); } } @@ -246,36 +417,216 @@ function validateArgv(argv) { return argv; } -function minimalEnvironment(extra = {}) { - const allowed = ['HOME', 'LANG', 'LC_ALL', 'PATH', 'PATHEXT', 'SystemRoot', 'TEMP', 'TMP', 'TMPDIR', 'WINDIR']; - const env = {}; +function forbiddenWorkerEnvironmentKey(key) { + const upper = key.toUpperCase(); + const exact = new Set([ + 'HOME', 'USERPROFILE', 'HOMEDRIVE', 'HOMEPATH', 'APPDATA', 'LOCALAPPDATA', + 'XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'XDG_STATE_HOME', 'GH_CONFIG_DIR', + ]); + return exact.has(upper) || upper.startsWith('GH_') || upper.startsWith('GITHUB_') + || upper.startsWith('GIT_') || upper.startsWith('SSH_') + || upper.includes('ASKPASS') || upper.includes('CREDENTIAL_HELPER'); +} + +function requireOwnedDirectory(directory, { privateDirectory = false } = {}) { + let created = false; + try { + fs.mkdirSync(directory, { mode: privateDirectory ? 0o700 : 0o755 }); + created = true; + } catch (error) { + if (error.code !== 'EEXIST') throw error; + } + const stat = fs.lstatSync(directory); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error(`worker environment directory is not a real directory: ${directory}`); + } + if (typeof process.getuid === 'function' && stat.uid !== process.getuid()) { + throw new Error(`worker environment directory has a foreign owner: ${directory}`); + } + if (privateDirectory && process.platform !== 'win32') { + if (!created && (stat.mode & 0o077) !== 0) { + throw new Error(`worker environment directory permissions are too broad: ${directory}`); + } + fs.chmodSync(directory, 0o700); + } + return fs.realpathSync(directory); +} + +function minimalEnvironment(extra = {}, isolationKey = 'shared-runtime-command') { + if (!extra || typeof extra !== 'object' || Array.isArray(extra)) { + throw new Error('command environment must be an object'); + } + const allowed = ['LANG', 'LC_ALL', 'PATH', 'PATHEXT', 'SystemRoot', 'TEMP', 'TMP', 'TMPDIR', 'WINDIR']; + const env = Object.create(null); for (const key of allowed) { if (typeof process.env[key] === 'string') env[key] = process.env[key]; } for (const [key, value] of Object.entries(extra)) { + if (!/^[A-Za-z_][A-Za-z0-9_]{0,127}$/.test(key) || forbiddenWorkerEnvironmentKey(key)) { + throw new Error(`command environment contains a forbidden worker authority key: ${key}`); + } if (typeof value !== 'string' || /[\x00]/.test(value)) throw new Error('command environment contains an invalid value'); env[key] = value; } + const identity = typeof isolationKey === 'string' ? { scope: isolationKey } : isolationKey; + if (!identity || typeof identity !== 'object' || Array.isArray(identity)) { + throw new Error('worker isolation identity is invalid'); + } + const identityBody = stableStringify(identity); + const configuredRuntimeRoot = path.resolve( + process.env.FKST_RUNTIME_ROOT || path.join('.testing', 'runtime'), + ); + fs.mkdirSync(configuredRuntimeRoot, { recursive: true }); + const runtimeRoot = requireOwnedDirectory(configuredRuntimeRoot); + const homesRoot = requireOwnedDirectory(path.join(runtimeRoot, 'worker-homes'), { privateDirectory: true }); + const home = fs.mkdtempSync(path.join(homesRoot, `${sha256(identityBody).slice(0, 24)}-`)); + if (process.platform !== 'win32') fs.chmodSync(home, 0o700); + const marker = `${stableStringify({ + schema: 'fkst.worker-home-identity.v1', + identity_sha256: sha256(identityBody), + lease_id: crypto.randomBytes(16).toString('hex'), + })}\n`; + const markerPath = path.join(home, '.fkst-worker-home.json'); + fs.writeFileSync(markerPath, marker, { flag: 'wx', mode: 0o600 }); + const configHome = path.join(home, '.config'); + requireOwnedDirectory(configHome, { privateDirectory: true }); + requireOwnedDirectory(path.join(configHome, 'gh'), { privateDirectory: true }); + const nullDevice = process.platform === 'win32' ? 'NUL' : '/dev/null'; + Object.assign(env, { + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: configHome, + XDG_DATA_HOME: path.join(home, '.local', 'share'), + XDG_STATE_HOME: path.join(home, '.local', 'state'), + GH_CONFIG_DIR: path.join(configHome, 'gh'), + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: nullDevice, + GIT_CONFIG_COUNT: '4', + GIT_CONFIG_KEY_0: 'credential.helper', + GIT_CONFIG_VALUE_0: '', + GIT_CONFIG_KEY_1: 'core.askPass', + GIT_CONFIG_VALUE_1: '', + GIT_CONFIG_KEY_2: 'core.fsmonitor', + GIT_CONFIG_VALUE_2: 'false', + GIT_CONFIG_KEY_3: 'core.hooksPath', + GIT_CONFIG_VALUE_3: nullDevice, + GIT_TERMINAL_PROMPT: '0', + GCM_INTERACTIVE: 'Never', + }); + Object.defineProperty(env, WORKER_ENVIRONMENT_LEASE, { + configurable: false, + enumerable: false, + writable: false, + value: { + schema: 'fkst.worker-home-lease.v1', + home, + home_identity: pathIdentity(home), + homes_root: homesRoot, + homes_root_identity: pathIdentity(homesRoot), + marker_sha256: sha256(marker), + identity_sha256: sha256(identityBody), + released: false, + }, + }); return env; } +function workerEnvironmentLease(environment) { + const lease = environment && environment[WORKER_ENVIRONMENT_LEASE]; + if (!lease || lease.schema !== 'fkst.worker-home-lease.v1') { + throw new Error('worker environment lease is unavailable'); + } + return { + schema: lease.schema, + home: lease.home, + home_identity: { ...lease.home_identity }, + homes_root: lease.homes_root, + homes_root_identity: { ...lease.homes_root_identity }, + marker_sha256: lease.marker_sha256, + identity_sha256: lease.identity_sha256, + }; +} + +function verifyWorkerEnvironmentLease(lease) { + if (!lease || lease.schema !== 'fkst.worker-home-lease.v1' + || typeof lease.home !== 'string' || typeof lease.homes_root !== 'string' + || !samePathIdentity(pathIdentity(lease.homes_root), lease.homes_root_identity) + || !samePathIdentity(pathIdentity(lease.home), lease.home_identity)) { + throw new Error('worker environment lease identity changed'); + } + const root = fs.realpathSync(lease.homes_root); + const home = fs.realpathSync(lease.home); + if (!home.startsWith(`${root}${path.sep}`)) throw new Error('worker environment home escaped its lease root'); + const markerPath = path.join(home, '.fkst-worker-home.json'); + const markerStat = fs.lstatSync(markerPath); + if (!markerStat.isFile() || markerStat.isSymbolicLink()) throw new Error('worker environment marker is invalid'); + const marker = fs.readFileSync(markerPath, 'utf8'); + if (sha256(marker) !== lease.marker_sha256) throw new Error('worker environment marker changed'); + const value = JSON.parse(marker); + if (value.schema !== 'fkst.worker-home-identity.v1' + || value.identity_sha256 !== lease.identity_sha256 + || typeof value.lease_id !== 'string' || !/^[0-9a-f]{32}$/.test(value.lease_id)) { + throw new Error('worker environment marker binding changed'); + } + for (const directory of [path.join(home, '.config'), path.join(home, '.config', 'gh')]) { + const stat = fs.lstatSync(directory); + if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error('worker environment config directory changed'); + } + return true; +} + +function verifyWorkerEnvironment(environment) { + const lease = workerEnvironmentLease(environment); + if (environment.HOME !== lease.home || environment.USERPROFILE !== lease.home + || environment.XDG_CONFIG_HOME !== path.join(lease.home, '.config') + || environment.GH_CONFIG_DIR !== path.join(lease.home, '.config', 'gh')) { + throw new Error('worker environment variables differ from the owned lease'); + } + return verifyWorkerEnvironmentLease(lease); +} + +function releaseWorkerEnvironmentLease(lease) { + if (!lease || lease.schema !== 'fkst.worker-home-lease.v1') { + throw new Error('worker environment lease is invalid'); + } + if (!fs.existsSync(lease.home)) return true; + verifyWorkerEnvironmentLease(lease); + return removeOwnedDirectory(lease.home, lease.home_identity, lease.homes_root); +} + +function releaseWorkerEnvironment(environment) { + const lease = environment && environment[WORKER_ENVIRONMENT_LEASE]; + if (!lease) return false; + if (lease.released) return true; + const released = releaseWorkerEnvironmentLease(lease); + lease.released = released; + return released; +} + function commandResult(argv, options = {}) { validateArgv(argv); const outputBytes = Math.max(1, Math.min(Number(options.outputBytes) || DEFAULT_OUTPUT_BYTES, MAX_JSON_BYTES)); - const result = spawnSync(argv[0], argv.slice(1), { - cwd: options.cwd, - env: options.env || minimalEnvironment(), - shell: false, - encoding: 'utf8', - timeout: Math.max(1, Number(options.timeoutMs) || 30_000), - maxBuffer: outputBytes, - }); - return { - exitCode: Number.isInteger(result.status) ? result.status : -1, - stdout: String(result.stdout || '').slice(0, outputBytes), - stderr: boundedText(result.stderr || (result.error && result.error.message), outputBytes), - error: result.error, - }; + const ownedEnvironment = options.env === undefined; + const environment = options.env || minimalEnvironment(); + try { + verifyWorkerEnvironment(environment); + const result = spawnSync(argv[0], argv.slice(1), { + cwd: options.cwd, + env: environment, + shell: false, + encoding: 'utf8', + timeout: Math.max(1, Number(options.timeoutMs) || 30_000), + maxBuffer: outputBytes, + }); + return { + exitCode: Number.isInteger(result.status) ? result.status : -1, + stdout: String(result.stdout || '').slice(0, outputBytes), + stderr: boundedText(result.stderr || (result.error && result.error.message), outputBytes), + error: result.error, + }; + } finally { + if (ownedEnvironment) releaseWorkerEnvironment(environment); + } } function sameArray(left, right) { @@ -298,6 +649,8 @@ module.exports = { processStartIdentity, readJson, removeOwnedDirectory, + releaseWorkerEnvironment, + releaseWorkerEnvironmentLease, runtimeConfig, sameArray, samePathIdentity, @@ -306,6 +659,9 @@ module.exports = { sleep, stableStringify, validateArgv, + verifyWorkerEnvironment, + verifyWorkerEnvironmentLease, + workerEnvironmentLease, writeJsonAtomic, writeJsonImmutable, }; diff --git a/packages/environment-factory/bin/runtime/lock-holder.js b/packages/environment-factory/bin/runtime/lock-holder.js new file mode 100644 index 00000000..d7810aec --- /dev/null +++ b/packages/environment-factory/bin/runtime/lock-holder.js @@ -0,0 +1,70 @@ +'use strict'; + +const fs = require('fs'); + +function fail(message) { + process.stderr.write(`lock-holder: ${message}\n`); + process.exit(2); +} + +function removeStaleMarker(markerPath) { + try { + const stat = fs.lstatSync(markerPath); + if (stat.isSymbolicLink() || !stat.isFile()) fail('takeover marker is not a regular file'); + } catch (error) { + if (error && error.code === 'ENOENT') return; + throw error; + } + // The system guard is already exclusive, so any pre-existing marker belongs to a prior holder. + fs.unlinkSync(markerPath); +} + +const [mode, lockPath, markerPath, token] = process.argv.slice(2); +if (!['linux', 'darwin'].includes(mode) || !lockPath || !markerPath + || !/^[0-9a-f]{32}$/.test(String(token || ''))) fail('arguments are invalid'); +let darwinGuard = null; +if (mode === 'darwin') { + const O_EXLOCK = 0x00000020; + darwinGuard = fs.openSync(lockPath, + fs.constants.O_RDWR | fs.constants.O_CREAT | (fs.constants.O_NOFOLLOW || 0) | O_EXLOCK, 0o600); + const opened = fs.fstatSync(darwinGuard); + const current = fs.lstatSync(lockPath); + if (!opened.isFile() || current.isSymbolicLink() || !current.isFile() + || opened.dev !== current.dev || opened.ino !== current.ino) { + fs.closeSync(darwinGuard); + fail('Darwin takeover guard identity changed'); + } +} + +removeStaleMarker(markerPath); +fs.writeFileSync(markerPath, `${JSON.stringify({ pid: process.pid, token })}\n`, { + flag: 'wx', + mode: 0o600, +}); + +let cleaned = false; +function cleanup() { + if (cleaned) return; + cleaned = true; + try { + const marker = JSON.parse(fs.readFileSync(markerPath, 'utf8')); + if (marker.pid === process.pid && marker.token === token) fs.unlinkSync(markerPath); + } catch (_error) {} + if (darwinGuard !== null) fs.closeSync(darwinGuard); +} + +for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) { + process.once(signal, () => { + cleanup(); + process.exit(0); + }); +} +process.stdin.resume(); +process.stdin.once('end', () => { + cleanup(); + process.exit(0); +}); +process.stdin.once('error', () => { + cleanup(); + process.exit(0); +}); diff --git a/packages/environment-factory/bin/runtime/platform.js b/packages/environment-factory/bin/runtime/platform.js index 2dbc7016..5e683bf3 100644 --- a/packages/environment-factory/bin/runtime/platform.js +++ b/packages/environment-factory/bin/runtime/platform.js @@ -34,21 +34,22 @@ function parsePsTime(value) { } function processTable() { - const result = directCommand(['ps', '-axo', 'pid=,pgid=,rss=,time='], { + const result = directCommand(['ps', '-axo', 'pid=,pgid=,stat=,rss=,time='], { timeoutMs: 2_000, outputBytes: 4 * 1024 * 1024, }); if (result.error || result.status !== 0) return null; const rows = []; for (const line of String(result.stdout || '').split('\n')) { - const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\d+)\s+(.+)$/); + const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\S+)\s+(\d+)\s+(.+)$/); if (!match) continue; - const cpuMillis = parsePsTime(match[4]); + if (match[3].startsWith('Z')) continue; + const cpuMillis = parsePsTime(match[5]); if (cpuMillis === null) return null; rows.push({ pid: Number(match[1]), pgid: Number(match[2]), - rssBytes: Number(match[3]) * 1024, + rssBytes: Number(match[4]) * 1024, cpuMillis, }); } diff --git a/packages/environment-factory/bin/runtime/supervised-process.js b/packages/environment-factory/bin/runtime/supervised-process.js new file mode 100644 index 00000000..89e40afe --- /dev/null +++ b/packages/environment-factory/bin/runtime/supervised-process.js @@ -0,0 +1,476 @@ +'use strict'; + +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); +const { spawn } = require('child_process'); +const { + acquireLock, + boundedText, + processAlive, + processStartIdentity, + releaseWorkerEnvironmentLease, + sha256, + sleep, + stableStringify, + validateArgv, + verifyWorkerEnvironment, + verifyWorkerEnvironmentLease, + workerEnvironmentLease, + writeJsonAtomic, +} = require('./common'); + +const CLAIM_SCHEMA = 'fkst.supervised-process-startup.v1'; +const SPEC_SCHEMA = 'fkst.supervised-process-launch.v1'; + +function readIfExists(filePath) { + try { + return readJsonNoFollow(filePath); + } catch (error) { + if (error && error.code === 'ENOENT') return null; + throw error; + } +} + +function readJsonNoFollow(filePath) { + const before = fs.lstatSync(filePath); + if (!before.isFile() || before.isSymbolicLink() || before.size > 2 * 1024 * 1024) { + throw new Error('supervised startup claim is not a bounded regular file'); + } + const descriptor = fs.openSync(filePath, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)); + try { + const openedBefore = fs.fstatSync(descriptor); + if (!sameFileIdentity(fileIdentity(before), fileIdentity(openedBefore))) { + throw new Error('supervised startup claim identity changed'); + } + const body = fs.readFileSync(descriptor, 'utf8'); + const openedAfter = fs.fstatSync(descriptor); + const after = fs.lstatSync(filePath); + if (!sameFileIdentity(fileIdentity(openedBefore), fileIdentity(openedAfter)) + || !sameFileIdentity(fileIdentity(openedAfter), fileIdentity(after))) { + throw new Error('supervised startup claim changed while reading'); + } + return JSON.parse(body); + } finally { + fs.closeSync(descriptor); + } +} + +function writeClaimAtomic(filePath, value) { + writeJsonAtomic(filePath, value); + fs.chmodSync(filePath, 0o600); +} + +function requireAbsoluteFile(filePath, label) { + if (typeof filePath !== 'string' || !path.isAbsolute(filePath) || path.basename(filePath) === '') { + throw new Error(`${label} must be an absolute file path`); + } + return path.resolve(filePath); +} + +function validClaim(claim, bindingSha256) { + return Boolean(claim && claim.schema === CLAIM_SCHEMA && claim.version === 1 + && /^[0-9a-f]{32}$/.test(String(claim.startup_token || '')) + && claim.binding_sha256 === bindingSha256 + && Number.isInteger(claim.created_at_epoch_ms) + && Number.isInteger(claim.registration_deadline_epoch_ms) + && typeof claim.launch_spec_path === 'string' + && /^[0-9a-f]{64}$/.test(String(claim.launch_spec_sha256 || '')) + && /^[0-9a-f]{64}$/.test(String(claim.argv_sha256 || '')) + && typeof claim.cwd === 'string' && path.isAbsolute(claim.cwd) + && claim.cwd_identity && typeof claim.cwd_identity === 'object' + && Number.isInteger(claim.inherited_fd_count) && claim.inherited_fd_count >= 0 + && Array.isArray(claim.inherited_fd_identities) + && claim.inherited_fd_identities.length === claim.inherited_fd_count + && /^[0-9a-f]{64}$/.test(String(claim.worker_environment_lease_sha256 || '')) + && claim.worker_environment_lease + && ['preparing', 'prepared', 'registered', 'running', 'exited', 'failed', 'revoked'].includes(claim.state) + && (claim.state === 'preparing' || sameFileIdentityShape(claim.launch_spec_identity))); +} + +function sameFileIdentityShape(identity) { + return Boolean(identity && typeof identity === 'object' + && typeof identity.device === 'string' && typeof identity.inode === 'string' + && Number.isInteger(identity.size) && identity.size >= 0 + && Number.isInteger(identity.mode)); +} + +function fileIdentity(stat) { + return { + device: String(stat.dev), + inode: String(stat.ino), + size: stat.size, + mode: stat.mode, + }; +} + +function sameFileIdentity(left, right) { + return sameFileIdentityShape(left) && sameFileIdentityShape(right) + && left.device === right.device && left.inode === right.inode + && left.size === right.size && left.mode === right.mode; +} + +function sameNodeIdentity(left, right) { + return sameFileIdentityShape(left) && sameFileIdentityShape(right) + && left.device === right.device && left.inode === right.inode + && (left.mode & fs.constants.S_IFMT) === (right.mode & fs.constants.S_IFMT); +} + +function descriptorIdentities(descriptors) { + return descriptors.map((descriptor) => fileIdentity(fs.fstatSync(descriptor))); +} + +function readBoundFile(filePath, expectedIdentity) { + const before = fs.lstatSync(filePath); + if (!before.isFile() || before.isSymbolicLink()) { + throw new Error('supervised launch spec is not a regular file'); + } + const flags = fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0); + const descriptor = fs.openSync(filePath, flags); + try { + const openedBefore = fs.fstatSync(descriptor); + if (!sameFileIdentity(fileIdentity(before), fileIdentity(openedBefore)) + || !sameFileIdentity(fileIdentity(openedBefore), expectedIdentity)) { + throw new Error('supervised launch spec identity changed'); + } + const body = fs.readFileSync(descriptor); + const openedAfter = fs.fstatSync(descriptor); + const after = fs.lstatSync(filePath); + if (!sameFileIdentity(fileIdentity(openedBefore), fileIdentity(openedAfter)) + || !sameFileIdentity(fileIdentity(openedAfter), fileIdentity(after))) { + throw new Error('supervised launch spec changed while reading'); + } + return body; + } finally { + fs.closeSync(descriptor); + } +} + +function transitionClaim(claimPath, token, allowedStates, update) { + const release = acquireLock(`${claimPath}.lock`); + try { + const current = readJsonNoFollow(claimPath); + if (current.schema !== CLAIM_SCHEMA || current.startup_token !== token) return false; + if (!allowedStates.includes(current.state)) return false; + writeClaimAtomic(claimPath, { ...current, ...update }); + return true; + } finally { + release(); + } +} + +function resourceFromClaim(binding, claim) { + return { + ...binding, + startup_state: claim.state, + startup_token_sha256: sha256(claim.startup_token), + pid: Number.isInteger(claim.pid) ? claim.pid : null, + pgid: Number.isInteger(claim.pgid) ? claim.pgid : null, + process_start_identity: typeof claim.process_start_identity === 'string' + ? claim.process_start_identity : null, + worker_environment_lease: claim.worker_environment_lease, + }; +} + +function startOrRecoverSupervisedProcess(options) { + const claimPath = requireAbsoluteFile(options.claimPath, 'supervised startup claim'); + const argv = validateArgv(options.argv); + const cwd = fs.realpathSync(path.resolve(options.cwd)); + const cwdIdentity = (() => { + const stat = fs.statSync(cwd); + if (!stat.isDirectory()) throw new Error('supervised process cwd is not a directory'); + return fileIdentity(stat); + })(); + const binding = options.binding; + if (!binding || typeof binding !== 'object' || Array.isArray(binding)) { + throw new Error('supervised process binding is invalid'); + } + const bindingSha256 = sha256(stableStringify(binding)); + const inheritedStdio = Array.isArray(options.inheritedStdio) ? options.inheritedStdio : []; + if (inheritedStdio.some((fd, index) => !Number.isInteger(fd) || fd !== index + 3)) { + throw new Error('supervised inherited descriptors are invalid'); + } + const inheritedFdIdentities = descriptorIdentities(inheritedStdio); + const registrationTimeoutMs = Math.max(250, Math.min(Number(options.registrationTimeoutMs) || 5_000, 30_000)); + let created = false; + let suppliedLease = null; + const release = acquireLock(`${claimPath}.lock`); + try { + let claim = readIfExists(claimPath); + if (claim !== null && !validClaim(claim, bindingSha256)) { + throw new Error('supervised startup claim binding differs'); + } + if (claim === null) { + const environment = options.environment || (typeof options.createEnvironment === 'function' + ? options.createEnvironment() : null); + if (!environment) throw new Error('supervised process environment is required for a new launch'); + verifyWorkerEnvironment(environment); + suppliedLease = workerEnvironmentLease(environment); + const token = crypto.randomBytes(16).toString('hex'); + const launchSpecPath = `${claimPath}.launch-${token}.json`; + const spec = { + schema: SPEC_SCHEMA, + startup_token: token, + binding_sha256: bindingSha256, + claim_path: claimPath, + argv, + cwd, + inherited_fd_count: inheritedStdio.length, + inherited_fd_identities: inheritedFdIdentities, + }; + const specBody = `${stableStringify(spec)}\n`; + const now = Date.now(); + claim = { + schema: CLAIM_SCHEMA, + version: 1, + state: 'preparing', + startup_token: token, + binding_sha256: bindingSha256, + created_at_epoch_ms: now, + registration_deadline_epoch_ms: now + registrationTimeoutMs, + launch_spec_path: launchSpecPath, + launch_spec_sha256: sha256(specBody), + launch_spec_identity: null, + argv_sha256: sha256(stableStringify(argv)), + cwd, + cwd_identity: cwdIdentity, + inherited_fd_count: inheritedStdio.length, + inherited_fd_identities: inheritedFdIdentities, + worker_environment_lease: suppliedLease, + worker_environment_lease_sha256: sha256(stableStringify(suppliedLease)), + pid: null, + pgid: null, + process_start_identity: null, + }; + fs.mkdirSync(path.dirname(claimPath), { recursive: true }); + try { + if (typeof options.beforeClaimPersist === 'function') options.beforeClaimPersist(); + writeClaimAtomic(claimPath, claim); + } catch (error) { + if (fs.existsSync(claimPath)) { + try { + writeClaimAtomic(claimPath, { + ...claim, + state: 'revoked', + failure_reason: 'startup-claim-persistence-failed', + failed_at_epoch_ms: Date.now(), + }); + } catch (_claimError) {} + } + releaseWorkerEnvironmentLease(suppliedLease); + throw error; + } + let launched = false; + try { + fs.writeFileSync(launchSpecPath, specBody, { flag: 'wx', mode: 0o600 }); + fs.chmodSync(launchSpecPath, 0o600); + const launchSpecStat = fs.lstatSync(launchSpecPath); + if (!launchSpecStat.isFile() || launchSpecStat.isSymbolicLink()) { + throw new Error('supervised launch spec is not a regular file'); + } + const launchSpecIdentity = fileIdentity(launchSpecStat); + claim = { ...claim, state: 'prepared', launch_spec_identity: launchSpecIdentity }; + writeClaimAtomic(claimPath, claim); + if (typeof options.beforeSupervisorLaunch === 'function') options.beforeSupervisorLaunch(claim); + const supervisor = spawn(process.execPath, [__filename, 'child', launchSpecPath], { + cwd, + env: environment, + shell: false, + detached: true, + stdio: ['ignore', 'ignore', 'ignore', ...inheritedStdio], + }); + supervisor.once('error', () => {}); + supervisor.unref(); + launched = true; + created = true; + if (typeof options.afterLaunch === 'function' && options.afterLaunch(supervisor.pid) === false) { + return { interrupted: true, environment_retained: true }; + } + } catch (error) { + if (!launched) { + claim = { + ...claim, + state: 'revoked', + failure_reason: 'supervisor-launch-failed', + failed_at_epoch_ms: Date.now(), + }; + writeClaimAtomic(claimPath, claim); + releaseWorkerEnvironmentLease(suppliedLease); + } + throw error; + } + } + } finally { + release(); + } + + const deadline = Date.now() + registrationTimeoutMs; + while (Date.now() < deadline) { + let claim = readJsonNoFollow(claimPath); + if (!validClaim(claim, bindingSha256)) throw new Error('supervised startup claim binding differs'); + if (claim.state === 'registered' || claim.state === 'running') { + verifyWorkerEnvironmentLease(claim.worker_environment_lease); + if (!Number.isInteger(claim.pid) || claim.pid < 1 + || !processAlive(claim.pid) + || processStartIdentity(claim.pid) !== claim.process_start_identity) { + transitionClaim(claimPath, claim.startup_token, ['registered', 'running'], { + state: 'failed', + failure_reason: 'registered-process-unavailable', + failed_at_epoch_ms: Date.now(), + }); + claim = readJsonNoFollow(claimPath); + } + } + if (claim.state !== 'preparing' && claim.state !== 'prepared' && claim.state !== 'registered') { + if (claim.state !== 'revoked') verifyWorkerEnvironmentLease(claim.worker_environment_lease); + return { + interrupted: false, + state: claim.state, + resource: resourceFromClaim(binding, claim), + environment_retained: created && suppliedLease && suppliedLease.home === claim.worker_environment_lease.home, + }; + } + sleep(25); + } + + const revoke = acquireLock(`${claimPath}.lock`); + let revoked; + try { + const claim = readJsonNoFollow(claimPath); + if (!validClaim(claim, bindingSha256)) throw new Error('supervised startup claim binding differs'); + if (claim.state === 'preparing' || claim.state === 'prepared') { + revoked = { ...claim, state: 'revoked', failure_reason: 'launcher-registration-timeout' }; + writeClaimAtomic(claimPath, revoked); + } else { + revoked = claim; + } + } finally { + revoke(); + } + if (revoked.state === 'revoked') releaseWorkerEnvironmentLease(revoked.worker_environment_lease); + return { + interrupted: false, + state: revoked.state, + resource: resourceFromClaim(binding, revoked), + environment_retained: false, + }; +} + +function childMain(specPath) { + const absoluteSpec = requireAbsoluteFile(specPath, 'supervised launch spec'); + const claimPathGuess = absoluteSpec.replace(/\.launch-[0-9a-f]{32}\.json$/, ''); + if (claimPathGuess === absoluteSpec) throw new Error('supervised launch spec path is invalid'); + const initialClaim = readJsonNoFollow(claimPathGuess); + if (!validClaim(initialClaim, initialClaim && initialClaim.binding_sha256) + || initialClaim.state !== 'prepared' || initialClaim.launch_spec_path !== absoluteSpec) { + throw new Error('supervised startup claim is unavailable or revoked'); + } + const specBody = readBoundFile(absoluteSpec, initialClaim.launch_spec_identity); + if (sha256(specBody) !== initialClaim.launch_spec_sha256) { + throw new Error('supervised launch spec digest differs'); + } + const spec = JSON.parse(specBody.toString('utf8')); + if (!spec || spec.schema !== SPEC_SCHEMA || spec.claim_path !== claimPathGuess + || spec.startup_token !== initialClaim.startup_token + || spec.binding_sha256 !== initialClaim.binding_sha256 + || !Number.isInteger(spec.inherited_fd_count) || spec.inherited_fd_count < 0 + || spec.inherited_fd_count > 32 + || sha256(`${stableStringify(spec)}\n`) !== initialClaim.launch_spec_sha256 + || sha256(stableStringify(validateArgv(spec.argv))) !== initialClaim.argv_sha256 + || path.resolve(spec.cwd) !== initialClaim.cwd + || !sameNodeIdentity(fileIdentity(fs.statSync(spec.cwd)), initialClaim.cwd_identity) + || spec.inherited_fd_count !== initialClaim.inherited_fd_count + || stableStringify(spec.inherited_fd_identities) !== stableStringify(initialClaim.inherited_fd_identities) + || sha256(stableStringify(initialClaim.worker_environment_lease)) + !== initialClaim.worker_environment_lease_sha256) { + throw new Error('supervised launch spec is invalid or unbound'); + } + const actualFdIdentities = descriptorIdentities( + Array.from({ length: spec.inherited_fd_count }, (_item, index) => index + 3), + ); + if (stableStringify(actualFdIdentities) !== stableStringify(initialClaim.inherited_fd_identities)) { + throw new Error('supervised inherited descriptor identity differs'); + } + const claimPath = claimPathGuess; + const release = acquireLock(`${claimPath}.lock`); + let child; + let inherited = []; + const closeInherited = () => { + for (const fd of inherited) { + try { fs.closeSync(fd); } catch (_error) {} + } + inherited = []; + }; + try { + const claim = readJsonNoFollow(claimPath); + if (!validClaim(claim, spec.binding_sha256) || claim.state !== 'prepared' + || claim.startup_token !== spec.startup_token || claim.launch_spec_path !== absoluteSpec) { + throw new Error('supervised startup claim is unavailable or revoked'); + } + const identity = processStartIdentity(process.pid); + if (identity === null) throw new Error('supervised process identity is unavailable'); + writeClaimAtomic(claimPath, { + ...claim, + state: 'registered', + pid: process.pid, + pgid: process.pid, + process_start_identity: identity, + registered_at_epoch_ms: Date.now(), + }); + inherited = Array.from({ length: spec.inherited_fd_count }, (_item, index) => index + 3); + child = spawn(validateArgv(spec.argv)[0], spec.argv.slice(1), { + cwd: path.resolve(spec.cwd), + env: process.env, + shell: false, + detached: false, + stdio: ['ignore', 'ignore', 'ignore', ...inherited], + }); + closeInherited(); + child.once('error', (error) => { + transitionClaim(claimPath, spec.startup_token, ['registered', 'running'], { + state: 'failed', + failure_reason: boundedText(error && error.message, 256) || 'target-spawn-failed', + failed_at_epoch_ms: Date.now(), + }); + process.exitCode = 1; + }); + writeClaimAtomic(claimPath, { + ...readJsonNoFollow(claimPath), + state: 'running', + target_pid: child.pid, + started_at_epoch_ms: Date.now(), + }); + } finally { + closeInherited(); + release(); + } + child.once('exit', (code, signal) => { + transitionClaim(claimPath, spec.startup_token, ['running'], { + state: 'exited', + exit_code: Number.isInteger(code) ? code : null, + exit_signal: typeof signal === 'string' ? signal : null, + exited_at_epoch_ms: Date.now(), + }); + process.exitCode = Number.isInteger(code) ? code : 1; + }); +} + +if (require.main === module) { + if (process.argv.length !== 4 || process.argv[2] !== 'child') { + process.stderr.write('supervised-process: expected child launch spec\n'); + process.exitCode = 2; + } else { + try { + childMain(process.argv[3]); + } catch (error) { + process.stderr.write(`supervised-process: ${boundedText(error && error.message, 1024)}\n`); + process.exitCode = 1; + } + } +} + +module.exports = { + CLAIM_SCHEMA, + SPEC_SCHEMA, + startOrRecoverSupervisedProcess, +}; diff --git a/packages/environment-factory/bin/runtime/target-execution-boundary.js b/packages/environment-factory/bin/runtime/target-execution-boundary.js new file mode 100644 index 00000000..2f7d4056 --- /dev/null +++ b/packages/environment-factory/bin/runtime/target-execution-boundary.js @@ -0,0 +1,114 @@ +'use strict'; + +const path = require('path'); + +const BOUNDARY_SCHEMA = 'testing-host.target-execution-boundary.v1'; +const SUPPORTED_MODE = 'trusted-fixture-exact'; + +function exactKeys(value, keys, label) { + if (!value || typeof value !== 'object' || Array.isArray(value) + || Object.keys(value).sort().join(',') !== [...keys].sort().join(',')) { + throw new Error(`${label} fields are invalid`); + } +} + +function validRepository(repository) { + if (!repository || typeof repository !== 'object' || Array.isArray(repository)) return false; + if (Object.keys(repository).sort().join(',') !== 'commit_sha,url') return false; + if (typeof repository.url !== 'string' || repository.url.length > 2048 + || !/^https:\/\/[^/@]+\/[^?#]+$/.test(repository.url) || repository.url.includes('@')) return false; + return /^[0-9a-f]{40}$/.test(String(repository.commit_sha || '')); +} + +function sameRepository(left, right) { + return validRepository(left) && validRepository(right) + && left.url === right.url && left.commit_sha === right.commit_sha; +} + +function safeHostPolicyAuthority(authority) { + if (!authority || typeof authority !== 'object' || Array.isArray(authority)) return false; + if (Object.keys(authority).sort().join(',') !== 'kind,ref') return false; + return authority.kind === 'host-policy' + && typeof authority.ref === 'string' && /^[A-Za-z0-9][A-Za-z0-9._/-]{2,255}$/.test(authority.ref) + && !authority.ref.includes('..') && !authority.ref.includes('//'); +} + +function safeArtifactPath(value) { + return typeof value === 'string' && value.startsWith('.testing/') && !path.isAbsolute(value) + && !value.includes('\\') && !/[\x00-\x20\x7f]/.test(value) + && value.split('/').every((segment) => segment !== '' && segment !== '.' && segment !== '..'); +} + +function assertConfigOutsideOperationArtifacts(runtimeConfigRef, artifactRoot) { + if (!runtimeConfigRef || runtimeConfigRef.kind !== 'artifact' + || !safeArtifactPath(runtimeConfigRef.ref) || !safeArtifactPath(artifactRoot)) { + throw new Error('target execution boundary requires safe runtime config and artifact root paths'); + } + const configPath = path.posix.normalize(runtimeConfigRef.ref); + const operationRoot = path.posix.normalize(artifactRoot); + if (!configPath.startsWith('.testing/host/')) { + throw new Error('target execution runtime config must be in the Host control namespace'); + } + if (!operationRoot.startsWith('.testing/runs/')) { + throw new Error('target execution artifact root must be in the run namespace'); + } + if (configPath === operationRoot || configPath.startsWith(`${operationRoot}/`)) { + throw new Error('target execution runtime config must be Host-owned outside the operation artifact root'); + } +} + +function validateTargetExecutionBoundary(boundary, repository, context = {}) { + if (boundary && typeof boundary === 'object' && !Array.isArray(boundary) + && (boundary.authorization_capability === true || boundary.execution_authorized === true)) { + throw new Error('target execution boundary must remain a non-authorizing admission prerequisite'); + } + if (!boundary || typeof boundary !== 'object' || Array.isArray(boundary) + || Object.keys(boundary).sort().join(',') !== [ + 'schema', 'mode', 'target_class', 'repository', 'authority', 'policy_revision', + 'authorization_capability', 'execution_authorized', + ].sort().join(',')) { + throw new Error('HOST_RUNTIME_ISOLATION_REQUIRED: target execution boundary is missing or malformed'); + } + exactKeys(boundary, [ + 'schema', 'mode', 'target_class', 'repository', 'authority', 'policy_revision', + 'authorization_capability', 'execution_authorized', + ], + 'target execution boundary'); + if (boundary.schema !== BOUNDARY_SCHEMA) { + throw new Error('HOST_RUNTIME_ISOLATION_REQUIRED: target execution boundary schema is unsupported'); + } + if (boundary.mode !== SUPPORTED_MODE) { + throw new Error('HOST_RUNTIME_ISOLATION_REQUIRED: unsupported target execution boundary mode'); + } + if (boundary.target_class !== 'host-owned-exact-trusted-fixture') { + throw new Error('HOST_RUNTIME_ISOLATION_REQUIRED: target is not a Host-owned exact trusted fixture'); + } + if (boundary.authorization_capability !== false || boundary.execution_authorized !== false) { + throw new Error('target execution boundary must remain a non-authorizing admission prerequisite'); + } + if (!validRepository(boundary.repository)) { + throw new Error('HOST_RUNTIME_ISOLATION_REQUIRED: target execution boundary repository is invalid'); + } + if (!safeHostPolicyAuthority(boundary.authority)) { + throw new Error('HOST_RUNTIME_ISOLATION_REQUIRED: target execution boundary authority is invalid'); + } + if (typeof boundary.policy_revision !== 'string' + || !/^[A-Za-z0-9][A-Za-z0-9._-]{2,179}$/.test(boundary.policy_revision)) { + throw new Error('HOST_RUNTIME_ISOLATION_REQUIRED: target execution boundary policy revision is invalid'); + } + if (repository !== undefined && !sameRepository(boundary.repository, repository)) { + throw new Error('HOST_RUNTIME_ISOLATION_REQUIRED: target repository is not the exact trusted fixture'); + } + if (context.runtimeConfigRef !== undefined || context.artifactRoot !== undefined) { + assertConfigOutsideOperationArtifacts(context.runtimeConfigRef, context.artifactRoot); + } + return boundary; +} + +module.exports = { + BOUNDARY_SCHEMA, + SUPPORTED_MODE, + assertConfigOutsideOperationArtifacts, + sameRepository, + validateTargetExecutionBoundary, +}; diff --git a/packages/environment-factory/bin/runtime/workspace.js b/packages/environment-factory/bin/runtime/workspace.js index d3b31c92..a7332ab9 100644 --- a/packages/environment-factory/bin/runtime/workspace.js +++ b/packages/environment-factory/bin/runtime/workspace.js @@ -4,6 +4,11 @@ const crypto = require('crypto'); const fs = require('fs'); const path = require('path'); const { spawnSync } = require('child_process'); +const { + minimalEnvironment, + releaseWorkerEnvironment, + verifyWorkerEnvironment, +} = require('./common'); function sha256(value) { return crypto.createHash('sha256').update(String(value)).digest('hex'); @@ -40,24 +45,36 @@ function sameRepository(left, right) { return left && right && left.url === right.url && left.commit_sha === right.commit_sha; } -function gitOutput(workspaceRoot, argv, label) { - const result = spawnSync('git', argv, { - cwd: workspaceRoot, - encoding: 'utf8', - shell: false, - timeout: 5_000, - windowsHide: true, +function gitOutput(workspaceRoot, argv, label, request) { + const environment = minimalEnvironment({}, { + schema: 'environment-factory.workspace-integrity-isolation.v1', + operation_id: request.operation_id, + repository: request.repository, + purpose: label, }); - if (result.error || result.status !== 0) throw new Error(`workspace ${label} is unavailable`); - return String(result.stdout || '').trim(); + try { + verifyWorkerEnvironment(environment); + const result = spawnSync('git', argv, { + cwd: workspaceRoot, + encoding: 'utf8', + env: environment, + shell: false, + timeout: 5_000, + windowsHide: true, + }); + if (result.error || result.status !== 0) throw new Error(`workspace ${label} is unavailable`); + return String(result.stdout || '').trim(); + } finally { + releaseWorkerEnvironment(environment); + } } -function currentCommit(workspaceRoot) { - return gitOutput(workspaceRoot, ['rev-parse', 'HEAD'], 'commit'); +function currentCommit(workspaceRoot, request) { + return gitOutput(workspaceRoot, ['rev-parse', 'HEAD'], 'commit', request); } -function trackedChanges(workspaceRoot) { - return gitOutput(workspaceRoot, ['status', '--porcelain', '--untracked-files=no'], 'tracked status'); +function trackedChanges(workspaceRoot, request) { + return gitOutput(workspaceRoot, ['status', '--porcelain', '--untracked-files=no'], 'tracked status', request); } function resolveWorkspace(request) { @@ -80,10 +97,11 @@ function resolveWorkspace(request) { throw new Error('working_directory differs from workspace binding'); } const workspaceRoot = fs.realpathSync(resource.path); - if (currentCommit(workspaceRoot) !== resource.repository.commit_sha) { + const isolationRequest = { ...request, repository: resource.repository }; + if (currentCommit(workspaceRoot, isolationRequest) !== resource.repository.commit_sha) { throw new Error('workspace commit binding is invalid'); } - if (request.require_clean === true && trackedChanges(workspaceRoot) !== '') { + if (request.require_clean === true && trackedChanges(workspaceRoot, isolationRequest) !== '') { throw new Error('workspace tracked files differ from the approved commit'); } const candidate = path.resolve(workspaceRoot, resource.working_directory); diff --git a/packages/environment-factory/runtime.lua b/packages/environment-factory/runtime.lua index 9f1915d1..41995461 100644 --- a/packages/environment-factory/runtime.lua +++ b/packages/environment-factory/runtime.lua @@ -419,6 +419,7 @@ function R.production(options) release_listener_claims(request.operation_id) end if not ok then error(verified, 0) end + verified.claim_id = nil return verified end diff --git a/packages/environment-factory/tests/fixtures/runtime/source/app.js b/packages/environment-factory/tests/fixtures/runtime/source/app.js index 528ff489..070ac81b 100644 --- a/packages/environment-factory/tests/fixtures/runtime/source/app.js +++ b/packages/environment-factory/tests/fixtures/runtime/source/app.js @@ -2,6 +2,9 @@ const http = require('http'); const protocol = require('fixture-protocol'); +const { assertCredentialIsolation } = require('./credential-isolation'); + +assertCredentialIsolation(); const middlewarePort = Number(process.argv[2]); diff --git a/packages/environment-factory/tests/fixtures/runtime/source/credential-isolation.js b/packages/environment-factory/tests/fixtures/runtime/source/credential-isolation.js new file mode 100644 index 00000000..cdc226a0 --- /dev/null +++ b/packages/environment-factory/tests/fixtures/runtime/source/credential-isolation.js @@ -0,0 +1,27 @@ +'use strict'; + +const path = require('path'); + +function assertCredentialIsolation() { + for (const key of ['GH_TOKEN', 'GITHUB_TOKEN', 'SSH_AUTH_SOCK', 'GIT_ASKPASS', 'SSH_ASKPASS']) { + if (process.env[key]) throw new Error(`worker inherited forbidden authority: ${key}`); + } + const home = process.env.HOME || ''; + const nullDevice = process.platform === 'win32' ? 'NUL' : '/dev/null'; + if (path.basename(path.dirname(home)) !== 'worker-homes' + || process.env.GIT_CONFIG_NOSYSTEM !== '1' + || process.env.GIT_CONFIG_GLOBAL !== nullDevice + || process.env.GIT_CONFIG_KEY_0 !== 'credential.helper' + || process.env.GIT_CONFIG_VALUE_0 !== '' + || process.env.GIT_CONFIG_KEY_1 !== 'core.askPass' + || process.env.GIT_CONFIG_VALUE_1 !== '' + || process.env.GIT_CONFIG_KEY_2 !== 'core.fsmonitor' + || process.env.GIT_CONFIG_VALUE_2 !== 'false' + || process.env.GIT_CONFIG_KEY_3 !== 'core.hooksPath' + || process.env.GIT_CONFIG_VALUE_3 !== nullDevice + || process.env.GIT_TERMINAL_PROMPT !== '0') { + throw new Error('worker credential isolation controls are incomplete'); + } +} + +module.exports = { assertCredentialIsolation }; diff --git a/packages/environment-factory/tests/fixtures/runtime/source/database_service.py b/packages/environment-factory/tests/fixtures/runtime/source/database_service.py index 2fbe9a1f..3f6806ba 100644 --- a/packages/environment-factory/tests/fixtures/runtime/source/database_service.py +++ b/packages/environment-factory/tests/fixtures/runtime/source/database_service.py @@ -14,6 +14,32 @@ EVIDENCE_ROOT = Path(os.environ["FKST_FIXTURE_EVIDENCE_DIR"]) +def assert_credential_isolation(): + for key in ("GH_TOKEN", "GITHUB_TOKEN", "SSH_AUTH_SOCK", "GIT_ASKPASS", "SSH_ASKPASS"): + if os.environ.get(key): + raise RuntimeError(f"worker inherited forbidden authority: {key}") + home = Path(os.environ.get("HOME", "")) + null_device = "NUL" if os.name == "nt" else "/dev/null" + if ( + home.parent.name != "worker-homes" + or os.environ.get("GIT_CONFIG_NOSYSTEM") != "1" + or os.environ.get("GIT_CONFIG_GLOBAL") != null_device + or os.environ.get("GIT_CONFIG_KEY_0") != "credential.helper" + or os.environ.get("GIT_CONFIG_VALUE_0") != "" + or os.environ.get("GIT_CONFIG_KEY_1") != "core.askPass" + or os.environ.get("GIT_CONFIG_VALUE_1") != "" + or os.environ.get("GIT_CONFIG_KEY_2") != "core.fsmonitor" + or os.environ.get("GIT_CONFIG_VALUE_2") != "false" + or os.environ.get("GIT_CONFIG_KEY_3") != "core.hooksPath" + or os.environ.get("GIT_CONFIG_VALUE_3") != null_device + or os.environ.get("GIT_TERMINAL_PROMPT") != "0" + ): + raise RuntimeError("worker credential isolation controls are incomplete") + + +assert_credential_isolation() + + def inherited_listener(): if os.environ.get("FKST_LISTEN_FDS") != "1": raise RuntimeError("database requires exactly one inherited listener") diff --git a/packages/environment-factory/tests/fixtures/runtime/source/middleware.js b/packages/environment-factory/tests/fixtures/runtime/source/middleware.js index e2c3e1ee..bc6aef35 100644 --- a/packages/environment-factory/tests/fixtures/runtime/source/middleware.js +++ b/packages/environment-factory/tests/fixtures/runtime/source/middleware.js @@ -2,6 +2,9 @@ const http = require('http'); const protocol = require('fixture-protocol'); +const { assertCredentialIsolation } = require('./credential-isolation'); + +assertCredentialIsolation(); const databasePort = Number(process.argv[2]); diff --git a/packages/environment-factory/tests/fixtures/runtime/source/phase.js b/packages/environment-factory/tests/fixtures/runtime/source/phase.js index cbcc77e7..ceb057a3 100644 --- a/packages/environment-factory/tests/fixtures/runtime/source/phase.js +++ b/packages/environment-factory/tests/fixtures/runtime/source/phase.js @@ -3,12 +3,14 @@ const fs = require('fs'); const path = require('path'); const protocol = require('fixture-protocol'); +const { assertCredentialIsolation } = require('./credential-isolation'); const phase = process.argv[2]; const targetPort = Number(process.argv[3]); const fixtureRoot = path.join(process.cwd(), '.fixture'); async function main() { + assertCredentialIsolation(); if (phase === 'build') { fs.mkdirSync(fixtureRoot, { recursive: true }); fs.writeFileSync(path.join(fixtureRoot, 'build'), `${protocol.packageName}\n`); diff --git a/packages/environment-factory/tests/hermetic_e2e_test.lua b/packages/environment-factory/tests/hermetic_e2e_test.lua index 22517abe..12cd8ed9 100644 --- a/packages/environment-factory/tests/hermetic_e2e_test.lua +++ b/packages/environment-factory/tests/hermetic_e2e_test.lua @@ -288,6 +288,16 @@ local function request_fixture(ctx, ports, commit_sha) now = "2026-07-16T00:00:30Z", trusted_authorities = {}, repository_mirrors = { [repository.url] = ctx.source_root }, + target_execution_boundary = { + schema = "testing-host.target-execution-boundary.v1", + mode = "trusted-fixture-exact", + target_class = "host-owned-exact-trusted-fixture", + repository = repository, + authority = { kind = "host-policy", ref = "fixtures/environment-factory-hermetic" }, + policy_revision = "environment-factory-hermetic-v1", + authorization_capability = false, + execution_authorized = false, + }, command_environment = { FKST_FIXTURE_EVIDENCE_DIR = ctx.evidence_root, }, diff --git a/packages/environment-factory/tests/node_runtime_test.js b/packages/environment-factory/tests/node_runtime_test.js index 665877d3..7c981a7b 100644 --- a/packages/environment-factory/tests/node_runtime_test.js +++ b/packages/environment-factory/tests/node_runtime_test.js @@ -9,10 +9,21 @@ const { spawn } = require('child_process'); const { acquireLock, authorizationArtifact, + minimalEnvironment, + releaseWorkerEnvironment, + releaseWorkerEnvironmentLease, stableStringify, + verifyWorkerEnvironment, + workerEnvironmentLease, } = require('../bin/runtime/common'); +const { validateTargetExecutionBoundary } = require('../bin/runtime/target-execution-boundary'); +const { startOrRecoverSupervisedProcess } = require('../bin/runtime/supervised-process'); const { runMeasuredCommand } = require('../bin/runtime/measured-command'); -const { listenersOwnedByProcessGroup } = require('../bin/runtime/platform'); +const { + listenersOwnedByProcessGroup, + processGroupState, + terminateProcessGroup, +} = require('../bin/runtime/platform'); const { dispatch, initialReadinessState, sha256 } = require('../bin/environment-factory-runtime'); function delay(ms) { @@ -107,10 +118,94 @@ async function main() { const artifactRoot = `.testing/runs/environment-node-runtime-${process.pid}`; const hostRoot = `.testing/host/environment-factory/environment-node-runtime-${process.pid}`; const previousDurable = process.env.FKST_DURABLE_ROOT; + const previousRuntime = process.env.FKST_RUNTIME_ROOT; process.env.FKST_DURABLE_ROOT = path.join(temp, 'durable'); + process.env.FKST_RUNTIME_ROOT = path.join(temp, 'runtime'); fs.rmSync(artifactRoot, { recursive: true, force: true }); fs.rmSync(hostRoot, { recursive: true, force: true }); + let crashWindowResource = null; + let firstStartupEnvironment = null; + let firstLease = null; try { + const ambientHome = path.join(temp, 'ambient-home'); + fs.mkdirSync(ambientHome); + const isolated = minimalEnvironment({ FKST_SAFE_MARKER: 'present' }, 'node-runtime-isolation'); + assert.notStrictEqual(isolated.HOME, ambientHome); + assert.strictEqual(path.basename(path.dirname(isolated.HOME)), 'worker-homes'); + assert.strictEqual(isolated.FKST_SAFE_MARKER, 'present'); + assert.strictEqual(isolated.GIT_CONFIG_NOSYSTEM, '1'); + assert.strictEqual(isolated.GIT_CONFIG_GLOBAL, process.platform === 'win32' ? 'NUL' : '/dev/null'); + assert.strictEqual(isolated.GIT_TERMINAL_PROMPT, '0'); + assert.strictEqual(isolated.GIT_CONFIG_COUNT, '4'); + assert.strictEqual(isolated.GIT_CONFIG_KEY_2, 'core.fsmonitor'); + assert.strictEqual(isolated.GIT_CONFIG_VALUE_2, 'false'); + assert.strictEqual(isolated.GIT_CONFIG_KEY_3, 'core.hooksPath'); + assert.strictEqual(isolated.GIT_CONFIG_VALUE_3, process.platform === 'win32' ? 'NUL' : '/dev/null'); + assert.strictEqual(verifyWorkerEnvironment(isolated), true); + const secondIsolated = minimalEnvironment({}, 'node-runtime-isolation'); + assert.notStrictEqual(secondIsolated.HOME, isolated.HOME); + assert.strictEqual(path.dirname(secondIsolated.HOME), path.dirname(isolated.HOME)); + assert.strictEqual(fs.lstatSync(secondIsolated.HOME).isSymbolicLink(), false); + if (process.platform !== 'win32') { + assert.strictEqual(fs.lstatSync(secondIsolated.HOME).mode & 0o077, 0); + } + const isolatedHome = isolated.HOME; + const secondIsolatedHome = secondIsolated.HOME; + assert.strictEqual(releaseWorkerEnvironment(isolated), true); + assert.strictEqual(releaseWorkerEnvironment(secondIsolated), true); + assert.strictEqual(fs.existsSync(isolatedHome), false); + assert.strictEqual(fs.existsSync(secondIsolatedHome), false); + for (const key of [ + 'HOME', 'USERPROFILE', 'XDG_CONFIG_HOME', 'GH_CONFIG_DIR', 'GH_TOKEN', 'GITHUB_TOKEN', + 'GIT_CONFIG_GLOBAL', 'GIT_ASKPASS', 'SSH_AUTH_SOCK', 'SSH_ASKPASS', 'CREDENTIAL_HELPER', + ]) { + assert.throws(() => minimalEnvironment({ [key]: 'forbidden' }, 'node-runtime-isolation'), + /forbidden worker authority key/); + } + const symlinkRuntime = path.join(temp, 'symlink-runtime'); + const symlinkTarget = path.join(temp, 'symlink-runtime-target'); + fs.mkdirSync(symlinkTarget); + fs.symlinkSync(symlinkTarget, symlinkRuntime); + process.env.FKST_RUNTIME_ROOT = symlinkRuntime; + assert.throws(() => minimalEnvironment({}, 'symlink-runtime'), /not a real directory/); + process.env.FKST_RUNTIME_ROOT = path.join(temp, 'runtime'); + + const trustedRepository = { + url: 'https://example.invalid/testing/trusted-fixture.git', + commit_sha: '1'.repeat(40), + }; + const boundary = { + schema: 'testing-host.target-execution-boundary.v1', + mode: 'trusted-fixture-exact', + target_class: 'host-owned-exact-trusted-fixture', + repository: trustedRepository, + authority: { kind: 'host-policy', ref: 'fixtures/runtime-target-boundary' }, + policy_revision: 'runtime-test-boundary-v1', + authorization_capability: false, + execution_authorized: false, + }; + assert.deepStrictEqual(validateTargetExecutionBoundary(boundary, trustedRepository, { + runtimeConfigRef: { kind: 'artifact', ref: `${hostRoot}/runtime-config.json` }, + artifactRoot, + }), boundary); + assert.throws(() => validateTargetExecutionBoundary(boundary, { + ...trustedRepository, commit_sha: '2'.repeat(40), + }), /HOST_RUNTIME_ISOLATION_REQUIRED/); + assert.throws(() => validateTargetExecutionBoundary({ ...boundary, mode: 'isolated-runtime' }), + /HOST_RUNTIME_ISOLATION_REQUIRED/); + assert.throws(() => validateTargetExecutionBoundary(undefined, trustedRepository), + /HOST_RUNTIME_ISOLATION_REQUIRED/); + assert.throws(() => validateTargetExecutionBoundary({ ...boundary, repository: { + url: 'git@example.invalid:testing/trusted-fixture.git', commit_sha: '1'.repeat(40), + } }, trustedRepository), /HOST_RUNTIME_ISOLATION_REQUIRED/); + assert.throws(() => validateTargetExecutionBoundary({ + ...boundary, authorization_capability: true, + }, trustedRepository), /non-authorizing admission prerequisite/); + assert.throws(() => validateTargetExecutionBoundary(boundary, trustedRepository, { + runtimeConfigRef: { kind: 'artifact', ref: `${artifactRoot}/runtime-config.json` }, + artifactRoot, + }), /Host control namespace/); + const lockPath = path.join(temp, 'stale.lock'); fs.mkdirSync(lockPath); fs.writeFileSync(path.join(lockPath, 'owner.json'), `${JSON.stringify({ @@ -125,6 +220,284 @@ async function main() { release(); assert.strictEqual(fs.existsSync(lockPath), false); + const concurrentLockPath = path.join(temp, 'concurrent-stale.lock'); + const concurrentActivePath = path.join(temp, 'concurrent-stale.active'); + const concurrentEntriesPath = path.join(temp, 'concurrent-stale.entries'); + const concurrentViolationPath = path.join(temp, 'concurrent-stale.violation'); + fs.mkdirSync(concurrentLockPath); + fs.writeFileSync(path.join(concurrentLockPath, 'owner.json'), `${JSON.stringify({ + schema: 'environment-factory.lock-owner.v1', + pid: 2147483647, + process_start_identity: 'dead process', + token: 'concurrent-stale-owner-token', + })}\n`); + const lockModulePath = path.resolve(__dirname, '../bin/runtime/common.js'); + const contenderSource = [ + "'use strict';", + "const fs = require('fs');", + 'const { acquireLock } = require(process.argv[1]);', + 'const lockPath = process.argv[2];', + 'const activePath = process.argv[3];', + 'const entriesPath = process.argv[4];', + 'const violationPath = process.argv[5];', + 'const release = acquireLock(lockPath, 5000);', + 'let ownsActive = false;', + 'try {', + " fs.writeFileSync(activePath, String(process.pid), { flag: 'wx' });", + ' ownsActive = true;', + " fs.appendFileSync(entriesPath, String(process.pid) + '\\n');", + ' Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100);', + '} catch (error) {', + " fs.appendFileSync(violationPath, String(process.pid) + ':' + error.code + '\\n');", + ' process.exitCode = 1;', + '} finally {', + ' if (ownsActive) fs.unlinkSync(activePath);', + ' release();', + '}', + ].join('\n'); + const contenders = Array.from({ length: 4 }, () => new Promise((resolve, reject) => { + const child = spawn(process.execPath, [ + '-e', contenderSource, lockModulePath, concurrentLockPath, concurrentActivePath, + concurrentEntriesPath, concurrentViolationPath, + ], { stdio: ['ignore', 'ignore', 'pipe'] }); + let stderr = ''; + child.stderr.on('data', (chunk) => { stderr += chunk.toString(); }); + child.once('error', reject); + child.once('close', (code) => { + if (code === 0) resolve(); + else reject(new Error(`concurrent stale-lock contender failed: ${stderr}`)); + }); + })); + await Promise.all(contenders); + assert.strictEqual(fs.existsSync(concurrentViolationPath), false); + assert.strictEqual(fs.readFileSync(concurrentEntriesPath, 'utf8').trim().split('\n').length, 4); + assert.strictEqual(fs.existsSync(concurrentLockPath), false); + assert.strictEqual(fs.existsSync(`${concurrentLockPath}.takeover.active`), false); + + const crashedTakeoverLockPath = path.join(temp, 'crashed-takeover.lock'); + const crashedTakeoverReadyPath = path.join(temp, 'crashed-takeover.ready'); + const crashedTakeoverEnteredPath = path.join(temp, 'crashed-takeover.entered'); + fs.mkdirSync(crashedTakeoverLockPath); + fs.writeFileSync(path.join(crashedTakeoverLockPath, 'owner.json'), `${JSON.stringify({ + schema: 'environment-factory.lock-owner.v1', + pid: 2147483647, + process_start_identity: 'dead process', + token: 'crashed-takeover-stale-owner-token', + })}\n`); + const crashedTakeoverSource = [ + "'use strict';", + "const fs = require('fs');", + 'const { acquireLock } = require(process.argv[1]);', + 'const release = acquireLock(process.argv[2], 5000, {', + ' afterTakeoverAcquired() {', + " fs.writeFileSync(process.argv[3], 'ready', { flag: 'wx' });", + ' Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1000);', + ' },', + '});', + "fs.writeFileSync(process.argv[4], 'entered', { flag: 'wx' });", + 'release();', + ].join('\n'); + const crashedTakeover = spawn(process.execPath, [ + '-e', crashedTakeoverSource, lockModulePath, crashedTakeoverLockPath, + crashedTakeoverReadyPath, crashedTakeoverEnteredPath, + ], { stdio: ['ignore', 'ignore', 'pipe'] }); + let crashedTakeoverStderr = ''; + crashedTakeover.stderr.on('data', (chunk) => { crashedTakeoverStderr += chunk.toString(); }); + const readyDeadline = Date.now() + 5_000; + while (!fs.existsSync(crashedTakeoverReadyPath) && Date.now() < readyDeadline) await delay(10); + assert.strictEqual(fs.existsSync(crashedTakeoverReadyPath), true); + const takeoverMarkerPath = `${crashedTakeoverLockPath}.takeover.active`; + const takeoverOwner = JSON.parse(fs.readFileSync(takeoverMarkerPath, 'utf8')); + process.kill(takeoverOwner.pid, 'SIGKILL'); + const crashedTakeoverExit = await new Promise((resolve, reject) => { + crashedTakeover.once('error', reject); + crashedTakeover.once('close', (code) => resolve(code)); + }); + assert.notStrictEqual(crashedTakeoverExit, 0, crashedTakeoverStderr); + assert.strictEqual(fs.existsSync(crashedTakeoverEnteredPath), false); + const recoveredAfterGuardCrash = acquireLock(crashedTakeoverLockPath, 5_000); + recoveredAfterGuardCrash(); + assert.strictEqual(fs.existsSync(crashedTakeoverLockPath), false); + assert.strictEqual(fs.existsSync(takeoverMarkerPath), false); + + const startupCounter = path.join(temp, 'supervised-startup-count.txt'); + const startupClaim = path.join(temp, 'supervised-startup', 'claim.json'); + const startupBinding = { + schema: 'environment-factory.resource.v1', + kind: 'process', + operation_id: 'crash-window-operation', + ref: 'crash-window-process', + effect_id: 'crash-window-effect', + argv_sha256: sha256('crash-window-argv'), + ownership_token: sha256('crash-window-owner'), + runtime_ports: [], + repository: trustedRepository, + cleaned: false, + }; + const startupArgv = [process.execPath, '-e', [ + "const fs = require('fs');", + `fs.appendFileSync(${JSON.stringify(startupCounter)}, 'started\\n');`, + 'setInterval(() => {}, 1000);', + ].join('')]; + let launchedSupervisorPid = null; + const interrupted = startOrRecoverSupervisedProcess({ + claimPath: startupClaim, + argv: startupArgv, + cwd: temp, + createEnvironment() { + firstStartupEnvironment = minimalEnvironment({}, 'supervised-crash-window-first'); + firstLease = workerEnvironmentLease(firstStartupEnvironment); + return firstStartupEnvironment; + }, + binding: startupBinding, + afterLaunch(pid) { + launchedSupervisorPid = pid; + return false; + }, + }); + assert.strictEqual(interrupted.interrupted, true); + assert.strictEqual(fs.existsSync(startupClaim), true); + assert.strictEqual(fs.existsSync(firstLease.home), true); + + let recoveryEnvironmentCreated = false; + const recoveredStartup = startOrRecoverSupervisedProcess({ + claimPath: startupClaim, + argv: startupArgv, + cwd: temp, + createEnvironment() { + recoveryEnvironmentCreated = true; + return minimalEnvironment({}, 'supervised-crash-window-recovery'); + }, + binding: startupBinding, + }); + assert.strictEqual(recoveredStartup.interrupted, false); + assert.strictEqual(recoveredStartup.state, 'running'); + assert.strictEqual(recoveredStartup.resource.pid, launchedSupervisorPid); + assert.strictEqual(recoveredStartup.resource.worker_environment_lease.home, firstLease.home); + assert.strictEqual(recoveredStartup.environment_retained, false); + assert.strictEqual(recoveryEnvironmentCreated, false); + crashWindowResource = recoveredStartup.resource; + + const counterDeadline = Date.now() + 2_000; + while (Date.now() < counterDeadline) { + if (fs.existsSync(startupCounter) && fs.readFileSync(startupCounter, 'utf8') === 'started\n') break; + await delay(10); + } + assert.strictEqual(fs.readFileSync(startupCounter, 'utf8'), 'started\n'); + let replayEnvironmentCreated = false; + const replayedStartup = startOrRecoverSupervisedProcess({ + claimPath: startupClaim, + argv: startupArgv, + cwd: temp, + createEnvironment() { + replayEnvironmentCreated = true; + return minimalEnvironment({}, 'supervised-crash-window-replay'); + }, + binding: startupBinding, + }); + assert.strictEqual(replayedStartup.resource.pid, launchedSupervisorPid); + assert.strictEqual(replayedStartup.resource.worker_environment_lease.home, firstLease.home); + assert.strictEqual(replayEnvironmentCreated, false); + await delay(50); + assert.strictEqual(fs.readFileSync(startupCounter, 'utf8'), 'started\n'); + assert.strictEqual(processGroupState(replayedStartup.resource).alive, true); + assert.strictEqual(terminateProcessGroup(replayedStartup.resource, 2_000).released, true); + crashWindowResource = null; + assert.strictEqual(releaseWorkerEnvironmentLease(firstLease), true); + assert.strictEqual(fs.existsSync(firstLease.home), false); + assert.strictEqual(releaseWorkerEnvironment(firstStartupEnvironment), true); + + let failedLaunchLease = null; + const failedLaunchClaim = path.join(temp, 'supervised-failed-launch', 'claim.json'); + assert.throws(() => startOrRecoverSupervisedProcess({ + claimPath: failedLaunchClaim, + argv: [process.execPath, '-e', 'process.exit(0)'], + cwd: temp, + createEnvironment() { + const environment = minimalEnvironment({}, 'supervised-failed-launch'); + failedLaunchLease = workerEnvironmentLease(environment); + return environment; + }, + binding: { ...startupBinding, effect_id: 'failed-launch-effect' }, + registrationTimeoutMs: 250, + beforeSupervisorLaunch() { + throw new Error('simulated supervisor launch failure'); + }, + }), /simulated supervisor launch failure/); + assert.strictEqual(JSON.parse(fs.readFileSync(failedLaunchClaim, 'utf8')).state, 'revoked'); + assert.strictEqual(fs.existsSync(failedLaunchLease.home), false); + + let failedClaimLease = null; + assert.throws(() => startOrRecoverSupervisedProcess({ + claimPath: path.join(temp, 'supervised-failed-claim', 'claim.json'), + argv: [process.execPath, '-e', 'process.exit(0)'], + cwd: temp, + createEnvironment() { + const environment = minimalEnvironment({}, 'supervised-failed-claim'); + failedClaimLease = workerEnvironmentLease(environment); + return environment; + }, + binding: { ...startupBinding, effect_id: 'failed-claim-effect' }, + beforeClaimPersist() { + throw new Error('simulated startup claim persistence failure'); + }, + }), /simulated startup claim persistence failure/); + assert.strictEqual(fs.existsSync(failedClaimLease.home), false); + + let substitutedLaunchLease = null; + const substitutedMarker = path.join(temp, 'supervised-substituted-command.txt'); + const substitutedLaunch = startOrRecoverSupervisedProcess({ + claimPath: path.join(temp, 'supervised-substituted-launch', 'claim.json'), + argv: [process.execPath, '-e', 'process.exit(0)'], + cwd: temp, + createEnvironment() { + const environment = minimalEnvironment({}, 'supervised-substituted-launch'); + substitutedLaunchLease = workerEnvironmentLease(environment); + return environment; + }, + binding: { ...startupBinding, effect_id: 'substituted-launch-effect' }, + registrationTimeoutMs: 250, + beforeSupervisorLaunch(claim) { + const substituted = { + schema: 'fkst.supervised-process-launch.v1', + startup_token: claim.startup_token, + binding_sha256: claim.binding_sha256, + claim_path: path.join(temp, 'supervised-substituted-launch', 'claim.json'), + argv: [process.execPath, '-e', `require('fs').writeFileSync(${JSON.stringify(substitutedMarker)}, 'bad')`], + cwd: temp, + inherited_fd_count: 0, + inherited_fd_identities: [], + }; + fs.writeFileSync(claim.launch_spec_path, `${stableStringify(substituted)}\n`); + }, + }); + assert.strictEqual(substitutedLaunch.state, 'revoked'); + assert.strictEqual(fs.existsSync(substitutedMarker), false); + assert.strictEqual(fs.existsSync(substitutedLaunchLease.home), false); + + let symlinkedLaunchLease = null; + const symlinkedMarker = path.join(temp, 'supervised-symlinked-command.txt'); + const symlinkedLaunch = startOrRecoverSupervisedProcess({ + claimPath: path.join(temp, 'supervised-symlinked-launch', 'claim.json'), + argv: [process.execPath, '-e', `require('fs').writeFileSync(${JSON.stringify(symlinkedMarker)}, 'bad')`], + cwd: temp, + createEnvironment() { + const environment = minimalEnvironment({}, 'supervised-symlinked-launch'); + symlinkedLaunchLease = workerEnvironmentLease(environment); + return environment; + }, + binding: { ...startupBinding, effect_id: 'symlinked-launch-effect' }, + registrationTimeoutMs: 250, + beforeSupervisorLaunch(claim) { + const original = `${claim.launch_spec_path}.original`; + fs.renameSync(claim.launch_spec_path, original); + fs.symlinkSync(original, claim.launch_spec_path); + }, + }); + assert.strictEqual(symlinkedLaunch.state, 'revoked'); + assert.strictEqual(fs.existsSync(symlinkedMarker), false); + assert.strictEqual(fs.existsSync(symlinkedLaunchLease.home), false); + for (let index = 0; index < 10; index += 1) { const result = await runMeasuredCommand([process.execPath, '-e', 'process.exit(0)'], { timeoutMs: 2_000, @@ -274,8 +647,13 @@ async function main() { const rotated = await dispatch('load-state', { ref: stateRef, runtime_config_ref: runtimeConfigRef }); assert.strictEqual(rotated.authenticated, false); } finally { + if (crashWindowResource) terminateProcessGroup(crashWindowResource, 2_000); + if (firstLease) releaseWorkerEnvironmentLease(firstLease); + if (firstStartupEnvironment) releaseWorkerEnvironment(firstStartupEnvironment); if (previousDurable === undefined) delete process.env.FKST_DURABLE_ROOT; else process.env.FKST_DURABLE_ROOT = previousDurable; + if (previousRuntime === undefined) delete process.env.FKST_RUNTIME_ROOT; + else process.env.FKST_RUNTIME_ROOT = previousRuntime; fs.rmSync(temp, { recursive: true, force: true }); fs.rmSync(artifactRoot, { recursive: true, force: true }); fs.rmSync(hostRoot, { recursive: true, force: true }); diff --git a/packages/local-qa-host-adapter/tests/host_boundary_test.lua b/packages/local-qa-host-adapter/tests/host_boundary_test.lua index 10169b31..8a524805 100644 --- a/packages/local-qa-host-adapter/tests/host_boundary_test.lua +++ b/packages/local-qa-host-adapter/tests/host_boundary_test.lua @@ -141,6 +141,7 @@ local function grant_runtime(mutate) claims = claims + 1 return { status = "claimed", claim_id = "local-qa-host-claim" } end, + reconcile_preauthorization_claim = function() return claims > 0 end, grant_values = function() return { grant_id = "local-qa-host-grant", @@ -216,6 +217,7 @@ local function terminal_fixture() write_artifact = function() return true end, artifact_digest = function() return digest("a") end, claim_preauthorization = function() return { status = "claimed", claim_id = "unused" } end, + reconcile_preauthorization_claim = function() return true end, grant_values = function() return {} end, record_terminal = function(value) records = records + 1 diff --git a/packages/testing-runner/structured_execution.lua b/packages/testing-runner/structured_execution.lua index 27008cde..847c9f28 100644 --- a/packages/testing-runner/structured_execution.lua +++ b/packages/testing-runner/structured_execution.lua @@ -630,7 +630,15 @@ function M.run(request, ports) local verified = ports.verify_grant({ grant = grant.value, grant_raw = grant.raw, + grant_ref = request.execution_grant_ref, grant_sha256 = grant.digest, + preauthorization_ref = request.preauthorization_ref, + preauthorization_sha256 = preauthorization.digest, + plan_ref = request.test_plan_ref, + plan_sha256 = plan.digest, + environment_receipt_ref = request.environment_receipt_ref, + environment_receipt_sha256 = environment.digest, + repository = copy(request.repository), now = now, artifact_root = request.artifact_root, operation_id = environment.value.operation_id, @@ -642,9 +650,14 @@ function M.run(request, ports) end local claim = ports.replay_guard({ grant_id = grant.value.grant_id, + grant_ref = request.execution_grant_ref, grant_sha256 = grant.digest, + preauthorization_ref = request.preauthorization_ref, + preauthorization_sha256 = preauthorization.digest, parent_authorization_sha256 = grant.value.parent_authorization_sha256, + plan_ref = request.test_plan_ref, plan_sha256 = plan.digest, + environment_receipt_ref = request.environment_receipt_ref, environment_receipt_sha256 = environment.digest, repository = request.repository, artifact_root = request.artifact_root, diff --git a/packages/testing-runner/tests/workflow_qa_host_adapter_test.lua b/packages/testing-runner/tests/workflow_qa_host_adapter_test.lua index cfefb38a..1b21bfd7 100644 --- a/packages/testing-runner/tests/workflow_qa_host_adapter_test.lua +++ b/packages/testing-runner/tests/workflow_qa_host_adapter_test.lua @@ -246,6 +246,13 @@ local function runtime(request, materials, mutate) claimed = claimed or { value = fixtures.copy(value), claim_id = "host-adapter-claim" } return { status = "claimed", claim_id = claimed.claim_id } end, + reconcile_preauthorization_claim = function(value) + if claimed == nil then return false end + for key, item in pairs(value) do + if not execution.equal(claimed.value[key], item) then return false end + end + return true + end, grant_values = function() return values() end, record_terminal = function(value) terminal = fixtures.copy(value) return true end, } @@ -287,6 +294,57 @@ return { t.eq(artifacts[request.grant_ref].value.grant_id, "host-adapter-grant") end, + test_grant_reads_are_digest_bound_and_replay_probe_is_durable_only = function() + local request, materials = fixture() + local ports = runtime(request, materials) + local original_load = ports.load_artifact + local reads = {} + ports.load_artifact = function(ref, expected_digest, options) + table.insert(reads, { + ref = ref, + expected_digest = expected_digest, + durable_only = type(options) == "table" and options.durable_only == true, + }) + return original_load(ref, expected_digest, options) + end + + adapter.handle_execution_grant(request, ports) + + t.eq(reads[1].ref, request.preauthorization_ref) + t.eq(reads[1].expected_digest, request.preauthorization_sha256) + t.eq(reads[2].ref, request.plan_ref) + t.eq(reads[2].expected_digest, request.plan_sha256) + t.eq(reads[3].ref, request.environment_receipt_ref) + t.eq(reads[3].expected_digest, request.environment_receipt_sha256) + t.eq(reads[4].ref, request.grant_ref) + t.eq(reads[4].expected_digest, nil) + t.eq(reads[4].durable_only, true) + t.eq(reads[5].ref, request.grant_ref) + t.eq(reads[5].expected_digest, digest("b")) + end, + + test_persisted_grant_reconciles_only_from_an_authenticated_host_claim = function() + local request, materials = fixture() + local ports, _, _, claims = runtime(request, materials) + adapter.handle_execution_grant(request, ports) + t.eq(claims(), 1) + local reconciliations = 0 + ports.reconcile_preauthorization_claim = function(value) + reconciliations = reconciliations + 1 + return value.preauthorization_ref == request.preauthorization_ref + and value.plan_ref == request.plan_ref + and value.environment_receipt_ref == request.environment_receipt_ref + end + adapter.handle_execution_grant(request, ports) + t.eq(reconciliations, 1) + t.eq(claims(), 1) + ports.reconcile_preauthorization_claim = function() return false end + t.raises(function() adapter.handle_execution_grant(request, ports) end) + t.eq(claims(), 1) + ports.reconcile_preauthorization_claim = nil + t.raises(function() adapter.handle_execution_grant(request, ports) end) + end, + test_grant_binding_environment_and_runtime_ports_fail_closed = function() local request, materials = fixture() local ports = runtime(request, materials) @@ -317,6 +375,44 @@ return { t.raises(function() adapter.handle_execution_grant(request, ports) end) end, + test_replayed_grant_rejects_authority_policy_evidence_and_capability_expansion = function() + local mutations = { + function(grant) grant.authority.ref = "foreign-host-policy" end, + function(grant) grant.policy_revision = "host-adapter-policy-v2" end, + function(grant) grant.evidence_ref.ref = "foreign-grant-evidence" end, + function(grant) table.insert(grant.http_capabilities[1].methods, "POST") end, + function(grant) table.insert(grant.http_capabilities[1].path_prefixes, "/admin") end, + } + for _, mutate in ipairs(mutations) do + local request, materials = fixture() + local ports, artifacts = runtime(request, materials) + adapter.handle_execution_grant(request, ports) + mutate(artifacts[request.grant_ref].value) + t.raises(function() adapter.handle_execution_grant(request, ports) end) + end + end, + + test_replayed_grant_rejects_cli_expansion_and_plan_capability_escalation = function() + local request, materials = fixture() + materials.plan.cases = { { + case_id = "health-cli", kind = "cli", argv = { "fixture-cli", "health" }, + timeout_seconds = 10, assertions = { { type = "exit-code", expected = 0 } }, + } } + materials.preauthorization.capabilities = { + cli = { { argv_prefix = { "fixture-cli", "health" } } }, http = {}, + } + local ports, artifacts = runtime(request, materials) + adapter.handle_execution_grant(request, ports) + artifacts[request.grant_ref].value.cli_capabilities[1].argv_prefix = { "fixture-cli" } + t.raises(function() adapter.handle_execution_grant(request, ports) end) + + request, materials = fixture() + ports, artifacts = runtime(request, materials) + adapter.handle_execution_grant(request, ports) + artifacts[request.plan_ref].value.cases[1].request.url = "http://127.0.0.1:4173/admin" + t.raises(function() adapter.handle_execution_grant(request, ports) end) + end, + test_failed_grant_write_emits_no_result_and_is_not_replayed = function() local request, materials = fixture() local ports, artifacts, _, claims = runtime(request, materials) diff --git a/packages/workflow-qa/core.lua b/packages/workflow-qa/core.lua index 601e4083..47d027e5 100644 --- a/packages/workflow-qa/core.lua +++ b/packages/workflow-qa/core.lua @@ -34,7 +34,7 @@ local function digest(ports, pointer) end local function load_bound(ports, pointer, expected_digest, label) - local artifact = ports.load_artifact(pointer) + local artifact = ports.load_artifact(pointer, expected_digest) if type(artifact) ~= "table" or artifact.digest ~= expected_digest or type(artifact.value) ~= "table" then error("workflow-qa: artifact-binding-unavailable: " .. label .. " immutable binding failed") end From 41a3263b52a027e02d59d6ed7bf6a976f7d6da4e Mon Sep 17 00:00:00 2001 From: Shaw Zheng Date: Fri, 11 Sep 2026 01:21:05 +0800 Subject: [PATCH 04/11] fix(testing): close crash recovery allocation windows --- .../generic-host/bin/generic-host-runtime.js | 6 +- .../bin/environment-factory-runtime.js | 3 +- .../bin/runtime/budgets.js | 3 +- .../environment-factory/bin/runtime/common.js | 202 ++++++++++++++---- .../bin/runtime/listener-claims.js | 2 +- .../bin/runtime/supervised-process.js | 174 ++++++++++----- .../tests/node_runtime_test.js | 157 ++++++++++++-- 7 files changed, 434 insertions(+), 113 deletions(-) diff --git a/examples/generic-host/bin/generic-host-runtime.js b/examples/generic-host/bin/generic-host-runtime.js index cc1daace..31e23e02 100755 --- a/examples/generic-host/bin/generic-host-runtime.js +++ b/examples/generic-host/bin/generic-host-runtime.js @@ -38,11 +38,11 @@ function sha256(value) { return crypto.createHash('sha256').update(String(value)).digest('hex'); } -function childProcessEnvironment(cwd) { +function childProcessEnvironment(cwd, reservation = null) { return minimalEnvironment({}, { schema: 'generic-host.worker-isolation.v1', cwd_sha256: sha256(path.resolve(cwd)), - }); + }, reservation && reservation.reservation_id); } function durableRoot() { @@ -1059,7 +1059,7 @@ function startApplication(projectRoot, payload) { claimPath: startupClaimPath, argv: payload.argv, cwd: workspace.path, - createEnvironment: () => childProcessEnvironment(workspace.path), + createEnvironment: (reservation) => childProcessEnvironment(workspace.path, reservation), binding, }); if (launch.interrupted || !launch.resource) fail('application startup was interrupted before registration'); diff --git a/packages/environment-factory/bin/environment-factory-runtime.js b/packages/environment-factory/bin/environment-factory-runtime.js index df4b3e9a..73f136bc 100644 --- a/packages/environment-factory/bin/environment-factory-runtime.js +++ b/packages/environment-factory/bin/environment-factory-runtime.js @@ -558,9 +558,10 @@ async function runArgvEffect(payload) { } const listenerNames = inheritedListenerNames(payload); const inheritedStdio = listenerNames.map((_name, index) => 3 + index); - const createSupervisedEnvironment = () => { + const createSupervisedEnvironment = (reservation) => { const environment = minimalEnvironment( config.command_environment || {}, workerIsolationIdentity(payload, 'run-argv'), + reservation && reservation.reservation_id, ); environment.FKST_LISTEN_FDS = String(listenerNames.length); environment.FKST_LISTEN_FDNAMES = listenerNames.join(':'); diff --git a/packages/environment-factory/bin/runtime/budgets.js b/packages/environment-factory/bin/runtime/budgets.js index d1997f44..50a5ca70 100644 --- a/packages/environment-factory/bin/runtime/budgets.js +++ b/packages/environment-factory/bin/runtime/budgets.js @@ -75,7 +75,8 @@ function createBudgetRuntime(deps) { const directory = path.join(durableRoot(), 'environment-factory', 'resources'); let names = []; try { names = fs.readdirSync(directory); } catch (error) { if (error.code !== 'ENOENT') throw error; } - return names.map((name) => readIfExists(path.join(directory, name))) + return names.filter((name) => name.endsWith('.json')) + .map((name) => readIfExists(path.join(directory, name))) .filter((resource) => resource && resource.kind === 'process' && resource.operation_id === operationId && !resource.cleaned && Number.isInteger(resource.pid)); } diff --git a/packages/environment-factory/bin/runtime/common.js b/packages/environment-factory/bin/runtime/common.js index 11de40e1..1b8adf2d 100644 --- a/packages/environment-factory/bin/runtime/common.js +++ b/packages/environment-factory/bin/runtime/common.js @@ -152,7 +152,19 @@ function removeOwnedDirectory(target, expectedIdentity, containmentRoot) { } function readLockOwner(lockPath) { - try { return readJson(path.join(lockPath, 'owner.json')); } catch (_error) { return null; } + try { + const lockStat = fs.lstatSync(lockPath); + if (lockStat.isSymbolicLink()) return null; + const ownerPath = lockStat.isDirectory() ? path.join(lockPath, 'owner.json') : lockPath; + const stat = fs.lstatSync(ownerPath); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_JSON_BYTES) return null; + const owner = JSON.parse(fs.readFileSync(ownerPath, 'utf8')); + if (!owner || owner.schema !== 'environment-factory.lock-owner.v1' + || !Number.isInteger(owner.pid) || owner.pid < 1 + || typeof owner.process_start_identity !== 'string' || owner.process_start_identity === '' + || typeof owner.token !== 'string' || owner.token === '') return null; + return owner; + } catch (_error) { return null; } } function lockOwnerIsStale(owner) { @@ -168,33 +180,92 @@ function sameLockOwner(left, right) { && left.process_start_identity === right.process_start_identity && left.token === right.token); } -function createDirectoryLock(lockPath, identity) { - fs.mkdirSync(lockPath); - const directoryIdentity = pathIdentity(lockPath); +function lockPathIdentity(lockPath) { + const stat = fs.lstatSync(lockPath); + if ((!stat.isDirectory() && !stat.isFile()) || stat.isSymbolicLink()) { + throw new Error(`lock path is not a real file or directory: ${lockPath}`); + } + return pathIdentity(lockPath); +} + +function lockPathStillMatches(lockPath, expectedIdentity) { + try { + return samePathIdentity(lockPathIdentity(lockPath), expectedIdentity); + } catch (error) { + if (error.code === 'ENOENT') return false; + throw error; + } +} + +function removeLockPath(lockPath, expectedIdentity) { + if (!samePathIdentity(lockPathIdentity(lockPath), expectedIdentity)) { + throw new Error(`lock path identity changed: ${lockPath}`); + } + const stat = fs.lstatSync(lockPath); + if (stat.isDirectory()) fs.rmSync(expectedIdentity.realpath, { recursive: true, force: true }); + else fs.unlinkSync(expectedIdentity.realpath); +} + +function lockOwnerBody(owner) { + return `${stableStringify(owner)}\n`; +} + +function pendingLockOwnerPath(lockPath, owner) { + return `${lockPath}.owner.${owner.pid}.${owner.token}`; +} + +function removeMatchingPendingLockOwner(lockPath, owner) { + const pendingPath = pendingLockOwnerPath(lockPath, owner); + try { + const stat = fs.lstatSync(pendingPath); + if (!stat.isFile() || stat.isSymbolicLink() + || fs.readFileSync(pendingPath, 'utf8') !== lockOwnerBody(owner)) return false; + fs.unlinkSync(pendingPath); + return true; + } catch (error) { + if (error.code === 'ENOENT') return false; + throw error; + } +} + +function createAtomicLock(lockPath, identity) { const owner = { schema: 'environment-factory.lock-owner.v1', pid: process.pid, process_start_identity: identity, token: crypto.randomBytes(16).toString('hex'), }; + const ownerBody = lockOwnerBody(owner); + const pendingPath = pendingLockOwnerPath(lockPath, owner); try { - fs.writeFileSync(path.join(lockPath, 'owner.json'), `${stableStringify(owner)}\n`, { flag: 'wx' }); + fs.writeFileSync(pendingPath, ownerBody, { flag: 'wx', mode: 0o600 }); + fs.linkSync(pendingPath, lockPath); } catch (error) { - if (samePathIdentity(pathIdentity(lockPath), directoryIdentity)) { - fs.rmSync(directoryIdentity.realpath, { recursive: true, force: true }); - } + try { fs.unlinkSync(pendingPath); } catch (_cleanupError) {} + throw error; + } + const lockIdentity = lockPathIdentity(lockPath); + if (readLockOwner(lockPath) === null || fs.readFileSync(lockPath, 'utf8') !== ownerBody) { + try { removeLockPath(lockPath, lockIdentity); } catch (_cleanupError) {} + try { fs.unlinkSync(pendingPath); } catch (_cleanupError) {} + throw new Error(`atomic lock publication failed: ${lockPath}`); + } + try { + fs.unlinkSync(pendingPath); + } catch (error) { + try { removeLockPath(lockPath, lockIdentity); } catch (_cleanupError) {} throw error; } let released = false; return { owner, - directoryIdentity, + lockIdentity, release() { if (released) return; const recorded = readLockOwner(lockPath); if (sameLockOwner(recorded, owner) - && samePathIdentity(pathIdentity(lockPath), directoryIdentity)) { - fs.rmSync(directoryIdentity.realpath, { recursive: true, force: true }); + && samePathIdentity(lockPathIdentity(lockPath), lockIdentity)) { + removeLockPath(lockPath, lockIdentity); } released = true; }, @@ -296,7 +367,7 @@ function acquireLock(lockPath, timeoutMs = LOCK_TIMEOUT_MS, options = {}) { continue; } try { - const acquired = createDirectoryLock(lockPath, identity); + const acquired = createAtomicLock(lockPath, identity); if (fs.existsSync(takeoverMarkerPath)) { acquired.release(); continue; @@ -305,7 +376,17 @@ function acquireLock(lockPath, timeoutMs = LOCK_TIMEOUT_MS, options = {}) { } catch (error) { if (error.code !== 'EEXIST') throw error; } + let observedIdentity; + try { + observedIdentity = lockPathIdentity(lockPath); + } catch (error) { + if (error.code === 'ENOENT') continue; + throw error; + } const observedOwner = readLockOwner(lockPath); + if (observedOwner === null) { + throw new Error(`ownerless or malformed lock cannot be recovered safely: ${lockPath}`); + } if (lockOwnerIsStale(observedOwner)) { let takeover; try { @@ -317,30 +398,27 @@ function acquireLock(lockPath, timeoutMs = LOCK_TIMEOUT_MS, options = {}) { try { if (typeof options.afterTakeoverAcquired === 'function') options.afterTakeoverAcquired(takeover); takeover.assertHeld(); - const currentOwner = readLockOwner(lockPath); - if (sameLockOwner(currentOwner, observedOwner) && lockOwnerIsStale(currentOwner)) { - const staleIdentity = pathIdentity(lockPath); - const confirmedOwner = readLockOwner(lockPath); - if (sameLockOwner(confirmedOwner, currentOwner) - && samePathIdentity(pathIdentity(lockPath), staleIdentity)) { - takeover.assertHeld(); - fs.rmSync(staleIdentity.realpath, { recursive: true, force: true }); - while (true) { + const confirmedOwner = readLockOwner(lockPath); + if (sameLockOwner(confirmedOwner, observedOwner) + && lockPathStillMatches(lockPath, observedIdentity) + && lockOwnerIsStale(confirmedOwner)) { + removeLockPath(lockPath, observedIdentity); + removeMatchingPendingLockOwner(lockPath, confirmedOwner); + while (true) { + try { + const acquired = createAtomicLock(lockPath, identity); try { - const acquired = createDirectoryLock(lockPath, identity); - try { - takeover.assertHeld(); - } catch (error) { - acquired.release(); - throw error; - } - takeover.release(); - return acquired.release; + takeover.assertHeld(); } catch (error) { - if (error.code !== 'EEXIST') throw error; - if (Date.now() >= deadline) throw new Error(`lock takeover timeout: ${lockPath}`); - sleep(10); + acquired.release(); + throw error; } + takeover.release(); + return acquired.release; + } catch (error) { + if (error.code !== 'EEXIST') throw error; + if (Date.now() >= deadline) throw new Error(`lock takeover timeout: ${lockPath}`); + sleep(10); } } } @@ -452,7 +530,7 @@ function requireOwnedDirectory(directory, { privateDirectory = false } = {}) { return fs.realpathSync(directory); } -function minimalEnvironment(extra = {}, isolationKey = 'shared-runtime-command') { +function minimalEnvironment(extra = {}, isolationKey = 'shared-runtime-command', reservationId = null) { if (!extra || typeof extra !== 'object' || Array.isArray(extra)) { throw new Error('command environment must be an object'); } @@ -473,21 +551,62 @@ function minimalEnvironment(extra = {}, isolationKey = 'shared-runtime-command') throw new Error('worker isolation identity is invalid'); } const identityBody = stableStringify(identity); + if (reservationId !== null && !/^[0-9a-f]{32}$/.test(String(reservationId))) { + throw new Error('worker environment reservation is invalid'); + } const configuredRuntimeRoot = path.resolve( process.env.FKST_RUNTIME_ROOT || path.join('.testing', 'runtime'), ); fs.mkdirSync(configuredRuntimeRoot, { recursive: true }); const runtimeRoot = requireOwnedDirectory(configuredRuntimeRoot); const homesRoot = requireOwnedDirectory(path.join(runtimeRoot, 'worker-homes'), { privateDirectory: true }); - const home = fs.mkdtempSync(path.join(homesRoot, `${sha256(identityBody).slice(0, 24)}-`)); + const identitySha256 = sha256(identityBody); + const leaseId = reservationId || crypto.randomBytes(16).toString('hex'); + const home = reservationId === null + ? fs.mkdtempSync(path.join(homesRoot, `${identitySha256.slice(0, 24)}-`)) + : path.join(homesRoot, `reserved-${reservationId}`); + let homeCreated = reservationId === null; + if (reservationId !== null) { + try { + fs.mkdirSync(home, { mode: 0o700 }); + homeCreated = true; + } catch (error) { + if (error.code !== 'EEXIST') throw error; + requireOwnedDirectory(home, { privateDirectory: true }); + } + } if (process.platform !== 'win32') fs.chmodSync(home, 0o700); const marker = `${stableStringify({ schema: 'fkst.worker-home-identity.v1', - identity_sha256: sha256(identityBody), - lease_id: crypto.randomBytes(16).toString('hex'), + identity_sha256: identitySha256, + lease_id: leaseId, })}\n`; const markerPath = path.join(home, '.fkst-worker-home.json'); - fs.writeFileSync(markerPath, marker, { flag: 'wx', mode: 0o600 }); + const pendingMarkerPath = `${markerPath}.pending`; + if (!homeCreated) { + if (fs.existsSync(markerPath)) { + const stat = fs.lstatSync(markerPath); + if (!stat.isFile() || stat.isSymbolicLink() || fs.readFileSync(markerPath, 'utf8') !== marker) { + throw new Error('worker environment reservation binding changed'); + } + } else { + const entries = fs.readdirSync(home); + if (entries.some((entry) => entry !== path.basename(pendingMarkerPath))) { + throw new Error('worker environment reservation is not recoverable'); + } + if (fs.existsSync(pendingMarkerPath)) { + const pending = fs.lstatSync(pendingMarkerPath); + if (!pending.isFile() || pending.isSymbolicLink()) { + throw new Error('worker environment reservation marker is invalid'); + } + fs.unlinkSync(pendingMarkerPath); + } + } + } + if (!fs.existsSync(markerPath)) { + fs.writeFileSync(pendingMarkerPath, marker, { flag: 'wx', mode: 0o600 }); + fs.renameSync(pendingMarkerPath, markerPath); + } const configHome = path.join(home, '.config'); requireOwnedDirectory(configHome, { privateDirectory: true }); requireOwnedDirectory(path.join(configHome, 'gh'), { privateDirectory: true }); @@ -519,12 +638,13 @@ function minimalEnvironment(extra = {}, isolationKey = 'shared-runtime-command') writable: false, value: { schema: 'fkst.worker-home-lease.v1', + lease_id: leaseId, home, home_identity: pathIdentity(home), homes_root: homesRoot, homes_root_identity: pathIdentity(homesRoot), marker_sha256: sha256(marker), - identity_sha256: sha256(identityBody), + identity_sha256: identitySha256, released: false, }, }); @@ -538,6 +658,7 @@ function workerEnvironmentLease(environment) { } return { schema: lease.schema, + lease_id: lease.lease_id, home: lease.home, home_identity: { ...lease.home_identity }, homes_root: lease.homes_root, @@ -549,6 +670,7 @@ function workerEnvironmentLease(environment) { function verifyWorkerEnvironmentLease(lease) { if (!lease || lease.schema !== 'fkst.worker-home-lease.v1' + || typeof lease.lease_id !== 'string' || !/^[0-9a-f]{32}$/.test(lease.lease_id) || typeof lease.home !== 'string' || typeof lease.homes_root !== 'string' || !samePathIdentity(pathIdentity(lease.homes_root), lease.homes_root_identity) || !samePathIdentity(pathIdentity(lease.home), lease.home_identity)) { @@ -565,7 +687,7 @@ function verifyWorkerEnvironmentLease(lease) { const value = JSON.parse(marker); if (value.schema !== 'fkst.worker-home-identity.v1' || value.identity_sha256 !== lease.identity_sha256 - || typeof value.lease_id !== 'string' || !/^[0-9a-f]{32}$/.test(value.lease_id)) { + || value.lease_id !== lease.lease_id) { throw new Error('worker environment marker binding changed'); } for (const directory of [path.join(home, '.config'), path.join(home, '.config', 'gh')]) { diff --git a/packages/environment-factory/bin/runtime/listener-claims.js b/packages/environment-factory/bin/runtime/listener-claims.js index a4d53bee..72eae6f7 100644 --- a/packages/environment-factory/bin/runtime/listener-claims.js +++ b/packages/environment-factory/bin/runtime/listener-claims.js @@ -55,7 +55,7 @@ function createListenerClaims(deps) { if (error.code === 'ENOENT') return []; throw error; } - return entries.flatMap((entry) => { + return entries.filter((entry) => entry.endsWith('.json')).flatMap((entry) => { const resource = readIfExists(path.join(root, entry)); if (!resource || resource.schema !== 'environment-factory.resource.v1' || resource.kind !== 'process' || resource.operation_id !== operationId || resource.cleaned === true) { diff --git a/packages/environment-factory/bin/runtime/supervised-process.js b/packages/environment-factory/bin/runtime/supervised-process.js index 89e40afe..14dc519c 100644 --- a/packages/environment-factory/bin/runtime/supervised-process.js +++ b/packages/environment-factory/bin/runtime/supervised-process.js @@ -33,27 +33,40 @@ function readIfExists(filePath) { } function readJsonNoFollow(filePath) { - const before = fs.lstatSync(filePath); - if (!before.isFile() || before.isSymbolicLink() || before.size > 2 * 1024 * 1024) { - throw new Error('supervised startup claim is not a bounded regular file'); - } - const descriptor = fs.openSync(filePath, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)); - try { - const openedBefore = fs.fstatSync(descriptor); - if (!sameFileIdentity(fileIdentity(before), fileIdentity(openedBefore))) { - throw new Error('supervised startup claim identity changed'); + const maxBytes = 2 * 1024 * 1024; + for (let attempt = 0; attempt < 20; attempt += 1) { + const before = fs.lstatSync(filePath); + if (!before.isFile() || before.isSymbolicLink() || before.size > maxBytes) { + throw new Error('supervised startup claim is not a bounded regular file'); } - const body = fs.readFileSync(descriptor, 'utf8'); - const openedAfter = fs.fstatSync(descriptor); - const after = fs.lstatSync(filePath); - if (!sameFileIdentity(fileIdentity(openedBefore), fileIdentity(openedAfter)) - || !sameFileIdentity(fileIdentity(openedAfter), fileIdentity(after))) { - throw new Error('supervised startup claim changed while reading'); + const descriptor = fs.openSync(filePath, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)); + try { + const openedBefore = fs.fstatSync(descriptor); + if (!sameFileIdentity(fileIdentity(before), fileIdentity(openedBefore))) { + sleep(2); + continue; + } + const buffer = Buffer.allocUnsafe(maxBytes + 1); + let length = 0; + while (length < buffer.length) { + const count = fs.readSync(descriptor, buffer, length, buffer.length - length, null); + if (count === 0) break; + length += count; + } + if (length > maxBytes) throw new Error('supervised startup claim exceeds its read bound'); + const openedAfter = fs.fstatSync(descriptor); + const after = fs.lstatSync(filePath); + if (!sameFileIdentity(fileIdentity(openedBefore), fileIdentity(openedAfter)) + || !sameFileIdentity(fileIdentity(openedAfter), fileIdentity(after))) { + sleep(2); + continue; + } + return JSON.parse(buffer.subarray(0, length).toString('utf8')); + } finally { + fs.closeSync(descriptor); } - return JSON.parse(body); - } finally { - fs.closeSync(descriptor); } + throw new Error('supervised startup claim did not stabilize while reading'); } function writeClaimAtomic(filePath, value) { @@ -69,6 +82,8 @@ function requireAbsoluteFile(filePath, label) { } function validClaim(claim, bindingSha256) { + const state = claim && claim.state; + const leasePending = state === 'allocating'; return Boolean(claim && claim.schema === CLAIM_SCHEMA && claim.version === 1 && /^[0-9a-f]{32}$/.test(String(claim.startup_token || '')) && claim.binding_sha256 === bindingSha256 @@ -82,10 +97,21 @@ function validClaim(claim, bindingSha256) { && Number.isInteger(claim.inherited_fd_count) && claim.inherited_fd_count >= 0 && Array.isArray(claim.inherited_fd_identities) && claim.inherited_fd_identities.length === claim.inherited_fd_count - && /^[0-9a-f]{64}$/.test(String(claim.worker_environment_lease_sha256 || '')) - && claim.worker_environment_lease - && ['preparing', 'prepared', 'registered', 'running', 'exited', 'failed', 'revoked'].includes(claim.state) - && (claim.state === 'preparing' || sameFileIdentityShape(claim.launch_spec_identity))); + && claim.worker_environment_reservation + && claim.worker_environment_reservation.schema === 'fkst.worker-home-reservation.v1' + && claim.worker_environment_reservation.reservation_id === claim.startup_token + && claim.worker_environment_reservation.binding_sha256 === sha256(stableStringify({ + binding_sha256: claim.binding_sha256, + argv_sha256: claim.argv_sha256, + cwd: claim.cwd, + inherited_fd_identities: claim.inherited_fd_identities, + })) + && (leasePending + ? claim.worker_environment_lease === null && claim.worker_environment_lease_sha256 === null + : claim.worker_environment_lease + && /^[0-9a-f]{64}$/.test(String(claim.worker_environment_lease_sha256 || ''))) + && ['allocating', 'preparing', 'prepared', 'registered', 'running', 'exited', 'failed', 'revoked'].includes(state) + && (state === 'allocating' || state === 'preparing' || sameFileIdentityShape(claim.launch_spec_identity))); } function sameFileIdentityShape(identity) { @@ -194,6 +220,7 @@ function startOrRecoverSupervisedProcess(options) { const registrationTimeoutMs = Math.max(250, Math.min(Number(options.registrationTimeoutMs) || 5_000, 30_000)); let created = false; let suppliedLease = null; + let environment = null; const release = acquireLock(`${claimPath}.lock`); try { let claim = readIfExists(claimPath); @@ -201,11 +228,6 @@ function startOrRecoverSupervisedProcess(options) { throw new Error('supervised startup claim binding differs'); } if (claim === null) { - const environment = options.environment || (typeof options.createEnvironment === 'function' - ? options.createEnvironment() : null); - if (!environment) throw new Error('supervised process environment is required for a new launch'); - verifyWorkerEnvironment(environment); - suppliedLease = workerEnvironmentLease(environment); const token = crypto.randomBytes(16).toString('hex'); const launchSpecPath = `${claimPath}.launch-${token}.json`; const spec = { @@ -220,10 +242,20 @@ function startOrRecoverSupervisedProcess(options) { }; const specBody = `${stableStringify(spec)}\n`; const now = Date.now(); + const reservation = { + schema: 'fkst.worker-home-reservation.v1', + reservation_id: token, + binding_sha256: sha256(stableStringify({ + binding_sha256: bindingSha256, + argv_sha256: sha256(stableStringify(argv)), + cwd, + inherited_fd_identities: inheritedFdIdentities, + })), + }; claim = { schema: CLAIM_SCHEMA, version: 1, - state: 'preparing', + state: 'allocating', startup_token: token, binding_sha256: bindingSha256, created_at_epoch_ms: now, @@ -236,35 +268,79 @@ function startOrRecoverSupervisedProcess(options) { cwd_identity: cwdIdentity, inherited_fd_count: inheritedStdio.length, inherited_fd_identities: inheritedFdIdentities, - worker_environment_lease: suppliedLease, - worker_environment_lease_sha256: sha256(stableStringify(suppliedLease)), + worker_environment_reservation: reservation, + worker_environment_lease: null, + worker_environment_lease_sha256: null, pid: null, pgid: null, process_start_identity: null, }; fs.mkdirSync(path.dirname(claimPath), { recursive: true }); - try { - if (typeof options.beforeClaimPersist === 'function') options.beforeClaimPersist(); - writeClaimAtomic(claimPath, claim); - } catch (error) { - if (fs.existsSync(claimPath)) { - try { - writeClaimAtomic(claimPath, { - ...claim, - state: 'revoked', - failure_reason: 'startup-claim-persistence-failed', - failed_at_epoch_ms: Date.now(), - }); - } catch (_claimError) {} + if (typeof options.beforeClaimPersist === 'function') options.beforeClaimPersist(); + writeClaimAtomic(claimPath, claim); + } + if (claim.state === 'allocating' || claim.state === 'preparing') { + if (options.environment || typeof options.createEnvironment !== 'function') { + throw new Error('recoverable supervised process environment factory is required'); + } + environment = options.createEnvironment({ + reservation_id: claim.worker_environment_reservation.reservation_id, + binding_sha256: claim.worker_environment_reservation.binding_sha256, + }); + if (!environment) throw new Error('supervised process environment is required for a new launch'); + verifyWorkerEnvironment(environment); + suppliedLease = workerEnvironmentLease(environment); + if (suppliedLease.lease_id !== claim.worker_environment_reservation.reservation_id) { + releaseWorkerEnvironmentLease(suppliedLease); + suppliedLease = null; + throw new Error('supervised worker environment reservation differs'); + } + if (claim.state === 'allocating') { + if (typeof options.afterEnvironmentCreated === 'function' + && options.afterEnvironmentCreated(environment, claim) === false) { + return { interrupted: true, environment_retained: true }; } + claim = { + ...claim, + state: 'preparing', + worker_environment_lease: suppliedLease, + worker_environment_lease_sha256: sha256(stableStringify(suppliedLease)), + }; + writeClaimAtomic(claimPath, claim); + } else if (stableStringify(suppliedLease) !== stableStringify(claim.worker_environment_lease)) { releaseWorkerEnvironmentLease(suppliedLease); - throw error; + suppliedLease = null; + throw new Error('supervised worker environment lease differs'); } + } + if (claim.state === 'preparing') { + const spec = { + schema: SPEC_SCHEMA, + startup_token: claim.startup_token, + binding_sha256: bindingSha256, + claim_path: claimPath, + argv, + cwd, + inherited_fd_count: inheritedStdio.length, + inherited_fd_identities: inheritedFdIdentities, + }; + const specBody = `${stableStringify(spec)}\n`; let launched = false; try { - fs.writeFileSync(launchSpecPath, specBody, { flag: 'wx', mode: 0o600 }); - fs.chmodSync(launchSpecPath, 0o600); - const launchSpecStat = fs.lstatSync(launchSpecPath); + if (fs.existsSync(claim.launch_spec_path)) { + const existing = fs.lstatSync(claim.launch_spec_path); + if (!existing.isFile() || existing.isSymbolicLink()) { + throw new Error('supervised launch spec is not a regular file'); + } + if (fs.readFileSync(claim.launch_spec_path, 'utf8') !== specBody) { + fs.unlinkSync(claim.launch_spec_path); + } + } + if (!fs.existsSync(claim.launch_spec_path)) { + fs.writeFileSync(claim.launch_spec_path, specBody, { flag: 'wx', mode: 0o600 }); + } + fs.chmodSync(claim.launch_spec_path, 0o600); + const launchSpecStat = fs.lstatSync(claim.launch_spec_path); if (!launchSpecStat.isFile() || launchSpecStat.isSymbolicLink()) { throw new Error('supervised launch spec is not a regular file'); } @@ -272,7 +348,7 @@ function startOrRecoverSupervisedProcess(options) { claim = { ...claim, state: 'prepared', launch_spec_identity: launchSpecIdentity }; writeClaimAtomic(claimPath, claim); if (typeof options.beforeSupervisorLaunch === 'function') options.beforeSupervisorLaunch(claim); - const supervisor = spawn(process.execPath, [__filename, 'child', launchSpecPath], { + const supervisor = spawn(process.execPath, [__filename, 'child', claim.launch_spec_path], { cwd, env: environment, shell: false, @@ -295,7 +371,7 @@ function startOrRecoverSupervisedProcess(options) { failed_at_epoch_ms: Date.now(), }; writeClaimAtomic(claimPath, claim); - releaseWorkerEnvironmentLease(suppliedLease); + if (suppliedLease) releaseWorkerEnvironmentLease(suppliedLease); } throw error; } diff --git a/packages/environment-factory/tests/node_runtime_test.js b/packages/environment-factory/tests/node_runtime_test.js index 7c981a7b..840586d4 100644 --- a/packages/environment-factory/tests/node_runtime_test.js +++ b/packages/environment-factory/tests/node_runtime_test.js @@ -30,6 +30,19 @@ function delay(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } +async function removeTreeEventually(target, timeoutMs = 2_000) { + const deadline = Date.now() + timeoutMs; + while (true) { + try { + fs.rmSync(target, { recursive: true, force: true }); + return; + } catch (error) { + if (!['ENOTEMPTY', 'EEXIST'].includes(error && error.code) || Date.now() >= deadline) throw error; + await delay(10); + } + } +} + function stopGroup(child) { if (!child || !Number.isInteger(child.pid)) return; try { process.kill(-child.pid, 'SIGKILL'); } catch (_error) { @@ -215,11 +228,39 @@ async function main() { token: 'stale-owner-token', })}\n`); const release = acquireLock(lockPath, 250); - const recovered = JSON.parse(fs.readFileSync(path.join(lockPath, 'owner.json'), 'utf8')); + const recovered = JSON.parse(fs.readFileSync(lockPath, 'utf8')); assert.strictEqual(recovered.pid, process.pid); release(); assert.strictEqual(fs.existsSync(lockPath), false); + const staleAtomicLockPath = path.join(temp, 'stale-atomic.lock'); + const staleAtomicOwner = { + schema: 'environment-factory.lock-owner.v1', + pid: 2147483647, + process_start_identity: 'dead process', + token: '1'.repeat(32), + }; + const staleAtomicPendingPath = `${staleAtomicLockPath}.owner.${staleAtomicOwner.pid}.${staleAtomicOwner.token}`; + fs.writeFileSync(staleAtomicPendingPath, `${stableStringify(staleAtomicOwner)}\n`); + fs.linkSync(staleAtomicPendingPath, staleAtomicLockPath); + const releaseStaleAtomic = acquireLock(staleAtomicLockPath, 250); + assert.strictEqual(fs.existsSync(staleAtomicPendingPath), false); + releaseStaleAtomic(); + assert.strictEqual(fs.existsSync(staleAtomicLockPath), false); + + const ownerlessLockPath = path.join(temp, 'ownerless.lock'); + fs.mkdirSync(ownerlessLockPath); + assert.throws(() => acquireLock(ownerlessLockPath, 250), /cannot be recovered safely/); + assert.strictEqual(fs.lstatSync(ownerlessLockPath).isDirectory(), true); + fs.rmSync(ownerlessLockPath, { recursive: true }); + + const malformedOwnerLockPath = path.join(temp, 'malformed-owner.lock'); + fs.mkdirSync(malformedOwnerLockPath); + fs.writeFileSync(path.join(malformedOwnerLockPath, 'owner.json'), '{}\n'); + assert.throws(() => acquireLock(malformedOwnerLockPath, 250), /cannot be recovered safely/); + assert.strictEqual(fs.lstatSync(malformedOwnerLockPath).isDirectory(), true); + fs.rmSync(malformedOwnerLockPath, { recursive: true }); + const concurrentLockPath = path.join(temp, 'concurrent-stale.lock'); const concurrentActivePath = path.join(temp, 'concurrent-stale.active'); const concurrentEntriesPath = path.join(temp, 'concurrent-stale.entries'); @@ -344,8 +385,10 @@ async function main() { claimPath: startupClaim, argv: startupArgv, cwd: temp, - createEnvironment() { - firstStartupEnvironment = minimalEnvironment({}, 'supervised-crash-window-first'); + createEnvironment(reservation) { + firstStartupEnvironment = minimalEnvironment( + {}, 'supervised-crash-window-first', reservation.reservation_id, + ); firstLease = workerEnvironmentLease(firstStartupEnvironment); return firstStartupEnvironment; }, @@ -364,9 +407,11 @@ async function main() { claimPath: startupClaim, argv: startupArgv, cwd: temp, - createEnvironment() { + createEnvironment(reservation) { recoveryEnvironmentCreated = true; - return minimalEnvironment({}, 'supervised-crash-window-recovery'); + return minimalEnvironment( + {}, 'supervised-crash-window-recovery', reservation.reservation_id, + ); }, binding: startupBinding, }); @@ -389,9 +434,11 @@ async function main() { claimPath: startupClaim, argv: startupArgv, cwd: temp, - createEnvironment() { + createEnvironment(reservation) { replayEnvironmentCreated = true; - return minimalEnvironment({}, 'supervised-crash-window-replay'); + return minimalEnvironment( + {}, 'supervised-crash-window-replay', reservation.reservation_id, + ); }, binding: startupBinding, }); @@ -413,8 +460,10 @@ async function main() { claimPath: failedLaunchClaim, argv: [process.execPath, '-e', 'process.exit(0)'], cwd: temp, - createEnvironment() { - const environment = minimalEnvironment({}, 'supervised-failed-launch'); + createEnvironment(reservation) { + const environment = minimalEnvironment( + {}, 'supervised-failed-launch', reservation.reservation_id, + ); failedLaunchLease = workerEnvironmentLease(environment); return environment; }, @@ -432,7 +481,7 @@ async function main() { claimPath: path.join(temp, 'supervised-failed-claim', 'claim.json'), argv: [process.execPath, '-e', 'process.exit(0)'], cwd: temp, - createEnvironment() { + createEnvironment(reservation) { const environment = minimalEnvironment({}, 'supervised-failed-claim'); failedClaimLease = workerEnvironmentLease(environment); return environment; @@ -442,7 +491,75 @@ async function main() { throw new Error('simulated startup claim persistence failure'); }, }), /simulated startup claim persistence failure/); - assert.strictEqual(fs.existsSync(failedClaimLease.home), false); + assert.strictEqual(failedClaimLease, null); + + const preClaimCrashClaim = path.join(temp, 'supervised-pre-claim-crash', 'claim.json'); + const preClaimCrashReady = path.join(temp, 'supervised-pre-claim-crash.ready.json'); + const preClaimCrashBinding = { ...startupBinding, effect_id: 'pre-claim-crash-effect' }; + const preClaimCrashArgv = [process.execPath, '-e', 'process.exit(0)']; + const supervisorModulePath = path.resolve(__dirname, '../bin/runtime/supervised-process.js'); + const commonModulePath = path.resolve(__dirname, '../bin/runtime/common.js'); + const preClaimCrashSource = [ + "'use strict';", + "const fs = require('fs');", + 'const { startOrRecoverSupervisedProcess } = require(process.argv[1]);', + 'const { minimalEnvironment } = require(process.argv[2]);', + 'const claimPath = process.argv[3];', + 'const cwd = process.argv[4];', + 'const readyPath = process.argv[5];', + 'const binding = JSON.parse(process.argv[6]);', + 'const argv = JSON.parse(process.argv[7]);', + 'startOrRecoverSupervisedProcess({', + ' claimPath, argv, cwd, binding,', + ' createEnvironment(reservation) {', + " return minimalEnvironment({}, 'supervised-pre-claim-hard-crash', reservation.reservation_id);", + ' },', + ' afterEnvironmentCreated(environment) {', + " fs.writeFileSync(readyPath, JSON.stringify({ home: environment.HOME }), { flag: 'wx' });", + ' Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10000);', + ' },', + '});', + ].join('\n'); + const preClaimCrash = spawn(process.execPath, [ + '-e', preClaimCrashSource, supervisorModulePath, commonModulePath, + preClaimCrashClaim, temp, preClaimCrashReady, JSON.stringify(preClaimCrashBinding), + JSON.stringify(preClaimCrashArgv), + ], { stdio: ['ignore', 'ignore', 'pipe'] }); + let preClaimCrashStderr = ''; + preClaimCrash.stderr.on('data', (chunk) => { preClaimCrashStderr += chunk.toString(); }); + const preClaimReadyDeadline = Date.now() + 5_000; + while (!fs.existsSync(preClaimCrashReady) && Date.now() < preClaimReadyDeadline) await delay(10); + assert.strictEqual(fs.existsSync(preClaimCrashReady), true, preClaimCrashStderr); + const reservedHome = JSON.parse(fs.readFileSync(preClaimCrashReady, 'utf8')).home; + const allocatingClaim = JSON.parse(fs.readFileSync(preClaimCrashClaim, 'utf8')); + assert.strictEqual(allocatingClaim.state, 'allocating'); + assert.strictEqual(allocatingClaim.worker_environment_lease, null); + process.kill(preClaimCrash.pid, 'SIGKILL'); + await new Promise((resolve, reject) => { + preClaimCrash.once('error', reject); + preClaimCrash.once('close', resolve); + }); + let recoveredPreClaimEnvironment = null; + const recoveredPreClaim = startOrRecoverSupervisedProcess({ + claimPath: preClaimCrashClaim, + argv: preClaimCrashArgv, + cwd: temp, + binding: preClaimCrashBinding, + createEnvironment(reservation) { + recoveredPreClaimEnvironment = minimalEnvironment( + {}, 'supervised-pre-claim-hard-crash', reservation.reservation_id, + ); + return recoveredPreClaimEnvironment; + }, + }); + assert.notStrictEqual(recoveredPreClaim.state, 'allocating'); + assert.strictEqual(recoveredPreClaim.resource.worker_environment_lease.home, reservedHome); + assert.strictEqual(workerEnvironmentLease(recoveredPreClaimEnvironment).home, reservedHome); + assert.strictEqual(releaseWorkerEnvironmentLease( + recoveredPreClaim.resource.worker_environment_lease, + ), true); + assert.strictEqual(fs.existsSync(reservedHome), false); + assert.strictEqual(releaseWorkerEnvironment(recoveredPreClaimEnvironment), true); let substitutedLaunchLease = null; const substitutedMarker = path.join(temp, 'supervised-substituted-command.txt'); @@ -450,8 +567,10 @@ async function main() { claimPath: path.join(temp, 'supervised-substituted-launch', 'claim.json'), argv: [process.execPath, '-e', 'process.exit(0)'], cwd: temp, - createEnvironment() { - const environment = minimalEnvironment({}, 'supervised-substituted-launch'); + createEnvironment(reservation) { + const environment = minimalEnvironment( + {}, 'supervised-substituted-launch', reservation.reservation_id, + ); substitutedLaunchLease = workerEnvironmentLease(environment); return environment; }, @@ -481,8 +600,10 @@ async function main() { claimPath: path.join(temp, 'supervised-symlinked-launch', 'claim.json'), argv: [process.execPath, '-e', `require('fs').writeFileSync(${JSON.stringify(symlinkedMarker)}, 'bad')`], cwd: temp, - createEnvironment() { - const environment = minimalEnvironment({}, 'supervised-symlinked-launch'); + createEnvironment(reservation) { + const environment = minimalEnvironment( + {}, 'supervised-symlinked-launch', reservation.reservation_id, + ); symlinkedLaunchLease = workerEnvironmentLease(environment); return environment; }, @@ -654,9 +775,9 @@ async function main() { else process.env.FKST_DURABLE_ROOT = previousDurable; if (previousRuntime === undefined) delete process.env.FKST_RUNTIME_ROOT; else process.env.FKST_RUNTIME_ROOT = previousRuntime; - fs.rmSync(temp, { recursive: true, force: true }); - fs.rmSync(artifactRoot, { recursive: true, force: true }); - fs.rmSync(hostRoot, { recursive: true, force: true }); + await removeTreeEventually(temp); + await removeTreeEventually(artifactRoot); + await removeTreeEventually(hostRoot); } } From 52d1bda4c129428872000e48827b98b7a4e017d1 Mon Sep 17 00:00:00 2001 From: Shaw Zheng Date: Fri, 11 Sep 2026 01:34:47 +0800 Subject: [PATCH 05/11] fix(testing): publish supervised launch specs safely --- .../bin/runtime/supervised-process.js | 79 ++++++++++++------- .../tests/node_runtime_test.js | 56 +++++++++++++ 2 files changed, 105 insertions(+), 30 deletions(-) diff --git a/packages/environment-factory/bin/runtime/supervised-process.js b/packages/environment-factory/bin/runtime/supervised-process.js index 14dc519c..797c8be4 100644 --- a/packages/environment-factory/bin/runtime/supervised-process.js +++ b/packages/environment-factory/bin/runtime/supervised-process.js @@ -22,6 +22,19 @@ const { const CLAIM_SCHEMA = 'fkst.supervised-process-startup.v1'; const SPEC_SCHEMA = 'fkst.supervised-process-launch.v1'; +const MAX_METADATA_BYTES = 2 * 1024 * 1024; + +function readDescriptorBounded(descriptor, label) { + const buffer = Buffer.allocUnsafe(MAX_METADATA_BYTES + 1); + let length = 0; + while (length < buffer.length) { + const count = fs.readSync(descriptor, buffer, length, buffer.length - length, null); + if (count === 0) break; + length += count; + } + if (length > MAX_METADATA_BYTES) throw new Error(`${label} exceeds its read bound`); + return buffer.subarray(0, length); +} function readIfExists(filePath) { try { @@ -33,10 +46,9 @@ function readIfExists(filePath) { } function readJsonNoFollow(filePath) { - const maxBytes = 2 * 1024 * 1024; for (let attempt = 0; attempt < 20; attempt += 1) { const before = fs.lstatSync(filePath); - if (!before.isFile() || before.isSymbolicLink() || before.size > maxBytes) { + if (!before.isFile() || before.isSymbolicLink() || before.size > MAX_METADATA_BYTES) { throw new Error('supervised startup claim is not a bounded regular file'); } const descriptor = fs.openSync(filePath, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)); @@ -46,14 +58,7 @@ function readJsonNoFollow(filePath) { sleep(2); continue; } - const buffer = Buffer.allocUnsafe(maxBytes + 1); - let length = 0; - while (length < buffer.length) { - const count = fs.readSync(descriptor, buffer, length, buffer.length - length, null); - if (count === 0) break; - length += count; - } - if (length > maxBytes) throw new Error('supervised startup claim exceeds its read bound'); + const body = readDescriptorBounded(descriptor, 'supervised startup claim'); const openedAfter = fs.fstatSync(descriptor); const after = fs.lstatSync(filePath); if (!sameFileIdentity(fileIdentity(openedBefore), fileIdentity(openedAfter)) @@ -61,7 +66,7 @@ function readJsonNoFollow(filePath) { sleep(2); continue; } - return JSON.parse(buffer.subarray(0, length).toString('utf8')); + return JSON.parse(body.toString('utf8')); } finally { fs.closeSync(descriptor); } @@ -148,7 +153,8 @@ function descriptorIdentities(descriptors) { function readBoundFile(filePath, expectedIdentity) { const before = fs.lstatSync(filePath); - if (!before.isFile() || before.isSymbolicLink()) { + if (!before.isFile() || before.isSymbolicLink() || before.size > MAX_METADATA_BYTES + || expectedIdentity.size > MAX_METADATA_BYTES) { throw new Error('supervised launch spec is not a regular file'); } const flags = fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0); @@ -159,7 +165,7 @@ function readBoundFile(filePath, expectedIdentity) { || !sameFileIdentity(fileIdentity(openedBefore), expectedIdentity)) { throw new Error('supervised launch spec identity changed'); } - const body = fs.readFileSync(descriptor); + const body = readDescriptorBounded(descriptor, 'supervised launch spec'); const openedAfter = fs.fstatSync(descriptor); const after = fs.lstatSync(filePath); if (!sameFileIdentity(fileIdentity(openedBefore), fileIdentity(openedAfter)) @@ -172,6 +178,33 @@ function readBoundFile(filePath, expectedIdentity) { } } +function publishLaunchSpecNoReplace(filePath, body) { + const pendingPath = `${filePath}.pending.${process.pid}.${crypto.randomBytes(16).toString('hex')}`; + let pendingCreated = false; + try { + fs.writeFileSync(pendingPath, body, { flag: 'wx', mode: 0o600 }); + pendingCreated = true; + try { + fs.linkSync(pendingPath, filePath); + } catch (error) { + if (error.code !== 'EEXIST') throw error; + } + } finally { + if (pendingCreated) { + try { fs.unlinkSync(pendingPath); } catch (_error) {} + } + } + const stat = fs.lstatSync(filePath); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_METADATA_BYTES) { + throw new Error('supervised launch spec is not a bounded regular file'); + } + const identity = fileIdentity(stat); + if (!readBoundFile(filePath, identity).equals(Buffer.from(body))) { + throw new Error('supervised launch spec immutable content differs'); + } + return identity; +} + function transitionClaim(claimPath, token, allowedStates, update) { const release = acquireLock(`${claimPath}.lock`); try { @@ -327,24 +360,10 @@ function startOrRecoverSupervisedProcess(options) { const specBody = `${stableStringify(spec)}\n`; let launched = false; try { - if (fs.existsSync(claim.launch_spec_path)) { - const existing = fs.lstatSync(claim.launch_spec_path); - if (!existing.isFile() || existing.isSymbolicLink()) { - throw new Error('supervised launch spec is not a regular file'); - } - if (fs.readFileSync(claim.launch_spec_path, 'utf8') !== specBody) { - fs.unlinkSync(claim.launch_spec_path); - } - } - if (!fs.existsSync(claim.launch_spec_path)) { - fs.writeFileSync(claim.launch_spec_path, specBody, { flag: 'wx', mode: 0o600 }); - } - fs.chmodSync(claim.launch_spec_path, 0o600); - const launchSpecStat = fs.lstatSync(claim.launch_spec_path); - if (!launchSpecStat.isFile() || launchSpecStat.isSymbolicLink()) { - throw new Error('supervised launch spec is not a regular file'); + if (typeof options.beforeLaunchSpecPublish === 'function') { + options.beforeLaunchSpecPublish(claim, specBody); } - const launchSpecIdentity = fileIdentity(launchSpecStat); + const launchSpecIdentity = publishLaunchSpecNoReplace(claim.launch_spec_path, specBody); claim = { ...claim, state: 'prepared', launch_spec_identity: launchSpecIdentity }; writeClaimAtomic(claimPath, claim); if (typeof options.beforeSupervisorLaunch === 'function') options.beforeSupervisorLaunch(claim); diff --git a/packages/environment-factory/tests/node_runtime_test.js b/packages/environment-factory/tests/node_runtime_test.js index 840586d4..3d7b3e4c 100644 --- a/packages/environment-factory/tests/node_runtime_test.js +++ b/packages/environment-factory/tests/node_runtime_test.js @@ -561,6 +561,62 @@ async function main() { assert.strictEqual(fs.existsSync(reservedHome), false); assert.strictEqual(releaseWorkerEnvironment(recoveredPreClaimEnvironment), true); + let noClobberLease = null; + let foreignLaunchSpecPath = null; + const foreignLaunchSpecBody = 'externally-created-launch-spec\n'; + assert.throws(() => startOrRecoverSupervisedProcess({ + claimPath: path.join(temp, 'supervised-launch-spec-no-clobber', 'claim.json'), + argv: [process.execPath, '-e', 'process.exit(0)'], + cwd: temp, + createEnvironment(reservation) { + const environment = minimalEnvironment( + {}, 'supervised-launch-spec-no-clobber', reservation.reservation_id, + ); + noClobberLease = workerEnvironmentLease(environment); + return environment; + }, + binding: { ...startupBinding, effect_id: 'launch-spec-no-clobber-effect' }, + beforeLaunchSpecPublish(claim) { + foreignLaunchSpecPath = claim.launch_spec_path; + fs.writeFileSync(foreignLaunchSpecPath, foreignLaunchSpecBody, { flag: 'wx' }); + }, + }), /immutable content differs/); + assert.strictEqual(fs.readFileSync(foreignLaunchSpecPath, 'utf8'), foreignLaunchSpecBody); + assert.strictEqual(fs.existsSync(noClobberLease.home), false); + + let oversizedLaunchLease = null; + const oversizedTargetMarker = path.join(temp, 'supervised-oversized-target.txt'); + const oversizedClaimPath = path.join(temp, 'supervised-oversized-launch', 'claim.json'); + const oversizedLaunch = startOrRecoverSupervisedProcess({ + claimPath: oversizedClaimPath, + argv: [process.execPath, '-e', `require('fs').writeFileSync(${JSON.stringify(oversizedTargetMarker)}, 'bad')`], + cwd: temp, + createEnvironment(reservation) { + const environment = minimalEnvironment( + {}, 'supervised-oversized-launch', reservation.reservation_id, + ); + oversizedLaunchLease = workerEnvironmentLease(environment); + return environment; + }, + binding: { ...startupBinding, effect_id: 'oversized-launch-effect' }, + registrationTimeoutMs: 250, + beforeSupervisorLaunch(claim) { + const oversizedBody = Buffer.alloc(2 * 1024 * 1024 + 1, 0x61); + fs.unlinkSync(claim.launch_spec_path); + fs.writeFileSync(claim.launch_spec_path, oversizedBody, { flag: 'wx', mode: 0o600 }); + const stat = fs.lstatSync(claim.launch_spec_path); + const substitutedClaim = JSON.parse(fs.readFileSync(oversizedClaimPath, 'utf8')); + substitutedClaim.launch_spec_identity = { + device: String(stat.dev), inode: String(stat.ino), size: stat.size, mode: stat.mode, + }; + substitutedClaim.launch_spec_sha256 = sha256(oversizedBody); + fs.writeFileSync(oversizedClaimPath, `${stableStringify(substitutedClaim)}\n`); + }, + }); + assert.strictEqual(oversizedLaunch.state, 'revoked'); + assert.strictEqual(fs.existsSync(oversizedTargetMarker), false); + assert.strictEqual(fs.existsSync(oversizedLaunchLease.home), false); + let substitutedLaunchLease = null; const substitutedMarker = path.join(temp, 'supervised-substituted-command.txt'); const substitutedLaunch = startOrRecoverSupervisedProcess({ From f5859e538d1ff4bdd5cefd6e634347f53b77183d Mon Sep 17 00:00:00 2001 From: Shaw Zheng Date: Fri, 11 Sep 2026 02:37:23 +0800 Subject: [PATCH 06/11] fix(testing): bind supervised launches to parent intent --- .../bin/runtime/supervised-process.js | 36 +++++++++++-- .../tests/node_runtime_test.js | 51 +++++++++++++++++++ 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/packages/environment-factory/bin/runtime/supervised-process.js b/packages/environment-factory/bin/runtime/supervised-process.js index 797c8be4..57faa537 100644 --- a/packages/environment-factory/bin/runtime/supervised-process.js +++ b/packages/environment-factory/bin/runtime/supervised-process.js @@ -231,6 +231,23 @@ function resourceFromClaim(binding, claim) { }; } +function launchInvariantSha256(claim) { + return sha256(stableStringify({ + startup_token: claim.startup_token, + binding_sha256: claim.binding_sha256, + launch_spec_path: claim.launch_spec_path, + launch_spec_sha256: claim.launch_spec_sha256, + launch_spec_identity: claim.launch_spec_identity, + argv_sha256: claim.argv_sha256, + cwd: claim.cwd, + cwd_identity: claim.cwd_identity, + inherited_fd_count: claim.inherited_fd_count, + inherited_fd_identities: claim.inherited_fd_identities, + worker_environment_reservation: claim.worker_environment_reservation, + worker_environment_lease_sha256: claim.worker_environment_lease_sha256, + })); +} + function startOrRecoverSupervisedProcess(options) { const claimPath = requireAbsoluteFile(options.claimPath, 'supervised startup claim'); const argv = validateArgv(options.argv); @@ -366,8 +383,11 @@ function startOrRecoverSupervisedProcess(options) { const launchSpecIdentity = publishLaunchSpecNoReplace(claim.launch_spec_path, specBody); claim = { ...claim, state: 'prepared', launch_spec_identity: launchSpecIdentity }; writeClaimAtomic(claimPath, claim); + const launchInvariant = launchInvariantSha256(claim); if (typeof options.beforeSupervisorLaunch === 'function') options.beforeSupervisorLaunch(claim); - const supervisor = spawn(process.execPath, [__filename, 'child', claim.launch_spec_path], { + const supervisor = spawn(process.execPath, [ + __filename, 'child', claim.launch_spec_path, claim.launch_spec_sha256, launchInvariant, + ], { cwd, env: environment, shell: false, @@ -451,13 +471,19 @@ function startOrRecoverSupervisedProcess(options) { }; } -function childMain(specPath) { +function childMain(specPath, expectedSpecSha256, expectedLaunchInvariantSha256) { + if (!/^[0-9a-f]{64}$/.test(String(expectedSpecSha256 || '')) + || !/^[0-9a-f]{64}$/.test(String(expectedLaunchInvariantSha256 || ''))) { + throw new Error('supervised launch trust binding is invalid'); + } const absoluteSpec = requireAbsoluteFile(specPath, 'supervised launch spec'); const claimPathGuess = absoluteSpec.replace(/\.launch-[0-9a-f]{32}\.json$/, ''); if (claimPathGuess === absoluteSpec) throw new Error('supervised launch spec path is invalid'); const initialClaim = readJsonNoFollow(claimPathGuess); if (!validClaim(initialClaim, initialClaim && initialClaim.binding_sha256) - || initialClaim.state !== 'prepared' || initialClaim.launch_spec_path !== absoluteSpec) { + || initialClaim.state !== 'prepared' || initialClaim.launch_spec_path !== absoluteSpec + || initialClaim.launch_spec_sha256 !== expectedSpecSha256 + || launchInvariantSha256(initialClaim) !== expectedLaunchInvariantSha256) { throw new Error('supervised startup claim is unavailable or revoked'); } const specBody = readBoundFile(absoluteSpec, initialClaim.launch_spec_identity); @@ -551,12 +577,12 @@ function childMain(specPath) { } if (require.main === module) { - if (process.argv.length !== 4 || process.argv[2] !== 'child') { + if (process.argv.length !== 6 || process.argv[2] !== 'child') { process.stderr.write('supervised-process: expected child launch spec\n'); process.exitCode = 2; } else { try { - childMain(process.argv[3]); + childMain(process.argv[3], process.argv[4], process.argv[5]); } catch (error) { process.stderr.write(`supervised-process: ${boundedText(error && error.message, 1024)}\n`); process.exitCode = 1; diff --git a/packages/environment-factory/tests/node_runtime_test.js b/packages/environment-factory/tests/node_runtime_test.js index 3d7b3e4c..bd2daac2 100644 --- a/packages/environment-factory/tests/node_runtime_test.js +++ b/packages/environment-factory/tests/node_runtime_test.js @@ -650,6 +650,57 @@ async function main() { assert.strictEqual(fs.existsSync(substitutedMarker), false); assert.strictEqual(fs.existsSync(substitutedLaunchLease.home), false); + let synchronizedSubstitutionLease = null; + const synchronizedSubstitutionMarker = path.join(temp, 'supervised-synchronized-substitution.txt'); + const synchronizedClaimPath = path.join(temp, 'supervised-synchronized-substitution', 'claim.json'); + const synchronizedSubstitution = startOrRecoverSupervisedProcess({ + claimPath: synchronizedClaimPath, + argv: [process.execPath, '-e', 'process.exit(0)'], + cwd: temp, + createEnvironment(reservation) { + const environment = minimalEnvironment( + {}, 'supervised-synchronized-substitution', reservation.reservation_id, + ); + synchronizedSubstitutionLease = workerEnvironmentLease(environment); + return environment; + }, + binding: { ...startupBinding, effect_id: 'synchronized-substitution-effect' }, + registrationTimeoutMs: 250, + beforeSupervisorLaunch(claim) { + const replacement = { + schema: 'fkst.supervised-process-launch.v1', + startup_token: claim.startup_token, + binding_sha256: claim.binding_sha256, + claim_path: synchronizedClaimPath, + argv: [process.execPath, '-e', + `require('fs').writeFileSync(${JSON.stringify(synchronizedSubstitutionMarker)}, 'bad')`], + cwd: temp, + inherited_fd_count: 0, + inherited_fd_identities: [], + }; + const replacementBody = `${stableStringify(replacement)}\n`; + fs.unlinkSync(claim.launch_spec_path); + fs.writeFileSync(claim.launch_spec_path, replacementBody, { flag: 'wx', mode: 0o600 }); + const stat = fs.lstatSync(claim.launch_spec_path); + const substitutedClaim = JSON.parse(fs.readFileSync(synchronizedClaimPath, 'utf8')); + substitutedClaim.launch_spec_identity = { + device: String(stat.dev), inode: String(stat.ino), size: stat.size, mode: stat.mode, + }; + substitutedClaim.launch_spec_sha256 = sha256(replacementBody); + substitutedClaim.argv_sha256 = sha256(stableStringify(replacement.argv)); + substitutedClaim.worker_environment_reservation.binding_sha256 = sha256(stableStringify({ + binding_sha256: substitutedClaim.binding_sha256, + argv_sha256: substitutedClaim.argv_sha256, + cwd: substitutedClaim.cwd, + inherited_fd_identities: substitutedClaim.inherited_fd_identities, + })); + fs.writeFileSync(synchronizedClaimPath, `${stableStringify(substitutedClaim)}\n`); + }, + }); + assert.strictEqual(synchronizedSubstitution.state, 'revoked'); + assert.strictEqual(fs.existsSync(synchronizedSubstitutionMarker), false); + assert.strictEqual(fs.existsSync(synchronizedSubstitutionLease.home), false); + let symlinkedLaunchLease = null; const symlinkedMarker = path.join(temp, 'supervised-symlinked-command.txt'); const symlinkedLaunch = startOrRecoverSupervisedProcess({ From 94fce9a1f876ab17182e87c9b3c0db09dec32eeb Mon Sep 17 00:00:00 2001 From: Shaw Zheng Date: Fri, 11 Sep 2026 04:41:14 +0800 Subject: [PATCH 07/11] fix(testing): authorize every target effect at runtime --- .../generic-host/bin/generic-host-runtime.js | 135 +++++++++++--- .../host_canonical_workflow_qa.lua | 53 +++++- .../generic-host/host_durable_workflow_qa.lua | 64 +++++-- ...ownstream_local_qa_acceptance_e2e_test.lua | 22 ++- .../tests/durable_host_store_test.lua | 14 +- .../durable_workflow_qa_recovery_test.lua | 16 +- libraries/contract/structured_execution.lua | 60 ++++++- .../bin/fkst-structured-execution-runtime.js | 168 +++++++++++++----- .../testing_runtime/structured_execution.lua | 7 +- .../structured_execution_runtime_test.js | 91 ++++++++-- .../testing-runner/structured_execution.lua | 58 +++++- ...structured_authorization_contract_test.lua | 40 +++++ .../structured_execution_department_test.lua | 1 + .../tests/structured_execution_edge_test.lua | 5 +- ...uctured_execution_runtime_adapter_test.lua | 6 +- .../tests/structured_execution_test.lua | 9 +- 16 files changed, 611 insertions(+), 138 deletions(-) diff --git a/examples/generic-host/bin/generic-host-runtime.js b/examples/generic-host/bin/generic-host-runtime.js index 31e23e02..96f1300b 100755 --- a/examples/generic-host/bin/generic-host-runtime.js +++ b/examples/generic-host/bin/generic-host-runtime.js @@ -1227,11 +1227,11 @@ const { structuredExecutionArtifacts } = require('./structured-execution-artifac }); function structuredAuthorizationKey(receiptId) { - return `testing-runner/cli-effect-authorizations/${sha256(stable(receiptId))}`; + return `testing-runner/effect-authorizations/${sha256(stable(receiptId))}`; } function structuredConsumptionKey(receiptId) { - return `testing-runner/cli-effect-consumptions/${sha256(stable(receiptId))}`; + return `testing-runner/effect-consumptions/${sha256(stable(receiptId))}`; } function localHttpRequest(request, timeoutSeconds) { @@ -1540,8 +1540,8 @@ function validWindow(value, now) { && Number.isFinite(current) && current >= issued && current < expires; } -function validCliEnvelope(envelope) { - const fields = [ +function validActionEnvelope(envelope, expectedKind) { + const commonFields = [ 'schema', 'effect_kind', 'capability', 'profile_ref', 'profile_artifact_sha256', 'profile_sha256', 'validation_receipt_ref', 'validation_receipt_sha256', 'preauthorization_ref', 'preauthorization_sha256', 'repository', 'run_id', @@ -1549,6 +1549,8 @@ function validCliEnvelope(envelope) { 'workspace_ref', 'plan_ref', 'plan_sha256', 'grant_ref', 'grant_sha256', 'case', 'resource_bounds', 'attempt', 'trace_id', 'dedup_key', 'expires_at', 'fence_id', ]; + const kind = envelope && envelope.effect_kind; + const fields = kind === 'http' ? [...commonFields, 'base_url'] : commonFields; const digestFields = [ 'profile_artifact_sha256', 'profile_sha256', 'validation_receipt_sha256', 'preauthorization_sha256', 'environment_receipt_sha256', 'plan_sha256', 'grant_sha256', @@ -1560,8 +1562,11 @@ function validCliEnvelope(envelope) { const action = envelope && envelope.case; const assertions = action && action.assertions; return exactKeys(envelope, fields) - && envelope.schema === 'testing-cli-action-envelope.v1' - && envelope.effect_kind === 'cli' && envelope.capability === 'direct-argv' + && (expectedKind === undefined || kind === expectedKind) + && (kind === 'cli' || kind === 'http') + && envelope.schema === (kind === 'cli' + ? 'testing-cli-action-envelope.v1' : 'testing-http-action-envelope.v1') + && envelope.capability === (kind === 'cli' ? 'direct-argv' : 'loopback-http') && envelope.run_id === envelope.operation_id && envelope.attempt === 1 && boundedString(envelope.run_id, 180) && /^[A-Za-z0-9._-]+$/.test(envelope.run_id) && boundedString(envelope.trace_id, 180) && boundedString(envelope.dedup_key, 180) @@ -1574,15 +1579,37 @@ function validCliEnvelope(envelope) { && Number.isInteger(envelope.resource_bounds.output_bytes) && envelope.resource_bounds.output_bytes >= 1024 && envelope.resource_bounds.output_bytes <= 1024 * 1024 - && exactKeys(action, ['case_id', 'kind', 'argv', 'timeout_seconds', 'assertions']) && boundedString(action.case_id, 180) && /^[A-Za-z0-9._-]+$/.test(action.case_id) - && action.kind === 'cli' && validArgv(action.argv) + && action.kind === kind && Number.isInteger(action.timeout_seconds) && action.timeout_seconds >= 1 && action.timeout_seconds <= 300 && Array.isArray(assertions) && assertions.length > 0 && assertions.length <= 16 - && assertions.every((assertion) => exactKeys(assertion, ['type', 'expected']) - && assertion.type === 'exit-code' && Number.isInteger(assertion.expected) - && assertion.expected >= 0 && assertion.expected <= 255); + && (kind === 'cli' + ? exactKeys(action, ['case_id', 'kind', 'argv', 'timeout_seconds', 'assertions']) + && validArgv(action.argv) + && assertions.every((assertion) => exactKeys(assertion, ['type', 'expected']) + && assertion.type === 'exit-code' && Number.isInteger(assertion.expected) + && assertion.expected >= 0 && assertion.expected <= 255) + : exactKeys(action, ['case_id', 'kind', 'request', 'timeout_seconds', 'assertions']) + && exactKeys(action.request, ['method', 'url', 'headers']) + && httpMethods.has(action.request.method) && Array.isArray(action.request.headers) + && action.request.headers.length === 0 && splitHttpUrl(envelope.base_url) !== null + && splitHttpUrl(action.request.url) !== null + && splitHttpUrl(action.request.url).origin === splitHttpUrl(envelope.base_url).origin + && assertions.every((assertion) => { + if (assertion && assertion.type === 'status-code') { + return exactKeys(assertion, ['type', 'expected']) && Number.isInteger(assertion.expected) + && assertion.expected >= 100 && assertion.expected <= 599; + } + if (assertion && assertion.type === 'body-contains') { + return exactKeys(assertion, ['type', 'expected']) && boundedString(assertion.expected, 512); + } + return assertion && assertion.type === 'json-path-equals' + && exactKeys(assertion, ['type', 'path', 'expected']) + && boundedString(assertion.path, 512) + && assertion.path.split('.').every((part) => /^[A-Za-z_][A-Za-z0-9_-]*$/.test(part)) + && ['string', 'number', 'boolean'].includes(typeof assertion.expected); + })); } function activeStructuredRequest(root, runId) { @@ -1628,7 +1655,7 @@ function structuredAuthorizationReceipt(runId, envelope, decision, reasonCode, i ? envelope.expires_at : '2026-07-22T00:21:00Z'; return { schema: 'testing-effect-authorization-receipt.v1', decision, reason_code: reasonCode, - receipt_id: `durable-cli-effect-${envelopeSha256.slice(0, 32)}`, + receipt_id: `durable-${envelope.effect_kind || 'invalid'}-effect-${envelopeSha256.slice(0, 32)}`, envelope_sha256: envelopeSha256, evaluated_input_digests: inputs, issued_at: issuedAt, expires_at: expiresAt, fence_id: typeof envelope.fence_id === 'string' ? envelope.fence_id : 'invalid-fence', @@ -1638,7 +1665,7 @@ function structuredAuthorizationReceipt(runId, envelope, decision, reasonCode, i }; } -function authorizeCliEffect(projectRoot, payload) { +function authorizeEffect(projectRoot, payload, expectedKind) { const runId = runIdFor(payload); const root = runRoot(runId); const config = loadConfig(projectRoot, runId); @@ -1651,7 +1678,7 @@ function authorizeCliEffect(projectRoot, payload) { const deny = (reason, inputs = empty) => structuredAuthorizationReceipt(runId, envelope, 'deny', reason, inputs); try { - if (!validCliEnvelope(envelope)) return deny('malformed-envelope'); + if (!validActionEnvelope(envelope, expectedKind)) return deny('malformed-envelope'); const request = activeStructuredRequest(root, runId); if (!envelopeMatchesRequest(envelope, request, payload)) return deny('foreign-binding'); const profile = artifactRead(projectRoot, envelope.profile_ref, envelope.profile_artifact_sha256); @@ -1696,10 +1723,12 @@ function authorizeCliEffect(projectRoot, payload) { if (!schemasValid) return deny('malformed-input', inputs); const preauthorizationValid = preauthorization.value.max_uses === 1 && validWindow(preauthorization.value, now) - && validCliCapabilities(preauthorization.value.capabilities && preauthorization.value.capabilities.cli); + && validCliCapabilities(preauthorization.value.capabilities && preauthorization.value.capabilities.cli) + && validHttpCapabilities(preauthorization.value.capabilities && preauthorization.value.capabilities.http); if (!preauthorizationValid) return deny('stale-preauthorization', inputs); const grantValid = grant.value.max_uses === 1 && validWindow(grant.value, now) - && validCliCapabilities(grant.value.cli_capabilities); + && validCliCapabilities(grant.value.cli_capabilities) + && validHttpCapabilities(grant.value.http_capabilities); if (!grantValid) return deny('stale-grant', inputs); if (!sameRun(validation.value) || !sameRun(preauthorization.value) || !sameRun(environment.value) || !sameRun(plan.value) || !sameRun(grant.value)) return deny('foreign-binding', inputs); @@ -1735,9 +1764,13 @@ function authorizeCliEffect(projectRoot, payload) { && grant.value.policy_revision === preauthorization.value.policy_revision && samePointer(grant.value.evidence_ref, expectedEvidence); if (!authenticated) return deny('foreign-binding', inputs); - if (stable(planned) !== stable(envelope.case) - || !argvAllowed(envelope.case.argv, preauthorization.value.capabilities.cli) - || !argvAllowed(envelope.case.argv, grant.value.cli_capabilities)) { + const effectAllowed = envelope.effect_kind === 'cli' + ? argvAllowed(envelope.case.argv, preauthorization.value.capabilities.cli) + && argvAllowed(envelope.case.argv, grant.value.cli_capabilities) + : envelope.base_url === environment.value.base_url + && httpAllowed(envelope.case.request, preauthorization.value.capabilities.http) + && httpAllowed(envelope.case.request, grant.value.http_capabilities); + if (stable(planned) !== stable(envelope.case) || !effectAllowed) { return deny('scope-denied', inputs); } const replayOwned = replay && replay.status === 'claimed' && replay.fence_id === envelope.fence_id @@ -1751,7 +1784,7 @@ function authorizeCliEffect(projectRoot, payload) { && replayBinding.trace_id === envelope.trace_id && replayBinding.dedup_key === envelope.dedup_key; if (!replayOwned) return deny('foreign-fence', inputs); const fixturePolicy = config.runtime_pep_denial; - if (fixturePolicy !== undefined) { + if (fixturePolicy !== undefined && envelope.effect_kind === 'cli') { const token = process.env.FKST_GENERIC_HOST_FIXTURE_CLI_DENY_TOKEN; const fixturePolicyValid = exactKeys(fixturePolicy, ['reason_code', 'token']) && fixturePolicy.reason_code === 'profile-policy-denied' @@ -1762,7 +1795,7 @@ function authorizeCliEffect(projectRoot, payload) { const receipt = structuredAuthorizationReceipt(runId, envelope, 'allow', 'authorized', inputs); const authorization = { receipt, grant_id: grant.value.grant_id, fence_id: envelope.fence_id }; const stored = recordImmutable(root, structuredAuthorizationKey(receipt.receipt_id), authorization); - if (!stored.written && !stored.replayed) fail('durable CLI authorization receipt conflict'); + if (!stored.written && !stored.replayed) fail('durable effect authorization receipt conflict'); return receipt; } catch (_error) { return deny('malformed-input'); @@ -2173,7 +2206,9 @@ function dispatch(name, payload, projectRoot) { return { status: 'claimed', claim_id: fenceId }; } case 'authorize-cli-effect': - return authorizeCliEffect(projectRoot, payload); + return authorizeEffect(projectRoot, payload, 'cli'); + case 'authorize-http-effect': + return authorizeEffect(projectRoot, payload, 'http'); case 'exec-argv': { const runId = runIdFor(payload); const root = runRoot(runId); @@ -2181,7 +2216,7 @@ function dispatch(name, payload, projectRoot) { const envelope = payload.action_envelope; const receipt = payload.authorization_receipt; const request = activeStructuredRequest(root, runId); - const validReceipt = validCliEnvelope(envelope) && envelopeMatchesRequest(envelope, request, payload) + const validReceipt = validActionEnvelope(envelope, 'cli') && envelopeMatchesRequest(envelope, request, payload) && exactKeys(receipt, [ 'schema', 'decision', 'reason_code', 'receipt_id', 'envelope_sha256', 'evaluated_input_digests', 'issued_at', 'expires_at', 'fence_id', 'trace_id', @@ -2240,12 +2275,56 @@ function dispatch(name, payload, projectRoot) { case 'http-request': { const runId = runIdFor(payload); const config = loadConfig(projectRoot, runId); - if (payload.operation_id !== runId || payload.base_url !== config.base_url - || !payload.request || payload.request.method !== 'GET' - || payload.request.url !== config.base_url) fail('structured HTTP request binding differs'); - const result = localHttpRequest(payload.request, payload.timeout_seconds); const request = activeStructuredRequest(runRoot(runId), runId); - const sequence = structuredCaseSequence(projectRoot, request, payload.case_id); + const envelope = payload.action_envelope; + const receipt = payload.authorization_receipt; + const validReceipt = validActionEnvelope(envelope, 'http') + && envelopeMatchesRequest(envelope, request, payload) + && exactKeys(receipt, [ + 'schema', 'decision', 'reason_code', 'receipt_id', 'envelope_sha256', + 'evaluated_input_digests', 'issued_at', 'expires_at', 'fence_id', 'trace_id', + 'dedup_key', 'auth_tag', + ]) + && exactKeys(receipt.evaluated_input_digests, [ + 'profile', 'validation_receipt', 'preauthorization', 'environment_receipt', 'plan', 'grant', + ]) + && Object.values(receipt.evaluated_input_digests).every(validDigest) + && receipt.schema === 'testing-effect-authorization-receipt.v1' + && receipt.decision === 'allow' && receipt.reason_code === 'authorized' + && receipt.envelope_sha256 === sha256(stable(envelope)) + && receipt.auth_tag === sha256(`${runId}\0${receipt.envelope_sha256}\0allow`) + && receipt.fence_id === envelope.fence_id && receipt.trace_id === envelope.trace_id + && receipt.dedup_key === envelope.dedup_key && receipt.expires_at === envelope.expires_at + && receipt.issued_at === '2026-07-22T00:20:00Z' + && Date.parse(receipt.expires_at) > Date.parse('2026-07-22T00:20:00Z'); + const authorization = validReceipt + ? recordRead(runRoot(runId), structuredAuthorizationKey(receipt.receipt_id)) : null; + const grant = validReceipt + ? artifactRead(projectRoot, envelope.grant_ref, envelope.grant_sha256) : null; + const replay = grant && grant.value + ? recordRead(runRoot(runId), structuredReplayKey(grant.value.grant_id)) : null; + if (!validReceipt || !authorization || !grant || !grant.value || typeof grant.value !== 'object' + || stable(authorization.receipt) !== stable(receipt) + || authorization.grant_id !== grant.value.grant_id || authorization.fence_id !== envelope.fence_id + || grant.digest !== envelope.grant_sha256 || !replay || replay.status !== 'claimed' + || replay.fence_id !== envelope.fence_id) { + fail('durable structured HTTP authorization receipt is unavailable'); + } + if (envelope.operation_id !== runId || envelope.base_url !== config.base_url + || envelope.case.request.url !== config.base_url) fail('structured HTTP request binding differs'); + const consumed = recordClaim(runRoot(runId), structuredConsumptionKey(receipt.receipt_id), { + binding: receipt, receipt_id: receipt.receipt_id, grant_id: grant.value.grant_id, + }); + if (!consumed.claimed || consumed.replayed) fail('durable structured HTTP authorization receipt is replayed'); + artifactWrite(projectRoot, `${payload.artifact_root}/authorization/${envelope.case.case_id}-consumption.json`, { + schema: 'generic-host.http-effect-consumption.v1', case_id: envelope.case.case_id, + receipt_id: receipt.receipt_id, grant_id: grant.value.grant_id, + consumption_fingerprint_sha256: claimFingerprint( + config, 'structured-execution-consumption', envelope.fence_id, + ), + }); + const result = localHttpRequest(envelope.case.request, envelope.case.timeout_seconds); + const sequence = structuredCaseSequence(projectRoot, request, envelope.case.case_id); const stored = recordImmutable(runRoot(runId), `testing-runner/target-effects/${sha256(stable(payload))}`, { sequence, binding: payload, result, }); diff --git a/examples/generic-host/host_canonical_workflow_qa.lua b/examples/generic-host/host_canonical_workflow_qa.lua index 44c15daf..88c58c09 100644 --- a/examples/generic-host/host_canonical_workflow_qa.lua +++ b/examples/generic-host/host_canonical_workflow_qa.lua @@ -479,13 +479,32 @@ function Context:_structured_runtime() end return false end + local function http_allowed(request, capabilities, base_url) + local base_origin = execution.local_http_origin(base_url) + local origin, request_path = execution.local_http_origin(request and request.url) + if base_origin == nil or origin ~= base_origin then return false end + for _, capability in ipairs(capabilities or {}) do + local capability_origin = execution.local_http_origin(capability.origin) + local method_allowed = false + for _, method in ipairs(capability.methods or {}) do + if method == request.method then method_allowed = true end + end + if capability_origin == base_origin and method_allowed then + for _, prefix in ipairs(capability.path_prefixes or {}) do + if request_path:sub(1, #prefix) == prefix then return true end + end + end + end + return false + end local function decision(envelope, value, reason, inputs) local envelope_sha256 = sha256_bytes(json_codec.encode(envelope)) local receipt = { schema = execution.schemas.effect_authorization_receipt, decision = value, reason_code = reason, - receipt_id = "canonical-cli-effect-" .. envelope_sha256:sub(1, 32), + receipt_id = "canonical-" .. tostring(envelope.effect_kind or "invalid") + .. "-effect-" .. envelope_sha256:sub(1, 32), envelope_sha256 = envelope_sha256, evaluated_input_digests = inputs, issued_at = "2026-07-22T00:20:00Z", @@ -498,7 +517,7 @@ function Context:_structured_runtime() if value == "allow" then authorizations[receipt.receipt_id] = copy(receipt) end return receipt end - return { + local runtime = { sha256_bytes = function(bytes) return sha256_bytes(bytes) end, load_artifact = function(path) return context.store:load(path) end, now = function(request) @@ -541,7 +560,7 @@ function Context:_structured_runtime() end, authorize_cli_effect = function(request) local envelope = request.action_envelope - local ok = pcall(execution.validate_cli_action_envelope, envelope) + local ok = pcall(execution.validate_action_envelope, envelope) local empty = { profile = string.rep("0", 64), validation_receipt = string.rep("0", 64), preauthorization = string.rep("0", 64), environment_receipt = string.rep("0", 64), @@ -581,6 +600,12 @@ function Context:_structured_runtime() if item.case_id == envelope.case.case_id then planned_case = item end end local replay = claims[grant.value.grant_id] + local effect_allowed = envelope.effect_kind == "cli" + and argv_allowed(envelope.case.argv, preauthorization.value.capabilities.cli) + and argv_allowed(envelope.case.argv, grant.value.cli_capabilities) + or envelope.effect_kind == "http" and envelope.base_url == environment.value.base_url + and http_allowed(envelope.case.request, preauthorization.value.capabilities.http, envelope.base_url) + and http_allowed(envelope.case.request, grant.value.http_capabilities, envelope.base_url) if not valid or profile.digest ~= envelope.profile_artifact_sha256 or project_profile.profile_sha256(profile.value, sha256_bytes) ~= envelope.profile_sha256 or validation.digest ~= envelope.validation_receipt_sha256 @@ -595,8 +620,7 @@ function Context:_structured_runtime() or grant.value.environment_receipt_sha256 ~= environment.digest or not equal(environment.value.workspace_ref, envelope.workspace_ref) or not equal(planned_case, envelope.case) - or not argv_allowed(envelope.case.argv, preauthorization.value.capabilities.cli) - or not argv_allowed(envelope.case.argv, grant.value.cli_capabilities) + or not effect_allowed or type(replay) ~= "table" or replay.fence_id ~= envelope.fence_id then return decision(envelope, "deny", "foreign-binding", inputs) end @@ -621,14 +645,23 @@ function Context:_structured_runtime() return direct_exec(envelope.case.argv, context.workspace_root) end, http_request = function(input) - if input.operation_id ~= context.run_id or input.base_url ~= context.base_url - or input.request.url ~= context.base_url then + local envelope = input.action_envelope + local receipt = input.authorization_receipt + execution.validate_http_action_envelope(envelope) + execution.validate_effect_authorization_receipt(receipt, envelope, "2026-07-22T00:20:00Z") + local issued = authorizations[receipt.receipt_id] + if receipt.decision ~= "allow" or issued == nil or not equal(issued, receipt) then + error("canonical structured HTTP authorization receipt is unavailable or replayed") + end + authorizations[receipt.receipt_id] = nil + if envelope.operation_id ~= context.run_id or envelope.base_url ~= context.base_url + or envelope.case.request.url ~= context.base_url then error("canonical structured HTTP request is not bound to the ready environment") end table.insert(context.target_effects, { - kind = "http", method = input.request.method, url = input.request.url, + kind = "http", method = envelope.case.request.method, url = envelope.case.request.url, }) - return http_request(input.request, input.timeout_seconds) + return http_request(envelope.case.request, envelope.case.timeout_seconds) end, write_artifact = function(path, value) return context.store:write(path, value) end, load_result = function(request) @@ -689,6 +722,8 @@ function Context:_structured_runtime() return false end, } + runtime.authorize_http_effect = runtime.authorize_cli_effect + return runtime end function Context:_ai_browser_runtime() diff --git a/examples/generic-host/host_durable_workflow_qa.lua b/examples/generic-host/host_durable_workflow_qa.lua index 926bdd67..e43bbdfb 100644 --- a/examples/generic-host/host_durable_workflow_qa.lua +++ b/examples/generic-host/host_durable_workflow_qa.lua @@ -1134,7 +1134,7 @@ function Context:_structured_runtime() local context = self local function replay_key(grant_id) return "testing-runner/replay/" .. context:_key(grant_id) end local function authorization_key(receipt_id) - return "testing-runner/cli-effect-authorizations/" .. context:_key(receipt_id) + return "testing-runner/effect-authorizations/" .. context:_key(receipt_id) end local function argv_allowed(argv, capabilities) for _, capability in ipairs(capabilities or {}) do @@ -1147,13 +1147,32 @@ function Context:_structured_runtime() end return false end + local function http_allowed(request, capabilities, base_url) + local base_origin = execution.local_http_origin(base_url) + local origin, request_path = execution.local_http_origin(request and request.url) + if base_origin == nil or origin ~= base_origin then return false end + for _, capability in ipairs(capabilities or {}) do + local capability_origin = execution.local_http_origin(capability.origin) + local method_allowed = false + for _, method in ipairs(capability.methods or {}) do + if method == request.method then method_allowed = true end + end + if capability_origin == base_origin and method_allowed then + for _, prefix in ipairs(capability.path_prefixes or {}) do + if request_path:sub(1, #prefix) == prefix then return true end + end + end + end + return false + end local function decision(envelope, value, reason, inputs) local envelope_sha256 = context.records:digest(json_codec.encode(envelope)) local receipt = { schema = execution.schemas.effect_authorization_receipt, decision = value, reason_code = reason, - receipt_id = "durable-cli-effect-" .. envelope_sha256:sub(1, 32), + receipt_id = "durable-" .. tostring(envelope.effect_kind or "invalid") + .. "-effect-" .. envelope_sha256:sub(1, 32), envelope_sha256 = envelope_sha256, evaluated_input_digests = inputs, issued_at = "2026-07-22T00:20:00Z", @@ -1166,12 +1185,12 @@ function Context:_structured_runtime() if value == "allow" then local stored = context.records:immutable(authorization_key(receipt.receipt_id), copy(receipt)) if stored.written ~= true and stored.replayed ~= true then - error("generic-host durable CLI authorization receipt conflict") + error("generic-host durable effect authorization receipt conflict") end end return receipt end - return { + local runtime = { sha256_bytes = function(bytes) return context.records:digest(bytes) end, load_artifact = function(path) return context.store:load(path) end, now = function(request) @@ -1208,7 +1227,7 @@ function Context:_structured_runtime() end, authorize_cli_effect = function(request) local envelope = request.action_envelope - local ok = pcall(execution.validate_cli_action_envelope, envelope) + local ok = pcall(execution.validate_action_envelope, envelope) local empty = { profile = string.rep("0", 64), validation_receipt = string.rep("0", 64), preauthorization = string.rep("0", 64), environment_receipt = string.rep("0", 64), @@ -1245,6 +1264,12 @@ function Context:_structured_runtime() for _, item in ipairs(plan.value.cases or {}) do if item.case_id == envelope.case.case_id then planned_case = item end end + local effect_allowed = envelope.effect_kind == "cli" + and argv_allowed(envelope.case.argv, preauthorization.value.capabilities.cli) + and argv_allowed(envelope.case.argv, grant.value.cli_capabilities) + or envelope.effect_kind == "http" and envelope.base_url == environment.value.base_url + and http_allowed(envelope.case.request, preauthorization.value.capabilities.http, envelope.base_url) + and http_allowed(envelope.case.request, grant.value.http_capabilities, envelope.base_url) if not valid or profile.digest ~= envelope.profile_artifact_sha256 or project_profile.profile_sha256(profile.value, function(body) return context.records:digest(body) end) ~= envelope.profile_sha256 @@ -1262,8 +1287,7 @@ function Context:_structured_runtime() or type(replay) ~= "table" or replay.status ~= "claimed" or replay.fence_id ~= envelope.fence_id or not equal(planned_case, envelope.case) - or not argv_allowed(envelope.case.argv, preauthorization.value.capabilities.cli) - or not argv_allowed(envelope.case.argv, grant.value.cli_capabilities) then + or not effect_allowed then return decision(envelope, "deny", "foreign-binding", inputs) end return decision(envelope, "allow", "authorized", inputs) @@ -1280,7 +1304,7 @@ function Context:_structured_runtime() error("generic-host durable structured CLI authorization receipt is unavailable") end local consumed = context.records:claim( - "testing-runner/cli-effect-consumptions/" .. context:_key(receipt.receipt_id), + "testing-runner/effect-consumptions/" .. context:_key(receipt.receipt_id), { binding = copy(receipt), receipt_id = receipt.receipt_id }) if consumed.claimed ~= true or consumed.replayed == true then error("generic-host durable structured CLI authorization receipt is replayed") @@ -1300,11 +1324,27 @@ function Context:_structured_runtime() return result end, http_request = function(input) - if input.operation_id ~= context.run_id or input.base_url ~= context.base_url - or input.request.url ~= context.base_url then + local envelope = input.action_envelope + local receipt = input.authorization_receipt + execution.validate_http_action_envelope(envelope) + execution.validate_effect_authorization_receipt(receipt, envelope, "2026-07-22T00:20:00Z") + local issued = context.records:read(authorization_key(receipt.receipt_id)) + if receipt.decision ~= "allow" + or receipt.envelope_sha256 ~= context.records:digest(json_codec.encode(envelope)) + or issued == nil or not equal(issued, receipt) then + error("generic-host durable structured HTTP authorization receipt is unavailable") + end + local consumed = context.records:claim( + "testing-runner/effect-consumptions/" .. context:_key(receipt.receipt_id), + { binding = copy(receipt), receipt_id = receipt.receipt_id }) + if consumed.claimed ~= true or consumed.replayed == true then + error("generic-host durable structured HTTP authorization receipt is replayed") + end + if envelope.operation_id ~= context.run_id or envelope.base_url ~= context.base_url + or envelope.case.request.url ~= context.base_url then error("generic-host durable structured HTTP request is not bound to the ready environment") end - local result = http_request(input.request, input.timeout_seconds) + local result = http_request(envelope.case.request, envelope.case.timeout_seconds) context.records:immutable("testing-runner/target-effects/" .. context:_key(input), { binding = copy(input), result = copy(result), }) @@ -1382,6 +1422,8 @@ function Context:_structured_runtime() return true end, } + runtime.authorize_http_effect = runtime.authorize_cli_effect + return runtime end function Context:_publication_runtime() diff --git a/examples/generic-host/tests/downstream_local_qa_acceptance_e2e_test.lua b/examples/generic-host/tests/downstream_local_qa_acceptance_e2e_test.lua index 941cf80d..86e23512 100644 --- a/examples/generic-host/tests/downstream_local_qa_acceptance_e2e_test.lua +++ b/examples/generic-host/tests/downstream_local_qa_acceptance_e2e_test.lua @@ -56,8 +56,8 @@ local function counts(context) profile = #context.records:list("generic-host/profile-approval"), preauthorization = #context.records:list("generic-host/preauthorization"), replay = #context.records:list("testing-runner/replay"), - authorization = #context.records:list("testing-runner/cli-effect-authorizations"), - consumption = #context.records:list("testing-runner/cli-effect-consumptions"), + authorization = #context.records:list("testing-runner/effect-authorizations"), + consumption = #context.records:list("testing-runner/effect-consumptions"), effects = #context.records:list("testing-runner/target-effects"), publication = #context.records:list("test-publication/effects"), terminal = #context.records:list("generic-host/terminal"), @@ -138,8 +138,8 @@ return { t.eq(effects[4].value.result.stdout, "") t.eq(effects[4].value.result.stderr, REJECTED) t.eq(effects[5].value.result.body, RESERVED) - t.eq(#at_barrier.records:list("testing-runner/cli-effect-authorizations"), 2) - t.eq(#at_barrier.records:list("testing-runner/cli-effect-consumptions"), 2) + t.eq(#at_barrier.records:list("testing-runner/effect-authorizations"), 5) + t.eq(#at_barrier.records:list("testing-runner/effect-consumptions"), 5) for _, expected in ipairs({ "inventory-reserve-three", "inventory-over-reserve-rejected" }) do local consumption = artifact(at_barrier, at_barrier.request.structured_execution.artifact_root .. "/authorization/" .. expected .. "-consumption.json") @@ -149,6 +149,16 @@ return { t.is_true(type(consumption.consumption_fingerprint_sha256) == "string") t.eq(#consumption.consumption_fingerprint_sha256, 64) end + for _, expected in ipairs({ + "inventory-initial-state", "inventory-state-after-reserve", "inventory-state-after-rejection", + }) do + local consumption = artifact(at_barrier, at_barrier.request.structured_execution.artifact_root + .. "/authorization/" .. expected .. "-consumption.json") + t.eq(consumption.schema, "generic-host.http-effect-consumption.v1") + t.eq(consumption.case_id, expected) + t.is_true(type(consumption.consumption_fingerprint_sha256) == "string") + t.eq(#consumption.consumption_fingerprint_sha256, 64) + end t.eq(at_barrier:terminal_record(), nil) local ownership = at_barrier:_fixture_effect("fixture-resource-status", { run_id = context.run_id, artifact_root = context.artifact_root, @@ -257,8 +267,8 @@ return { t.eq(before.profile, 1) t.eq(before.preauthorization, 1) t.eq(before.replay, 1) - t.eq(before.authorization, 2) - t.eq(before.consumption, 2) + t.eq(before.authorization, 5) + t.eq(before.consumption, 5) t.eq(before.effects, 5) t.eq(before.publication, 16) local published = durable.load(context.project_root, context.durable_root, context.run_id) diff --git a/examples/generic-host/tests/durable_host_store_test.lua b/examples/generic-host/tests/durable_host_store_test.lua index 1e853f0d..f1a55f1d 100644 --- a/examples/generic-host/tests/durable_host_store_test.lua +++ b/examples/generic-host/tests/durable_host_store_test.lua @@ -279,8 +279,8 @@ return { local function assert_gateway_unchanged() local current = durable.load(context.project_root, context.durable_root, context.run_id) - t.eq(#current.records:list("testing-runner/cli-effect-authorizations"), 1) - t.eq(#current.records:list("testing-runner/cli-effect-consumptions"), 0) + t.eq(#current.records:list("testing-runner/effect-authorizations"), 1) + t.eq(#current.records:list("testing-runner/effect-consumptions"), 0) t.eq(#current.records:list("testing-runner/target-effects"), 0) t.eq(process.effect_count(context), 0) end @@ -364,7 +364,7 @@ return { end) local consumed = durable.load(context.project_root, context.durable_root, context.run_id) t.eq(process.effect_count(context), 1) - t.eq(#consumed.records:list("testing-runner/cli-effect-consumptions"), 1) + t.eq(#consumed.records:list("testing-runner/effect-consumptions"), 1) t.eq(#consumed.records:list("testing-runner/target-effects"), 1) return result end @@ -381,8 +381,8 @@ return { t.eq(outcome.replayed, false) t.is_true(type(authorized_request) == "table") local recovered = durable.load(context.project_root, context.durable_root, context.run_id) - local authorizations = recovered.records:list("testing-runner/cli-effect-authorizations") - local consumptions = recovered.records:list("testing-runner/cli-effect-consumptions") + local authorizations = recovered.records:list("testing-runner/effect-authorizations") + local consumptions = recovered.records:list("testing-runner/effect-consumptions") local replays = recovered.records:list("testing-runner/replay") local effects = recovered.records:list("testing-runner/target-effects") t.eq(#authorizations, 1) @@ -425,8 +425,8 @@ return { t.eq(replayed.status, "passed") t.eq(replayed.replayed, true) recovered = durable.load(context.project_root, context.durable_root, context.run_id) - t.eq(#recovered.records:list("testing-runner/cli-effect-authorizations"), 1) - t.eq(#recovered.records:list("testing-runner/cli-effect-consumptions"), 1) + t.eq(#recovered.records:list("testing-runner/effect-authorizations"), 1) + t.eq(#recovered.records:list("testing-runner/effect-consumptions"), 1) t.eq(#recovered.records:list("testing-runner/target-effects"), 1) t.eq(process.effect_count(context), 1) end) diff --git a/examples/generic-host/tests/durable_workflow_qa_recovery_test.lua b/examples/generic-host/tests/durable_workflow_qa_recovery_test.lua index aeb4157e..c994d25f 100644 --- a/examples/generic-host/tests/durable_workflow_qa_recovery_test.lua +++ b/examples/generic-host/tests/durable_workflow_qa_recovery_test.lua @@ -123,8 +123,8 @@ local function run_to_barrier(context, live_pids, label) t.eq(barrier.case_results_sha256, recovered.store:digest(barrier.case_results_ref)) t.eq(barrier.replay_status, "completed") t.eq(effect_count(context), 1) - t.eq(#recovered.records:list("testing-runner/cli-effect-authorizations"), 1) - t.eq(#recovered.records:list("testing-runner/cli-effect-consumptions"), 1) + t.eq(#recovered.records:list("testing-runner/effect-authorizations"), 1) + t.eq(#recovered.records:list("testing-runner/effect-consumptions"), 1) t.eq(recovered:terminal_record(), nil) local ownership = recovered:_fixture_effect("fixture-resource-status", { run_id = context.run_id, @@ -159,8 +159,8 @@ local function assert_terminal(context) t.eq(#durable.list_pending(context.project_root, context.durable_root, 10), 0) t.eq(effect_count(context), 1) t.eq(#recovered.records:list("testing-runner/target-effects"), 1) - t.eq(#recovered.records:list("testing-runner/cli-effect-authorizations"), 1) - t.eq(#recovered.records:list("testing-runner/cli-effect-consumptions"), 1) + t.eq(#recovered.records:list("testing-runner/effect-authorizations"), 1) + t.eq(#recovered.records:list("testing-runner/effect-consumptions"), 1) local recovery = recovered.records:read("generic-host/recovery/execution") if context.completed_replay_failpoint ~= nil then t.eq(recovery.replayed, true) else t.eq(recovery, nil) end @@ -483,8 +483,8 @@ return { t.eq(authorization.value.schema, "testing-effect-authorization-receipt.v1") t.eq(authorization.value.decision, "deny") t.eq(authorization.value.reason_code, "profile-policy-denied") - t.eq(#recovered.records:list("testing-runner/cli-effect-authorizations"), 0) - t.eq(#recovered.records:list("testing-runner/cli-effect-consumptions"), 0) + t.eq(#recovered.records:list("testing-runner/effect-authorizations"), 0) + t.eq(#recovered.records:list("testing-runner/effect-consumptions"), 0) t.eq(#recovered.records:list("testing-runner/target-effects"), 0) t.eq(effect_count(context), 0) local cleanup = recovered.store:load(terminal.cleanup_receipt_ref).value @@ -512,8 +512,8 @@ return { stop_live(noop_pid, live_pids) local after = durable.load(context.project_root, context.durable_root, context.run_id) assert_counts_equal(before, record_counts(after)) - t.eq(#after.records:list("testing-runner/cli-effect-authorizations"), 0) - t.eq(#after.records:list("testing-runner/cli-effect-consumptions"), 0) + t.eq(#after.records:list("testing-runner/effect-authorizations"), 0) + t.eq(#after.records:list("testing-runner/effect-consumptions"), 0) t.eq(effect_count(context), 0) end) end, diff --git a/libraries/contract/structured_execution.lua b/libraries/contract/structured_execution.lua index 383eddd9..073d6ce4 100644 --- a/libraries/contract/structured_execution.lua +++ b/libraries/contract/structured_execution.lua @@ -14,6 +14,7 @@ M.schemas = { grant_request = "workflow-qa.execution-grant.request.v1", grant_result = "workflow-qa.execution-grant.result.v1", cli_action_envelope = "testing-cli-action-envelope.v1", + http_action_envelope = "testing-http-action-envelope.v1", effect_authorization_receipt = "testing-effect-authorization-receipt.v1", } @@ -471,6 +472,63 @@ function M.validate_cli_action_envelope(value) return value end +function M.validate_http_action_envelope(value) + only_fields(value, { + schema = true, effect_kind = true, capability = true, profile_ref = true, + profile_artifact_sha256 = true, profile_sha256 = true, validation_receipt_ref = true, + validation_receipt_sha256 = true, preauthorization_ref = true, + preauthorization_sha256 = true, repository = true, run_id = true, + operation_id = true, environment_receipt_ref = true, + environment_receipt_sha256 = true, workspace_ref = true, base_url = true, + plan_ref = true, plan_sha256 = true, grant_ref = true, grant_sha256 = true, + case = true, resource_bounds = true, attempt = true, trace_id = true, + dedup_key = true, expires_at = true, fence_id = true, + }, "http-action-envelope") + if value.schema ~= M.schemas.http_action_envelope then fail("unknown-schema", "HTTP action envelope schema") end + if value.effect_kind ~= "http" or value.capability ~= "loopback-http" then + fail("unsupported-effect", "only the loopback HTTP effect is supported") + end + for _, field in ipairs({ "profile_ref", "validation_receipt_ref", "preauthorization_ref", "environment_receipt_ref", "plan_ref", "grant_ref" }) do + pointer(value[field], field) + end + for _, field in ipairs({ "profile_artifact_sha256", "profile_sha256", "validation_receipt_sha256", "preauthorization_sha256", "environment_receipt_sha256", "plan_sha256", "grant_sha256" }) do + digest(value[field], field) + end + validate_repository(value.repository, "action-repository") + validate_ref(value.workspace_ref, "workspace-ref") + local base_origin = M.local_http_origin(value.base_url) + local request_origin = value.case and value.case.request + and M.local_http_origin(value.case.request.url) or nil + if value.workspace_ref.kind ~= "workspace" or not bounded(value.run_id, 180) + or not bounded(value.operation_id, 180) or value.run_id ~= value.operation_id + or value.attempt ~= 1 or not bounded(value.trace_id, 180) or not bounded(value.dedup_key, 180) + or not bounded(value.fence_id, 180) or time.iso_timestamp_epoch_seconds(value.expires_at) == nil + or base_origin == nil or request_origin ~= base_origin then + fail("malformed-envelope", "HTTP action identity, target, expiry, attempt, or fence is invalid") + end + validate_case(value.case, {}, false) + if value.case.kind ~= "http" or value.case.skip_reason ~= nil then + fail("unsupported-effect", "the action envelope must contain one executable HTTP case") + end + only_fields(value.resource_bounds, { output_bytes = true }, "resource-bounds") + if type(value.resource_bounds.output_bytes) ~= "number" + or value.resource_bounds.output_bytes ~= math.floor(value.resource_bounds.output_bytes) + or value.resource_bounds.output_bytes < 1024 or value.resource_bounds.output_bytes > 1048576 then + fail("unbounded-value", "output_bytes must be from 1024 to 1048576") + end + return value +end + +function M.validate_action_envelope(value) + if type(value) == "table" and value.schema == M.schemas.cli_action_envelope then + return M.validate_cli_action_envelope(value) + end + if type(value) == "table" and value.schema == M.schemas.http_action_envelope then + return M.validate_http_action_envelope(value) + end + fail("unknown-schema", "action envelope schema") +end + function M.validate_effect_authorization_receipt(value, envelope, now) only_fields(value, { schema = true, decision = true, reason_code = true, receipt_id = true, @@ -499,7 +557,7 @@ function M.validate_effect_authorization_receipt(value, envelope, now) if current == nil or current < issued or current >= expires then fail("stale-receipt", "authorization receipt is expired") end end if envelope ~= nil then - M.validate_cli_action_envelope(envelope) + M.validate_action_envelope(envelope) if value.fence_id ~= envelope.fence_id or value.trace_id ~= envelope.trace_id or value.dedup_key ~= envelope.dedup_key or value.expires_at ~= envelope.expires_at then fail("foreign-receipt", "authorization receipt differs from the action envelope") diff --git a/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js b/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js index 77a67396..b82e0b73 100644 --- a/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js +++ b/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js @@ -614,22 +614,44 @@ function argvWithin(argv, capabilities) { && capability.argv_prefix.every((item, index) => item === argv[index])); } +function httpWithin(request, capabilities, baseUrl) { + let target; + try { + target = new URL(request.url); + } catch (_error) { + return false; + } + if (target.search || target.hash || localOrigin(target.toString()) !== localOrigin(baseUrl)) return false; + return (capabilities || []).some((capability) => { + if (!capability || !Array.isArray(capability.methods) + || !Array.isArray(capability.path_prefixes)) return false; + try { + return localOrigin(capability.origin) === localOrigin(baseUrl) + && capability.methods.includes(request.method) + && capability.path_prefixes.some((prefix) => typeof prefix === 'string' + && prefix.startsWith('/') && target.pathname.startsWith(prefix)); + } catch (_error) { + return false; + } + }); +} + function authorizationPath(receiptId) { - return path.join(durableRoot(), 'testing-runner', 'cli-effect-authorization', `${sha256(receiptId)}.json`); + return path.join(durableRoot(), 'testing-runner', 'effect-authorization', `${sha256(receiptId)}.json`); } function receiptTag(config, receipt) { const unsigned = { ...receipt }; delete unsigned.auth_tag; return crypto.createHmac('sha256', config.state_auth_key) - .update(`${config.state_mac_generation}\0cli-effect-receipt\0${stableStringify(unsigned)}`).digest('hex'); + .update(`${config.state_mac_generation}\0effect-receipt\0${stableStringify(unsigned)}`).digest('hex'); } function authorizationReceipt(config, envelope, decision, reasonCode, inputs, now) { const envelopeSha256 = sha256(stableStringify(envelope)); const receipt = { schema: 'testing-effect-authorization-receipt.v1', decision, reason_code: reasonCode, - receipt_id: `cli-effect-${envelopeSha256.slice(0, 40)}`, + receipt_id: `${envelope.effect_kind || 'invalid'}-effect-${envelopeSha256.slice(0, 40)}`, envelope_sha256: envelopeSha256, evaluated_input_digests: inputs, issued_at: utcTimestamp(now), expires_at: envelope.expires_at, @@ -639,30 +661,30 @@ function authorizationReceipt(config, envelope, decision, reasonCode, inputs, no return receipt; } -function validateEnvelope(envelope) { - exactKeys(envelope, [ +function validateEnvelope(envelope, expectedKind) { + const commonFields = [ 'schema', 'effect_kind', 'capability', 'profile_ref', 'profile_artifact_sha256', 'profile_sha256', 'validation_receipt_ref', 'validation_receipt_sha256', 'preauthorization_ref', 'preauthorization_sha256', 'repository', 'run_id', 'operation_id', 'environment_receipt_ref', 'environment_receipt_sha256', 'workspace_ref', 'plan_ref', 'plan_sha256', 'grant_ref', 'grant_sha256', 'case', 'resource_bounds', 'attempt', 'trace_id', 'dedup_key', 'expires_at', 'fence_id', - ], 'CLI action envelope'); + ]; + const kind = envelope && envelope.effect_kind; + exactKeys(envelope, kind === 'http' ? [...commonFields, 'base_url'] : commonFields, + 'action envelope'); exactKeys(envelope.repository, ['url', 'commit_sha'], 'action repository'); exactKeys(envelope.workspace_ref, ['kind', 'ref'], 'action workspace'); exactKeys(envelope.resource_bounds, ['output_bytes'], 'action resource bounds'); - exactKeys(envelope.case, ['case_id', 'kind', 'argv', 'timeout_seconds', 'assertions'], 'action case'); - if (!Array.isArray(envelope.case.assertions) || envelope.case.assertions.length < 1 - || envelope.case.assertions.length > 16 - || envelope.case.assertions.some((assertion) => !assertion - || Object.keys(assertion).sort().join(',') !== 'expected,type' - || assertion.type !== 'exit-code' || !Number.isInteger(assertion.expected) - || assertion.expected < 0 || assertion.expected > 255)) { - throw new Error('CLI action assertions are malformed'); + if (expectedKind !== undefined && kind !== expectedKind) { + throw new Error('action envelope effect kind differs'); } - if (envelope.schema !== 'testing-cli-action-envelope.v1' || envelope.effect_kind !== 'cli' - || envelope.capability !== 'direct-argv' || envelope.attempt !== 1 - || envelope.run_id !== envelope.operation_id || envelope.case.kind !== 'cli' + if (!['cli', 'http'].includes(kind) + || envelope.schema !== (kind === 'cli' + ? 'testing-cli-action-envelope.v1' : 'testing-http-action-envelope.v1') + || envelope.capability !== (kind === 'cli' ? 'direct-argv' : 'loopback-http') + || envelope.attempt !== 1 || envelope.run_id !== envelope.operation_id + || envelope.case.kind !== kind || envelope.workspace_ref.kind !== 'workspace' || !Number.isInteger(envelope.case.timeout_seconds) || envelope.case.timeout_seconds < 1 || envelope.case.timeout_seconds > 300 @@ -671,12 +693,47 @@ function validateEnvelope(envelope) { || !Number.isFinite(Date.parse(envelope.expires_at))) { throw new Error('CLI action envelope is malformed'); } - validateArgv(envelope.case.argv); + if (!Array.isArray(envelope.case.assertions) || envelope.case.assertions.length < 1 + || envelope.case.assertions.length > 16) throw new Error('action assertions are malformed'); + if (kind === 'cli') { + exactKeys(envelope.case, ['case_id', 'kind', 'argv', 'timeout_seconds', 'assertions'], 'action case'); + if (envelope.case.assertions.some((assertion) => !assertion + || Object.keys(assertion).sort().join(',') !== 'expected,type' + || assertion.type !== 'exit-code' || !Number.isInteger(assertion.expected) + || assertion.expected < 0 || assertion.expected > 255)) { + throw new Error('CLI action assertions are malformed'); + } + validateArgv(envelope.case.argv); + } else { + exactKeys(envelope.case, ['case_id', 'kind', 'request', 'timeout_seconds', 'assertions'], 'action case'); + exactKeys(envelope.case.request, ['method', 'url', 'headers'], 'HTTP action request'); + if (!['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE'].includes(envelope.case.request.method) + || !Array.isArray(envelope.case.request.headers) || envelope.case.request.headers.length !== 0 + || envelope.case.assertions.some((assertion) => { + if (!assertion || typeof assertion !== 'object' || Array.isArray(assertion)) return true; + if (assertion.type === 'status-code') { + return Object.keys(assertion).sort().join(',') !== 'expected,type' + || !Number.isInteger(assertion.expected) || assertion.expected < 100 || assertion.expected > 599; + } + if (assertion.type === 'body-contains') { + return Object.keys(assertion).sort().join(',') !== 'expected,type' + || typeof assertion.expected !== 'string' || assertion.expected.length > 512; + } + return assertion.type !== 'json-path-equals' + || Object.keys(assertion).sort().join(',') !== 'expected,path,type' + || typeof assertion.path !== 'string' || assertion.path.length < 1 || assertion.path.length > 512 + || !assertion.path.split('.').every((item) => /^[A-Za-z_][A-Za-z0-9_-]*$/.test(item)) + || !['string', 'number', 'boolean'].includes(typeof assertion.expected); + })) throw new Error('HTTP action assertions are malformed'); + if (localOrigin(envelope.case.request.url) !== localOrigin(envelope.base_url)) { + throw new Error('HTTP action origin differs from ready environment'); + } + } return envelope; } -function evaluateCliEnvelope(config, envelope, now) { - validateEnvelope(envelope); +function evaluateEnvelope(config, envelope, now, expectedKind) { + validateEnvelope(envelope, expectedKind); const profile = boundArtifact(envelope.profile_ref, envelope.profile_artifact_sha256); const validation = boundArtifact(envelope.validation_receipt_ref, envelope.validation_receipt_sha256); const preauthorization = boundArtifact(envelope.preauthorization_ref, envelope.preauthorization_sha256); @@ -727,9 +784,16 @@ function evaluateCliEnvelope(config, envelope, now) { if (!planned || stableStringify(planned) !== stableStringify(envelope.case)) { throw new Error('approved plan scope differs'); } - if (!argvWithin(envelope.case.argv, preauthorization.value.capabilities && preauthorization.value.capabilities.cli) - || !argvWithin(envelope.case.argv, grant.value.cli_capabilities)) { - throw new Error('CLI capability is not authorized'); + if (envelope.effect_kind === 'cli') { + if (!argvWithin(envelope.case.argv, preauthorization.value.capabilities && preauthorization.value.capabilities.cli) + || !argvWithin(envelope.case.argv, grant.value.cli_capabilities)) { + throw new Error('CLI capability is not authorized'); + } + } else if (environment.value.base_url !== envelope.base_url + || !httpWithin(envelope.case.request, + preauthorization.value.capabilities && preauthorization.value.capabilities.http, envelope.base_url) + || !httpWithin(envelope.case.request, grant.value.http_capabilities, envelope.base_url)) { + throw new Error('HTTP capability is not authorized'); } const attested = (config.grant_attestations || []).some((entry) => entry.grant_sha256 === grant.digest && sameAuthority(entry.authority, grant.value.authority) @@ -745,7 +809,7 @@ function evaluateCliEnvelope(config, envelope, now) { return inputs; } -function authorizeCliEffect(payload) { +function authorizeEffect(payload, expectedKind) { const envelope = payload.action_envelope || {}; const config = runtimeConfig(payload); const now = new Date(); @@ -759,7 +823,7 @@ function authorizeCliEffect(payload) { runtimeConfigRef: payload.runtime_config_ref, artifactRoot: payload.artifact_root, }); - inputs = evaluateCliEnvelope(config, envelope, now); + inputs = evaluateEnvelope(config, envelope, now, expectedKind); const receipt = authorizationReceipt(config, envelope, 'allow', 'authorized', inputs, now); const target = authorizationPath(receipt.receipt_id); const release = acquireLock(`${target}.lock`); @@ -793,15 +857,23 @@ function authorizeCliEffect(payload) { } } -async function execArgv(payload) { - const envelope = validateEnvelope(payload.action_envelope); +function authorizeCliEffect(payload) { + return authorizeEffect(payload, 'cli'); +} + +function authorizeHttpEffect(payload) { + return authorizeEffect(payload, 'http'); +} + +async function consumeAuthorizedEffect(payload, expectedKind, execute) { + const envelope = validateEnvelope(payload.action_envelope, expectedKind); const config = targetExecutionConfig(payload, envelope.repository); const receipt = payload.authorization_receipt; exactKeys(receipt, [ 'schema', 'decision', 'reason_code', 'receipt_id', 'envelope_sha256', 'evaluated_input_digests', 'issued_at', 'expires_at', 'fence_id', 'trace_id', 'dedup_key', 'auth_tag', - ], 'CLI authorization receipt'); + ], 'effect authorization receipt'); exactKeys(receipt.evaluated_input_digests, [ 'profile', 'validation_receipt', 'preauthorization', 'environment_receipt', 'plan', 'grant', ], 'evaluated authorization inputs'); @@ -814,7 +886,7 @@ async function execArgv(payload) { || receipt.auth_tag !== receiptTag(config, receipt) || receipt.fence_id !== envelope.fence_id || receipt.trace_id !== envelope.trace_id || receipt.dedup_key !== envelope.dedup_key || Date.now() >= Date.parse(receipt.expires_at)) { - throw new Error('CLI authorization receipt is missing, denied, malformed, expired, or foreign'); + throw new Error('effect authorization receipt is missing, denied, malformed, expired, or foreign'); } const target = authorizationPath(receipt.receipt_id); const release = acquireLock(`${target}.lock`); @@ -822,9 +894,17 @@ async function execArgv(payload) { const current = fs.existsSync(target) ? readJson(target) : null; if (!current || current.status !== 'issued' || stableStringify(current.receipt) !== stableStringify(receipt)) { - throw new Error('CLI authorization receipt was replayed or is unavailable'); + throw new Error('effect authorization receipt was replayed or is unavailable'); } writeJsonAtomic(target, { status: 'consumed', receipt }); + return await execute(envelope, config); + } finally { + release(); + } +} + +async function execArgv(payload) { + return consumeAuthorizedEffect(payload, 'cli', async (envelope, config) => { const workspace = resolveWorkspace({ operation_id: envelope.operation_id, repository: envelope.repository, environment_receipt_sha256: envelope.environment_receipt_sha256, @@ -860,7 +940,7 @@ async function execArgv(payload) { } finally { releaseWorkerEnvironment(environment); } - } finally { release(); } + }); } function localOrigin(value) { @@ -873,26 +953,19 @@ function localOrigin(value) { } function httpRequest(payload) { - const config = targetExecutionConfig(payload, payload.repository); - const allowedMethods = new Set(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE']); - if (!payload.request || !allowedMethods.has(payload.request.method) - || !Array.isArray(payload.request.headers) || payload.request.headers.length !== 0) { - throw new Error('HTTP effect request is invalid'); - } - const target = new URL(payload.request.url); - if (target.search || target.hash) throw new Error('HTTP effect URL must not contain query or fragment'); - if (localOrigin(target.toString()) !== localOrigin(payload.base_url)) { - throw new Error('HTTP effect origin differs from ready environment'); - } - const maximum = Math.max(1024, Math.min(Number(config.http_response_bytes) || 256 * 1024, 1024 * 1024)); - return new Promise((resolve, reject) => { + return consumeAuthorizedEffect(payload, 'http', (envelope, config) => new Promise((resolve, reject) => { + const target = new URL(envelope.case.request.url); + const maximum = Math.min( + Math.max(1024, Math.min(Number(config.http_response_bytes) || 256 * 1024, 1024 * 1024)), + envelope.resource_bounds.output_bytes, + ); const request = http.request({ hostname: target.hostname, port: target.port, path: `${target.pathname}${target.search}`, - method: payload.request.method, + method: envelope.case.request.method, headers: {}, - timeout: Math.max(1, Number(payload.timeout_seconds) || 1) * 1000, + timeout: envelope.case.timeout_seconds * 1000, }, (response) => { const chunks = []; let size = 0; @@ -910,7 +983,7 @@ function httpRequest(payload) { request.on('timeout', () => request.destroy(new Error('HTTP effect timed out'))); request.on('error', reject); request.end(); - }); + })); } function loadResult(payload) { @@ -956,6 +1029,7 @@ async function dispatch(name, payload) { if (name === 'verify-grant') return verifyGrant(payload); if (name === 'replay-guard') return replayGuard(payload); if (name === 'authorize-cli-effect') return authorizeCliEffect(payload); + if (name === 'authorize-http-effect') return authorizeHttpEffect(payload); if (name === 'complete-replay') return completeReplay(payload); if (name === 'exec-argv') return execArgv(payload); if (name === 'http-request') return httpRequest(payload); diff --git a/libraries/testing_runtime/structured_execution.lua b/libraries/testing_runtime/structured_execution.lua index d119cc60..daf2110f 100644 --- a/libraries/testing_runtime/structured_execution.lua +++ b/libraries/testing_runtime/structured_execution.lua @@ -107,11 +107,16 @@ function M.production(options) authorize_cli_effect = function(request) return call_cli("authorize-cli-effect", request, 15, options) end, + authorize_http_effect = function(request) + return call_cli("authorize-http-effect", request, 15, options) + end, exec_argv = function(request) return call_cli("exec-argv", request, (request.timeout_seconds or 30) + 3, options) end, http_request = function(request) - return call_cli("http-request", request, (request.timeout_seconds or 30) + 3, options) + local envelope = request.action_envelope or {} + local case = envelope.case or {} + return call_cli("http-request", request, (case.timeout_seconds or 30) + 3, options) end, write_artifact = function(path, value) local result = call_cli("write-artifact", { diff --git a/libraries/testing_runtime/tests/structured_execution_runtime_test.js b/libraries/testing_runtime/tests/structured_execution_runtime_test.js index 51ea88ef..c912fed6 100644 --- a/libraries/testing_runtime/tests/structured_execution_runtime_test.js +++ b/libraries/testing_runtime/tests/structured_execution_runtime_test.js @@ -21,9 +21,11 @@ function run(argv, cwd) { function listen() { return new Promise((resolve) => { const server = http.createServer((_request, response) => { + server.requestCount += 1; response.writeHead(200, { 'content-type': 'application/json' }); response.end('{"status":"healthy"}'); }); + server.requestCount = 0; server.listen(0, '127.0.0.1', () => resolve(server)); }); } @@ -140,6 +142,9 @@ async function main() { assert.strictEqual(checkout.status, 'passed'); workspaceRef = checkout.workspace_ref; workspace = JSON.parse(fs.readFileSync(resourcePath(workspaceRef.ref), 'utf8')).path; + server = await listen(); + const address = server.address(); + const baseUrl = `http://127.0.0.1:${address.port}/health`; const fsmonitorCanary = path.join(temp, 'ambient-fsmonitor-canary.json'); const fsmonitorHook = path.join(temp, 'ambient-fsmonitor-hook.js'); @@ -184,12 +189,16 @@ async function main() { }); const preauthorization = writeAuthority('preauthorization', { schema: 'testing-structured-execution-authorization.v1', profile_sha256: validation.value.profile_sha256, - repository, capabilities: { cli: [{ argv_prefix: [process.execPath] }], http: [] }, + repository, capabilities: { + cli: [{ argv_prefix: [process.execPath] }], + http: [{ origin: `http://127.0.0.1:${address.port}`, methods: ['GET'], path_prefixes: ['/health'] }], + }, trace_id: traceId, dedup_key: dedupKey, }); const environment = writeAuthority('environment', { schema: 'environment-factory.receipt.v2', status: 'ready', operation_id: operationId, profile_sha256: validation.value.profile_sha256, repository, workspace_ref: workspaceRef, + base_url: baseUrl, trace_id: traceId, dedup_key: dedupKey, }); const cliCase = { @@ -205,6 +214,11 @@ async function main() { ].join(';'), ambientHome], timeout_seconds: 10, assertions: [{ type: 'exit-code', expected: 0 }], }; + const httpCase = { + case_id: 'http-health', kind: 'http', + request: { method: 'GET', url: baseUrl, headers: [] }, timeout_seconds: 10, + assertions: [{ type: 'status-code', expected: 200 }], + }; const plan = writeAuthority('plan', { schema: 'testing-structured-plan.v2', execution_mode: 'structured-api-cli', repository, environment_receipt_sha256: environment.digest, cases: [cliCase], @@ -214,11 +228,28 @@ async function main() { schema: 'testing-structured-execution-grant.v1', grant_id: `${runId}-effect-grant`, parent_authorization_sha256: preauthorization.digest, plan_sha256: plan.digest, environment_receipt_sha256: environment.digest, repository, - cli_capabilities: [{ argv_prefix: [process.execPath] }], authority, + cli_capabilities: [{ argv_prefix: [process.execPath] }], + http_capabilities: [{ + origin: `http://127.0.0.1:${address.port}`, methods: ['GET'], path_prefixes: ['/health'], + }], authority, policy_revision: 'runtime-test-policy-v1', evidence_ref: evidenceRef, expires_at: expiresAt, max_uses: 1, trace_id: traceId, dedup_key: dedupKey, }); grantSha256 = grant.digest; + const httpPlan = writeAuthority('http-plan', { + schema: 'testing-structured-plan.v2', execution_mode: 'structured-api-cli', repository, + environment_receipt_sha256: environment.digest, cases: [httpCase], + trace_id: traceId, dedup_key: dedupKey, + }); + const httpGrant = writeAuthority('http-grant', { + schema: 'testing-structured-execution-grant.v1', grant_id: `${runId}-http-effect-grant`, + parent_authorization_sha256: preauthorization.digest, plan_sha256: httpPlan.digest, + environment_receipt_sha256: environment.digest, repository, + cli_capabilities: [], http_capabilities: [{ + origin: `http://127.0.0.1:${address.port}`, methods: ['GET'], path_prefixes: ['/health'], + }], authority, policy_revision: 'runtime-test-policy-v1', evidence_ref: evidenceRef, + expires_at: expiresAt, max_uses: 1, trace_id: traceId, dedup_key: dedupKey, + }); fs.mkdirSync(path.dirname(configRef), { recursive: true }); fs.writeFileSync(configRef, `${stableStringify({ @@ -237,8 +268,12 @@ async function main() { authorization_capability: false, execution_authorized: false, }, - grant_attestations: [{ grant_sha256: grantSha256, authority, - policy_revision: 'runtime-test-policy-v1', evidence_ref: evidenceRef }], + grant_attestations: [ + { grant_sha256: grantSha256, authority, + policy_revision: 'runtime-test-policy-v1', evidence_ref: evidenceRef }, + { grant_sha256: httpGrant.digest, authority, + policy_revision: 'runtime-test-policy-v1', evidence_ref: evidenceRef }, + ], })}\n`); const common = { @@ -312,22 +347,54 @@ async function main() { ...common, action_envelope: foreignEnvelope, authorization_receipt: denied, }), /missing, denied, malformed, expired, or foreign/); - server = await listen(); - const address = server.address(); - const baseUrl = `http://127.0.0.1:${address.port}/health`; - const response = await dispatch('http-request', { + const httpEffectClaim = await dispatch('replay-guard', { + ...common, grant_id: httpGrant.value.grant_id, grant_sha256: httpGrant.digest, + parent_authorization_sha256: preauthorization.digest, plan_sha256: httpPlan.digest, + environment_receipt_sha256: environment.digest, + }); + const httpEnvelope = { + schema: 'testing-http-action-envelope.v1', effect_kind: 'http', capability: 'loopback-http', + profile_ref: profile.ref, profile_artifact_sha256: profile.digest, + profile_sha256: validation.value.profile_sha256, + validation_receipt_ref: validation.ref, validation_receipt_sha256: validation.digest, + preauthorization_ref: preauthorization.ref, preauthorization_sha256: preauthorization.digest, + repository, run_id: operationId, operation_id: operationId, + environment_receipt_ref: environment.ref, environment_receipt_sha256: environment.digest, + workspace_ref: workspaceRef, base_url: baseUrl, + plan_ref: httpPlan.ref, plan_sha256: httpPlan.digest, + grant_ref: httpGrant.ref, grant_sha256: httpGrant.digest, case: httpCase, + resource_bounds: { output_bytes: 65536 }, attempt: 1, + trace_id: traceId, dedup_key: dedupKey, expires_at: expiresAt, + fence_id: httpEffectClaim.claim_id, + }; + const requestsBeforeUnauthorized = server.requestCount; + await assert.rejects(() => dispatch('http-request', { ...common, base_url: baseUrl, request: { method: 'GET', url: baseUrl, headers: [] }, timeout_seconds: 10, + }), /action envelope/); + assert.strictEqual(server.requestCount, requestsBeforeUnauthorized); + const httpAuthorization = await dispatch('authorize-http-effect', { + ...common, action_envelope: httpEnvelope, + }); + assert.strictEqual(httpAuthorization.decision, 'allow'); + const response = await dispatch('http-request', { + ...common, action_envelope: httpEnvelope, authorization_receipt: httpAuthorization, }); assert.strictEqual(response.status, 200); assert.match(response.body, /healthy/); await assert.rejects(() => dispatch('http-request', { - ...common, - base_url: baseUrl, - request: { method: 'GET', url: 'http://example.invalid/health', headers: [] }, - timeout_seconds: 10, + ...common, action_envelope: httpEnvelope, authorization_receipt: httpAuthorization, + }), /replayed or is unavailable/); + const foreignHttpEnvelope = copy(httpEnvelope); + foreignHttpEnvelope.case.request.url = 'http://example.invalid/health'; + const deniedHttp = await dispatch('authorize-http-effect', { + ...common, action_envelope: foreignHttpEnvelope, + }); + assert.strictEqual(deniedHttp.decision, 'deny'); + await assert.rejects(() => dispatch('http-request', { + ...common, action_envelope: foreignHttpEnvelope, authorization_receipt: deniedHttp, }), /loopback HTTP/); fs.symlinkSync(temp, linkPath); diff --git a/packages/testing-runner/structured_execution.lua b/packages/testing-runner/structured_execution.lua index 847c9f28..8f788b17 100644 --- a/packages/testing-runner/structured_execution.lua +++ b/packages/testing-runner/structured_execution.lua @@ -116,7 +116,7 @@ end local required_ports = { "sha256_bytes", "load_artifact", "now", "verify_grant", "replay_guard", "authorize_cli_effect", - "exec_argv", "http_request", "write_artifact", "load_result", "complete_replay", + "authorize_http_effect", "exec_argv", "http_request", "write_artifact", "load_result", "complete_replay", } local function decode_json_response(body) @@ -336,9 +336,57 @@ local function execute_case(case, grant, ports, context) if not http_allowed(case.request, grant.http_capabilities, context.environment.base_url) then error("testing-runner: structured-execution: unauthorized http capability") end - effect.base_url = context.environment.base_url - effect.request = copy(case.request) - ok, response = pcall(ports.http_request, effect) + local envelope = { + schema = execution_contract.schemas.http_action_envelope, + effect_kind = "http", capability = "loopback-http", + profile_ref = context.request.project_profile_ref, + profile_artifact_sha256 = context.profile.digest, + profile_sha256 = context.request.profile_sha256, + validation_receipt_ref = context.request.validation_receipt_ref, + validation_receipt_sha256 = context.validation.digest, + preauthorization_ref = context.request.preauthorization_ref, + preauthorization_sha256 = context.preauthorization.digest, + repository = copy(context.request.repository), + run_id = context.request.source_ref.ref, operation_id = context.environment.operation_id, + environment_receipt_ref = context.request.environment_receipt_ref, + environment_receipt_sha256 = context.environment_digest, + workspace_ref = copy(context.environment.workspace_ref), + base_url = context.environment.base_url, + plan_ref = context.request.test_plan_ref, plan_sha256 = context.plan.digest, + grant_ref = context.request.execution_grant_ref, grant_sha256 = context.grant.digest, + case = copy(case), + resource_bounds = { output_bytes = context.profile.value.resource_budgets.output_bytes }, + attempt = 1, trace_id = context.request.trace_id, dedup_key = context.request.dedup_key, + expires_at = grant.expires_at, fence_id = context.claim.claim_id, + } + execution_contract.validate_http_action_envelope(envelope) + local receipt = ports.authorize_http_effect({ + action_envelope = envelope, artifact_root = context.request.artifact_root, + operation_id = context.environment.operation_id, + trace_id = context.request.trace_id, dedup_key = context.request.dedup_key, + }) + local receipt_ok = pcall(execution_contract.validate_effect_authorization_receipt, + receipt, envelope, context.now) + local authorization_path = context.request.artifact_root + .. "/authorization/" .. case.case_id .. ".json" + if not receipt_ok or ports.write_artifact(authorization_path, receipt) ~= true then + error("testing-runner: structured-execution: malformed HTTP authorization receipt") + end + if receipt.decision ~= "allow" then + return { + case_id = case.case_id, kind = case.kind, status = "error", + classification = "harness-tooling-issue", assertions = {}, + evidence = { + authorization_receipt_path = authorization_path, + authorization_reason = receipt.reason_code, + }, + } + end + ok, response = pcall(ports.http_request, { + action_envelope = envelope, + authorization_receipt = receipt, + artifact_root = context.request.artifact_root, + }) end local case_result = not ok and effect_error(case, response) or nil if case_result == nil then @@ -378,6 +426,8 @@ local function execute_case(case, grant, ports, context) } or { status_code = tonumber(response.status) or 0, body_excerpt = tostring(response.body or ""):sub(1, 600), + authorization_receipt_path = context.request.artifact_root + .. "/authorization/" .. case.case_id .. ".json", }, } end diff --git a/packages/testing-runner/tests/structured_authorization_contract_test.lua b/packages/testing-runner/tests/structured_authorization_contract_test.lua index 5b3cf5be..d3dd6a08 100644 --- a/packages/testing-runner/tests/structured_authorization_contract_test.lua +++ b/packages/testing-runner/tests/structured_authorization_contract_test.lua @@ -125,6 +125,46 @@ return { t.raises(function() contract.validate_effect_authorization_receipt(foreign, envelope) end) end, + test_http_action_envelope_and_receipt_are_grant_bound = function() + local request = fixtures.request() + local case = { + case_id = "health", kind = "http", timeout_seconds = 10, + request = { method = "GET", url = "http://127.0.0.1:4173/health", headers = {} }, + assertions = { { type = "status-code", expected = 200 } }, + } + local envelope = { + schema = contract.schemas.http_action_envelope, effect_kind = "http", + capability = "loopback-http", profile_ref = request.project_profile_ref, + profile_artifact_sha256 = request.project_profile_artifact_sha256, + profile_sha256 = request.profile_sha256, + validation_receipt_ref = request.validation_receipt_ref, + validation_receipt_sha256 = request.validation_receipt_sha256, + preauthorization_ref = request.preauthorization_ref, + preauthorization_sha256 = request.preauthorization_sha256, + repository = fixtures.copy(request.repository), run_id = request.source_ref.ref, + operation_id = request.source_ref.ref, environment_receipt_ref = request.environment_receipt_ref, + environment_receipt_sha256 = request.environment_receipt_sha256, + workspace_ref = { kind = "workspace", ref = "run-110-workspace" }, + base_url = "http://127.0.0.1:4173/health", + plan_ref = request.test_plan_ref, plan_sha256 = request.test_plan_sha256, + grant_ref = request.execution_grant_ref, grant_sha256 = request.execution_grant_sha256, + case = case, resource_bounds = { output_bytes = 32768 }, attempt = 1, + trace_id = request.trace_id, dedup_key = request.dedup_key, + expires_at = "2026-07-20T01:00:00Z", fence_id = "claim-110", + } + t.eq(contract.validate_http_action_envelope(envelope), envelope) + t.eq(contract.validate_action_envelope(envelope), envelope) + local receipt = fixtures.authorization_receipt(envelope) + t.eq(contract.validate_effect_authorization_receipt(receipt, envelope, + "2026-07-20T00:30:00Z"), receipt) + local foreign = fixtures.copy(envelope) + foreign.base_url = "http://127.0.0.1:4174/health" + t.raises(function() contract.validate_http_action_envelope(foreign) end) + local cli_schema = fixtures.copy(envelope) + cli_schema.schema = contract.schemas.cli_action_envelope + t.raises(function() contract.validate_action_envelope(cli_schema) end) + end, + test_derives_exact_plan_and_environment_bound_single_use_grant = function() local request = fixtures.request() diff --git a/packages/testing-runner/tests/structured_execution_department_test.lua b/packages/testing-runner/tests/structured_execution_department_test.lua index da847395..dc9e072c 100644 --- a/packages/testing-runner/tests/structured_execution_department_test.lua +++ b/packages/testing-runner/tests/structured_execution_department_test.lua @@ -18,6 +18,7 @@ return { verify_grant = function() return fixtures.attestation() end, replay_guard = function() return { status = "claimed", claim_id = "claim-110" } end, authorize_cli_effect = function(input) return fixtures.authorization_receipt(input.action_envelope) end, + authorize_http_effect = function(input) return fixtures.authorization_receipt(input.action_envelope) end, exec_argv = function() return { exit_code = 0, stdout = "fixture 1.0", stderr = "" } end, http_request = function() error("unexpected HTTP request") end, write_artifact = function(path, value) diff --git a/packages/testing-runner/tests/structured_execution_edge_test.lua b/packages/testing-runner/tests/structured_execution_edge_test.lua index ac52759e..6776e515 100644 --- a/packages/testing-runner/tests/structured_execution_edge_test.lua +++ b/packages/testing-runner/tests/structured_execution_edge_test.lua @@ -51,6 +51,9 @@ local function run_edge(mutate, options) authorize_cli_effect = function(input) return fixtures.authorization_receipt(input.action_envelope) end, + authorize_http_effect = function(input) + return fixtures.authorization_receipt(input.action_envelope) + end, exec_argv = function() if options.exec_error then error("cli unavailable") end if options.exec_result == false then return nil end @@ -125,7 +128,7 @@ return { local ports = {} for _, name in ipairs({ "sha256_bytes", "load_artifact", "now", "verify_grant", "replay_guard", - "authorize_cli_effect", "exec_argv", "http_request", "write_artifact", "load_result", + "authorize_cli_effect", "authorize_http_effect", "exec_argv", "http_request", "write_artifact", "load_result", "complete_replay", }) do ports[name] = function() return true end end _G.structured_execution_runtime = ports diff --git a/packages/testing-runner/tests/structured_execution_runtime_adapter_test.lua b/packages/testing-runner/tests/structured_execution_runtime_adapter_test.lua index 552dd029..e350cc95 100644 --- a/packages/testing-runner/tests/structured_execution_runtime_adapter_test.lua +++ b/packages/testing-runner/tests/structured_execution_runtime_adapter_test.lua @@ -25,6 +25,7 @@ local function fake_host() ["verify-grant"] = { grant_sha256 = string.rep("b", 64) }, ["replay-guard"] = { status = "claimed", claim_id = "claim", grant_id = "grant" }, ["authorize-cli-effect"] = { decision = "allow" }, + ["authorize-http-effect"] = { decision = "allow" }, ["exec-argv"] = { exit_code = 0, stdout = "ok", stderr = "" }, ["http-request"] = { status = 200, body = "ok", headers = {} }, ["write-artifact"] = { written = true }, @@ -83,8 +84,11 @@ return { t.eq(ports.verify_grant({ artifact_root = root, operation_id = "op" }).grant_sha256, string.rep("b", 64)) t.eq(ports.replay_guard({ artifact_root = root, grant_id = "grant" }).status, "claimed") t.eq(ports.authorize_cli_effect({ artifact_root = root, operation_id = "op" }).decision, "allow") + t.eq(ports.authorize_http_effect({ artifact_root = root, operation_id = "op" }).decision, "allow") t.eq(ports.exec_argv({ artifact_root = root, case_id = "cli", timeout_seconds = 7 }).exit_code, 0) - t.eq(ports.http_request({ artifact_root = root, case_id = "http", timeout_seconds = 9 }).status, 200) + t.eq(ports.http_request({ artifact_root = root, action_envelope = { + case = { timeout_seconds = 9 }, + } }).status, 200) t.eq(ports.write_artifact(root .. "/result.json", { status = "passed" }), true) t.eq(ports.load_result({ artifact_root = root, result_ref = root .. "/execution.json" }).status, "passed") t.eq(ports.complete_replay({ artifact_root = root, result_ref = root .. "/execution.json" }), true) diff --git a/packages/testing-runner/tests/structured_execution_test.lua b/packages/testing-runner/tests/structured_execution_test.lua index c184ec82..08d17c7f 100644 --- a/packages/testing-runner/tests/structured_execution_test.lua +++ b/packages/testing-runner/tests/structured_execution_test.lua @@ -39,6 +39,10 @@ local function runtime(artifacts, options) if options.authorize_cli_effect then return options.authorize_cli_effect(input) end return fixtures.authorization_receipt(input.action_envelope) end, + authorize_http_effect = function(input) + if options.authorize_http_effect then return options.authorize_http_effect(input) end + return fixtures.authorization_receipt(input.action_envelope) + end, exec_argv = function(input) table.insert(effects, { kind = "cli", request = input }) if options.exec_error then error("cli unavailable") end @@ -247,8 +251,9 @@ return { t.eq(effects[1].request.action_envelope.repository.commit_sha, request.repository.commit_sha) t.eq(effects[1].request.action_envelope.case.argv[2], "--version") t.eq(effects[2].kind, "http") - t.eq(effects[2].request.base_url, "http://127.0.0.1:4173/health") - t.eq(effects[2].request.request.url, "http://127.0.0.1:4173/health") + t.eq(effects[2].request.action_envelope.base_url, "http://127.0.0.1:4173/health") + t.eq(effects[2].request.action_envelope.case.request.url, "http://127.0.0.1:4173/health") + t.eq(effects[2].request.authorization_receipt.decision, "allow") local set = writes[result.case_result_set_path] local manifest = writes[result.evidence_manifest_path] local legacy = writes[result.case_results_path] From f5b1e4327190f92a7aaa6722dddcc6373e2373e4 Mon Sep 17 00:00:00 2001 From: Shaw Zheng Date: Sat, 12 Sep 2026 09:41:04 +0800 Subject: [PATCH 08/11] fix(testing): preserve isolated execution ownership --- .../execution-authorization-lineage.v1.md | 6 +- contracts/target-execution-boundary.v1.md | 10 +- .../generic-host/bin/authorization-lineage.js | 16 +- .../generic-host/bin/generic-host-runtime.js | 437 +++++-- .../generic-host/bin/worker-home-ledger.js | 353 ++++++ .../host_canonical_workflow_qa.lua | 29 + .../generic-host/host_durable_workflow_qa.lua | 37 +- .../durable_workflow_qa_process.lua | 13 + ...thorization_lineage_node_validator_test.js | 59 +- ...ical_browser_walking_skeleton_e2e_test.lua | 2 +- ...ownstream_local_qa_acceptance_e2e_test.lua | 79 +- .../tests/durable_host_store_test.lua | 11 + .../durable_workflow_qa_recovery_test.lua | 3 + .../tests/worker_home_ledger_test.js | 221 ++++ .../tests/worker_home_ledger_test.lua | 18 + .../tests/workspace_checkout_recovery_test.js | 123 ++ .../workspace_checkout_recovery_test.lua | 18 + libraries/contract/environment_factory.lua | 79 +- .../execution_authorization_lineage.lua | 14 +- libraries/contract/structured_execution.lua | 24 +- .../authorization_lineage_projection.lua | 6 + .../bin/fkst-structured-execution-runtime.js | 138 ++- .../structured_execution_runtime_test.js | 137 ++- .../workflow_qa_host_adapter.lua | 8 +- .../bin/environment-factory-runtime.js | 536 +++++---- .../bin/object-bound-cleanup-broker.py | 999 ++++++++++++++++ .../bin/runtime/budgets.js | 1 + .../environment-factory/bin/runtime/common.js | 762 ++++++++++-- .../bin/runtime/lock-holder.js | 30 +- .../bin/runtime/measured-command.js | 33 +- .../bin/runtime/object-bound-exec.js | 77 ++ .../bin/runtime/supervised-process.js | 58 +- .../bin/runtime/target-execution-boundary.js | 14 +- .../bin/runtime/worker-home-resource.js | 418 +++++++ .../bin/runtime/workspace-reservation.js | 122 ++ .../bin/runtime/workspace.js | 63 +- packages/environment-factory/core.lua | 110 +- packages/environment-factory/ports.lua | 2 + packages/environment-factory/runtime.lua | 3 + .../tests/contract_test.lua | 65 + .../environment-factory/tests/core_test.lua | 66 +- .../runtime/source/credential-isolation.js | 7 +- .../runtime/source/database_service.py | 7 +- .../tests/hermetic_e2e_test.lua | 103 +- .../tests/node_runtime_test.js | 1056 +++++++++++++++-- .../object_bound_cleanup_backend_test.lua | 18 + .../object_bound_cleanup_backend_test.py | 77 ++ .../environment-factory/tests/ports_test.lua | 3 +- .../testing-runner/structured_execution.lua | 23 +- .../authorization_lineage_projection_test.lua | 270 +++++ ...on_authorization_lineage_contract_test.lua | 146 ++- .../tests/generic_host_workflow_qa_test.lua | 17 + ...structured_authorization_contract_test.lua | 32 + .../tests/structured_execution_helpers.lua | 4 +- .../tests/structured_execution_test.lua | 104 +- .../tests/workflow_qa_host_adapter_test.lua | 16 + packages/workflow-qa/cleanup_state.lua | 79 ++ packages/workflow-qa/core.lua | 28 +- .../workflow-qa/tests/cleanup_state_test.lua | 53 + packages/workflow-qa/tests/core_test.lua | 12 + ...orkflow_cleanup_retention_test_helpers.lua | 86 ++ .../tests/workflow_core_coverage_helpers.lua | 23 +- .../tests/workflow_core_test_helpers.lua | 69 +- 63 files changed, 6643 insertions(+), 790 deletions(-) create mode 100644 examples/generic-host/bin/worker-home-ledger.js create mode 100644 examples/generic-host/tests/worker_home_ledger_test.js create mode 100644 examples/generic-host/tests/worker_home_ledger_test.lua create mode 100644 examples/generic-host/tests/workspace_checkout_recovery_test.js create mode 100644 examples/generic-host/tests/workspace_checkout_recovery_test.lua create mode 100644 packages/environment-factory/bin/object-bound-cleanup-broker.py create mode 100644 packages/environment-factory/bin/runtime/object-bound-exec.js create mode 100644 packages/environment-factory/bin/runtime/worker-home-resource.js create mode 100644 packages/environment-factory/bin/runtime/workspace-reservation.js create mode 100644 packages/environment-factory/tests/object_bound_cleanup_backend_test.lua create mode 100644 packages/environment-factory/tests/object_bound_cleanup_backend_test.py create mode 100644 packages/testing-runner/tests/authorization_lineage_projection_test.lua create mode 100644 packages/testing-runner/tests/generic_host_workflow_qa_test.lua create mode 100644 packages/workflow-qa/cleanup_state.lua create mode 100644 packages/workflow-qa/tests/cleanup_state_test.lua create mode 100644 packages/workflow-qa/tests/workflow_cleanup_retention_test_helpers.lua diff --git a/contracts/execution-authorization-lineage.v1.md b/contracts/execution-authorization-lineage.v1.md index dc1cbccf..9bef3e7d 100644 --- a/contracts/execution-authorization-lineage.v1.md +++ b/contracts/execution-authorization-lineage.v1.md @@ -6,8 +6,10 @@ logs, counters, self-digests, or replay handles into an execution capability. ## Boundary -Every exported receipt fixes `evidence_role = audit-only`, `authorization_capability = false`, -`reusable = false`, and `source_max_uses = 1`. It binds one immutable repository commit plus the run, +Every exported receipt fixes `evidence_role = audit-only`, `human_approval_required = false`, +`authorization_capability = false`, `execution_authorized = false`, +`promotion_authorized = false`, `reusable = false`, and +`source_max_uses = 1`. It binds one immutable repository commit plus the run, trace, and dedup identities. Receipt validators require complete source bindings; shape validation or partial caller-provided expectations are insufficient. diff --git a/contracts/target-execution-boundary.v1.md b/contracts/target-execution-boundary.v1.md index 19feb040..2656a672 100644 --- a/contracts/target-execution-boundary.v1.md +++ b/contracts/target-execution-boundary.v1.md @@ -20,8 +20,10 @@ The current package accepts exactly one mode: "ref": "fixtures/reviewed-fixture-boundary" }, "policy_revision": "reviewed-fixture-policy-v1", + "human_approval_required": false, "authorization_capability": false, - "execution_authorized": false + "execution_authorized": false, + "promotion_authorized": false } ``` @@ -46,3 +48,9 @@ that prevents target code from reading or replacing Host credentials. The private HOME lease, disabled Git credential helpers/hooks/fsmonitor, and removed GitHub and SSH environment variables are defense-in-depth controls for an admitted fixture. They are not an operating-system sandbox and do not make arbitrary code under the Host UID safe. + +The trusted Host may set `FKST_WORKER_RUNTIME_ROOT` to keep worker HOME allocation separate from the +FKST framework's own `FKST_RUNTIME_ROOT`. The worker root must be a Host-owned, non-symlink private +directory with no group or other permission bits. Existing Hosts that omit it use `FKST_RUNTIME_ROOT` +for compatibility, but the same private-directory checks still apply and fail closed. Neither root, +the object-bound allocation/cleanup broker paths, nor their digests are inherited by target workers. diff --git a/examples/generic-host/bin/authorization-lineage.js b/examples/generic-host/bin/authorization-lineage.js index e80b2106..5e95070b 100644 --- a/examples/generic-host/bin/authorization-lineage.js +++ b/examples/generic-host/bin/authorization-lineage.js @@ -23,7 +23,9 @@ const paths = Object.freeze({ const receiptNames = Object.freeze(Object.keys(paths)); const commonFields = [ 'schema', 'status', 'receipt_id', 'repository', 'run_id', 'trace_id', 'dedup_key', - 'recorded_at', 'source_max_uses', 'evidence_role', 'authorization_capability', 'reusable', + 'recorded_at', 'source_max_uses', 'evidence_role', 'human_approval_required', + 'authorization_capability', 'reusable', + 'execution_authorized', 'promotion_authorized', ]; const definitions = Object.freeze({ @@ -253,7 +255,9 @@ function validateCommon(value, name, definition) { timestamp(value.recorded_at, `${name}.recorded_at`); repository(value.repository, `${name}.repository`); if (value.source_max_uses !== 1 || value.evidence_role !== 'audit-only' - || value.authorization_capability !== false || value.reusable !== false) { + || value.human_approval_required !== false + || value.authorization_capability !== false || value.execution_authorized !== false + || value.promotion_authorized !== false || value.reusable !== false) { fail(`${name} must remain non-reusable audit evidence`); } for (const field of definition.pointers) { @@ -312,12 +316,14 @@ function canonicalDigest(value) { function validateLineageIndex(value, artifacts, expected) { exactKeys(value, [ 'schema', 'status', 'repository', 'run_id', 'trace_id', 'dedup_key', 'recorded_at', - 'receipts', 'lineage_complete', 'source_max_uses', 'evidence_role', - 'authorization_capability', 'reusable', + 'receipts', 'lineage_complete', 'source_max_uses', 'evidence_role', 'human_approval_required', + 'authorization_capability', 'execution_authorized', 'promotion_authorized', 'reusable', ], 'lineage index'); if (value.schema !== schemas.lineage_index || value.status !== 'complete' || value.lineage_complete !== true || value.source_max_uses !== 1 - || value.evidence_role !== 'audit-only' || value.authorization_capability !== false + || value.evidence_role !== 'audit-only' || value.human_approval_required !== false + || value.authorization_capability !== false + || value.execution_authorized !== false || value.promotion_authorized !== false || value.reusable !== false) fail('lineage index must remain complete non-reusable audit evidence'); repository(value.repository, 'lineage index.repository'); identity(value.run_id, 'lineage index.run_id'); diff --git a/examples/generic-host/bin/generic-host-runtime.js b/examples/generic-host/bin/generic-host-runtime.js index 96f1300b..5f1d0324 100755 --- a/examples/generic-host/bin/generic-host-runtime.js +++ b/examples/generic-host/bin/generic-host-runtime.js @@ -20,11 +20,18 @@ function environmentRuntimeHelper(name) { } const { - minimalEnvironment, pathIdentity, releaseWorkerEnvironment, - releaseWorkerEnvironmentLease, removeOwnedDirectory, samePathIdentity, sleep, - verifyWorkerEnvironment, verifyWorkerEnvironmentLease, + OBJECT_BOUND_CLEANUP_UNAVAILABLE, minimalEnvironment, pathEntryExists, pathIdentity, + ownedDirectoryReleaseProven, + releaseWorkerEnvironmentLease, releaseWorkerEnvironmentReservation, + removeOwnedDirectory, requireOwnedDirectory, reservationMatchesLease, samePathIdentity, sleep, + verifyWorkerEnvironment, verifyWorkerEnvironmentLease, workerEnvironmentReleaseProven, + workerEnvironmentReservation, } = environmentRuntimeHelper('common'); const { validateTargetExecutionBoundary } = environmentRuntimeHelper('target-execution-boundary'); +const { prepareReservedWorkspace } = environmentRuntimeHelper('workspace-reservation'); +const { + closeDirectoryAnchor, objectBoundExec, openDirectoryAnchor, +} = environmentRuntimeHelper('object-bound-exec'); const { startOrRecoverSupervisedProcess } = environmentRuntimeHelper('supervised-process'); const { listenerOwners, listenersOwnedByProcessGroup, listenersReleased, processGroupState, terminateProcessGroup, @@ -38,13 +45,6 @@ function sha256(value) { return crypto.createHash('sha256').update(String(value)).digest('hex'); } -function childProcessEnvironment(cwd, reservation = null) { - return minimalEnvironment({}, { - schema: 'generic-host.worker-isolation.v1', - cwd_sha256: sha256(path.resolve(cwd)), - }, reservation && reservation.reservation_id); -} - function durableRoot() { const value = process.env.FKST_GENERIC_HOST_DURABLE_ROOT || process.env.FKST_DURABLE_ROOT; if (typeof value !== 'string' || !path.isAbsolute(value)) fail('generic Host durable root must be absolute'); @@ -414,7 +414,9 @@ function lineageEnvelope(config, schema, status, receiptId, recordedAt, fields) repository: { url: config.repository.url, commit_sha: config.repository.commit_sha }, run_id: config.run_id, trace_id: config.request.trace_id, dedup_key: config.request.dedup_key, recorded_at: recordedAt, source_max_uses: 1, evidence_role: 'audit-only', - authorization_capability: false, reusable: false, ...fields, + human_approval_required: false, + authorization_capability: false, execution_authorized: false, + promotion_authorized: false, reusable: false, ...fields, }; } @@ -567,6 +569,8 @@ function writeProfileClaimReceipt(projectRoot, config, durableClaim) { const validation = boundLineageArtifact(projectRoot, start.validation_receipt_ref.ref, null, 'profile validation'); if (profile.value.revision !== validation.value.profile_revision || approval.value.approval_id !== validation.value.approval_id + || sha256(stable(profile.value)) !== validation.value.profile_sha256 + || sha256(stable(approval.value)) !== validation.value.approval_sha256 || validation.value.profile_sha256 !== config.validation_receipt.profile_sha256 || validation.value.approval_sha256 !== config.validation_receipt.approval_sha256) { fail('profile claim source artifacts differ'); @@ -794,7 +798,9 @@ function writeLineageIndex(projectRoot, config, recordedAt, completionReceipt) { repository: { url: config.repository.url, commit_sha: config.repository.commit_sha }, run_id: config.run_id, trace_id: config.request.trace_id, dedup_key: config.request.dedup_key, recorded_at: recordedAt, receipts, lineage_complete: true, source_max_uses: 1, - evidence_role: 'audit-only', authorization_capability: false, reusable: false, + evidence_role: 'audit-only', human_approval_required: false, + authorization_capability: false, execution_authorized: false, + promotion_authorized: false, reusable: false, }; lineageContract.validateLineageIndex(value, artifacts, expected); const path = `${lineageRoot(config)}/authorization-lineage/index.json`; @@ -899,29 +905,37 @@ function publicationResult(projectRoot, payload) { return result; } -function directExec(argv, cwd, timeoutSeconds, outputBytes) { +function directExec(argv, cwd, timeoutSeconds, outputBytes, environment, cwdIdentity) { if (!Array.isArray(argv) || argv.length === 0 || argv.some((item) => typeof item !== 'string')) { fail('argv must be a non-empty string list'); } - const environment = childProcessEnvironment(cwd); + if (!environment) fail('direct execution requires a durable worker environment'); + verifyWorkerEnvironment(environment); + let anchor; + let result; try { - verifyWorkerEnvironment(environment); + anchor = openDirectoryAnchor(cwd, cwdIdentity); + const launch = objectBoundExec(anchor, argv, 3); const options = { - cwd, + cwd: '/', encoding: 'utf8', timeout: Math.max(1, Number(timeoutSeconds) || 30) * 1000, env: environment, + shell: false, + stdio: ['ignore', 'pipe', 'pipe', anchor.descriptor], }; if (Number.isInteger(outputBytes) && outputBytes >= 1024) options.maxBuffer = outputBytes; - const result = spawnSync(argv[0], argv.slice(1), options); - return { - exit_code: result.status == null ? -1 : result.status, - stdout: result.stdout || '', - stderr: result.stderr || (result.error ? String(result.error.message || result.error) : ''), - }; + result = spawnSync(launch.command, launch.argv, options); + } catch (error) { + return { exit_code: -1, stdout: '', stderr: String(error.message || error) }; } finally { - releaseWorkerEnvironment(environment); + closeDirectoryAnchor(anchor); } + return { + exit_code: result.status == null ? -1 : result.status, + stdout: result.stdout || '', + stderr: result.stderr || (result.error ? String(result.error.message || result.error) : ''), + }; } function environmentStateKey(ref) { @@ -933,6 +947,23 @@ function environmentResourceKey(ref) { return `environment-factory/resources/${sha256(stable(ref))}`; } +const workerHomeLedger = require('./worker-home-ledger').create({ + artifactWrite, fail, minimalEnvironment, recordCas, recordImmutable, recordRead, + releaseWorkerEnvironmentLease, releaseWorkerEnvironmentReservation, + reservationMatchesLease, resourceKey: environmentResourceKey, sha256, stable, + verifyWorkerEnvironmentLease, workerEnvironmentReleaseProven, workerEnvironmentReservation, + workerEnvironmentLease: environmentRuntimeHelper('common').workerEnvironmentLease, +}); + +function ledgerWorkerRequest(config, payload, effectId) { + return { + operation_id: config.run_id, + effect_id: effectId || payload.effect_id, + repository: config.profile.repository, + worker_home_ledger_ref: payload.worker_home_ledger_ref, + }; +} + function exactRuntimePorts(value) { if (!Array.isArray(value) || value.length === 0 || value.length > 32) fail('runtime_ports are invalid'); const names = new Set(); @@ -969,18 +1000,32 @@ function workspaceResource(root, ref) { return resource; } -function verifyWorkspace(config, resource) { +function verifyWorkspace(config, resource, options = {}) { if (resource.operation_id !== config.run_id || resource.path !== config.workspace_root || typeof resource.ownership_token !== 'string' || resource.ownership_token === '') { fail('workspace ownership binding differs'); } - if (!fs.existsSync(resource.path)) return { owned: false, reason: 'workspace-missing' }; + if (!pathEntryExists(resource.path)) { + return options.allowMissingForCleanup === true + ? { owned: true, absent: true } + : { owned: false, reason: 'workspace-missing' }; + } const identity = pathIdentity(resource.path); if (!samePathIdentity(identity, resource.path_identity)) return { owned: false, reason: 'workspace-identity-changed' }; return { owned: true, identity }; } -function registerWorkspace(projectRoot, payload) { +function releaseWorkspaceResource(config, resource) { + const workspaceState = verifyWorkspace(config, resource, { allowMissingForCleanup: true }); + if (!workspaceState.owned) { + fail(`workspace cleanup ownership cannot be verified: ${workspaceState.reason}`); + } + return removeOwnedDirectory( + resource.path, resource.path_identity, config.temp_root, resource.cleanup_capture_id, + ); +} + +function registerWorkspace(projectRoot, payload, reservation = null, identityOverride = null) { const runId = runIdFor(payload); const root = runRoot(runId); const config = loadConfig(projectRoot, runId); @@ -989,12 +1034,20 @@ function registerWorkspace(projectRoot, payload) { || payload.path !== config.workspace_root || payload.repository.commit_sha !== config.commit_sha) { fail('workspace registration binding differs'); } - const identity = pathIdentity(payload.path); + const identity = identityOverride || pathIdentity(payload.path); + if (reservation && (!samePathIdentity(reservation.path_identity, identity) + || reservation.path !== payload.path || reservation.operation_id !== runId + || reservation.repository.url !== payload.repository.url + || reservation.repository.commit_sha !== payload.repository.commit_sha)) { + fail('workspace reservation binding differs at registration'); + } const resource = { - schema: 'generic-host.environment-resource.v1', kind: 'workspace', operation_id: runId, + schema: 'generic-host.environment-resource.v1', kind: 'workspace', operation_id: runId, workspace_ref: payload.workspace_ref, cleanup_ref: payload.cleanup_ref, path: payload.path, path_identity: identity, repository: payload.repository, - ownership_token: crypto.randomBytes(16).toString('hex'), + ownership_token: reservation ? reservation.ownership_token : crypto.randomBytes(16).toString('hex'), + cleanup_capture_id: reservation ? reservation.cleanup_capture_id : crypto.randomBytes(32).toString('hex'), + ...(reservation ? { reservation_id: reservation.reservation_id } : {}), }; for (const ref of [payload.workspace_ref, payload.cleanup_ref]) { const stored = recordImmutable(root, environmentResourceKey(ref), resource); @@ -1003,6 +1056,83 @@ function registerWorkspace(projectRoot, payload) { return { registered: true, path_identity: identity }; } +function workspaceReservationKey(runId) { + return `environment-factory/workspace-reservations/${sha256(stable(`${runId}-workspace`))}`; +} + +function durableWorkspaceReservation(root, config) { + const workspaceRoot = path.resolve(config.workspace_root); + const requestedTempRoot = path.resolve(config.temp_root); + if (workspaceRoot === requestedTempRoot || path.dirname(workspaceRoot) !== requestedTempRoot + || !workspaceRoot.startsWith(`${requestedTempRoot}${path.sep}`)) { + fail('environment checkout workspace escaped the durable temp root'); + } + fs.mkdirSync(requestedTempRoot, { recursive: true, mode: 0o700 }); + requireOwnedDirectory(requestedTempRoot, { privateDirectory: true }); + const tempRoot = requestedTempRoot; + const binding = { + reservation_schema: 'environment-factory.workspace-reservation.v1', + reservation_id: `${config.run_id}-workspace`, + operation_id: config.run_id, + path: workspaceRoot, + containment_root: tempRoot, + containment_root_identity: pathIdentity(tempRoot), + repository: { url: config.repository.url, commit_sha: config.repository.commit_sha }, + }; + const key = workspaceReservationKey(config.run_id); + let current = recordRead(root, key); + let created = false; + if (!current) { + if (pathEntryExists(workspaceRoot)) { + fail('environment checkout workspace exists before durable reservation'); + } + const initial = { + ...binding, + version: 1, + reservation_state: 'reserved', + path_identity: null, + ownership_token: crypto.randomBytes(32).toString('hex'), + cleanup_capture_id: crypto.randomBytes(32).toString('hex'), + }; + const saved = recordCas(root, key, initial, 0); + current = saved.value; + created = saved.saved === true; + } + const observed = { ...current }; + for (const field of [ + 'version', 'reservation_state', 'path_identity', 'ownership_token', 'cleanup_capture_id', + ]) delete observed[field]; + if (stable(observed) !== stable(binding) + || !Number.isInteger(current.version) || current.version < 1 + || !['reserved', 'allocated'].includes(current.reservation_state) + || (current.reservation_state === 'reserved' && current.path_identity !== null) + || (current.reservation_state === 'allocated' && !current.path_identity) + || !/^[0-9a-f]{64}$/.test(String(current.ownership_token || '')) + || !/^[0-9a-f]{64}$/.test(String(current.cleanup_capture_id || ''))) { + fail('durable workspace reservation binding differs'); + } + return { key, record: current, created }; +} + +function recoverRegisteredCheckout(projectRoot, config, payload, workspaceRef, cleanupRef) { + const root = runRoot(config.run_id); + const existing = recordRead(root, environmentResourceKey(workspaceRef)); + if (!existing) return null; + const state = verifyWorkspace(config, existing); + if (!state.owned || existing.reservation_id !== `${config.run_id}-workspace` + || stable(existing.repository) !== stable(config.repository)) { + fail(`registered checkout recovery failed: ${state.reason || 'binding-differs'}`); + } + const cleanupStored = recordImmutable(root, environmentResourceKey(cleanupRef), existing); + if (!cleanupStored.written && !cleanupStored.replayed) { + fail('registered checkout cleanup resource binding differs'); + } + return { + status: 'passed', resolved_commit: config.commit_sha, + workspace_ref: workspaceRef, cleanup_ref: cleanupRef, + }; +} + function startApplication(projectRoot, payload) { const runId = runIdFor(payload); const root = runRoot(runId); @@ -1017,6 +1147,9 @@ function startApplication(projectRoot, payload) { const workspace = workspaceResource(root, payload.workspace_ref); const workspaceState = verifyWorkspace(config, workspace); if (!workspaceState.owned) fail(`workspace ownership failed: ${workspaceState.reason}`); + const purpose = `supervised:${payload.effect_id}`; + const commandEnvironment = config.command_environment || {}; + const workerRequest = ledgerWorkerRequest(config, payload); const binding = { schema: 'generic-host.environment-resource.v1', kind: 'process', operation_id: runId, effect_id: payload.effect_id, cleanup_ref: payload.cleanup_ref, workspace_ref: payload.workspace_ref, @@ -1032,14 +1165,16 @@ function startApplication(projectRoot, payload) { if (existing) { const volatile = new Set([ 'startup_state', 'startup_token_sha256', 'pid', 'pgid', - 'process_start_identity', 'worker_environment_lease', + 'process_start_identity', 'worker_environment_lease', 'worker_home_ledger_ref', + 'worker_home_slot_id', ]); const existingBinding = Object.fromEntries( Object.entries(existing).filter(([key]) => !volatile.has(key)), ); if (stable(existingBinding) !== stable(binding)) fail('application replay binding differs'); try { - verifyWorkerEnvironmentLease(existing.worker_environment_lease); + workerHomeLedger.verifyPersisted(root, workerRequest, purpose, commandEnvironment, + existing.worker_home_slot_id, existing.worker_environment_lease); } catch (_error) { fail('application replay worker environment binding changed'); } @@ -1059,14 +1194,29 @@ function startApplication(projectRoot, payload) { claimPath: startupClaimPath, argv: payload.argv, cwd: workspace.path, - createEnvironment: (reservation) => childProcessEnvironment(workspace.path, reservation), + cwdIdentity: workspace.path_identity, + createEnvironment: (reservation) => { + const allocation = workerHomeLedger.allocate(root, workerRequest, purpose, + commandEnvironment, reservation && reservation.reservation_id); + return { environment: allocation.environment, worker_home_slot_id: allocation.slot_id }; + }, binding, }); - if (launch.interrupted || !launch.resource) fail('application startup was interrupted before registration'); + if (launch.interrupted || !launch.resource) { + fail('application startup was interrupted before worker-home registration'); + } const resource = launch.resource; - if (launch.state !== 'running') { + resource.worker_home_ledger_ref = workerRequest.worker_home_ledger_ref; + workerHomeLedger.verifyPersisted(root, workerRequest, purpose, commandEnvironment, + resource.worker_home_slot_id, resource.worker_environment_lease); + try { const stored = recordImmutable(root, environmentResourceKey(payload.cleanup_ref), resource); if (!stored.written && !stored.replayed) fail('application resource binding differs'); + } catch (error) { + terminateProcessGroup(resource, 500); + throw error; + } + if (launch.state !== 'running') { return { status: 'blocked', cleanup_ref: payload.cleanup_ref, early_exit: true, runtime_ports: ports }; } const deadline = Date.now() + 5_000; @@ -1078,13 +1228,8 @@ function startApplication(projectRoot, payload) { } if (!listenerState || !listenerState.supported || !listenerState.owned) { terminateProcessGroup(resource, 500); - releaseWorkerEnvironmentLease(resource.worker_environment_lease); - const stored = recordImmutable(root, environmentResourceKey(payload.cleanup_ref), resource); - if (!stored.written && !stored.replayed) fail('application resource binding differs'); fail(`application ownership could not be verified: ${listenerState && listenerState.reason || 'process-start-failed'}`); } - const stored = recordImmutable(root, environmentResourceKey(payload.cleanup_ref), resource); - if (!stored.written && !stored.replayed) fail('application resource binding differs'); return { status: 'running', cleanup_ref: payload.cleanup_ref, early_exit: false, runtime_ports: ports }; } @@ -1124,14 +1269,23 @@ function releasedResources(projectRoot, payload) { const root = runRoot(runId); const config = loadConfig(projectRoot, runId); const process = resourceRecord(root, { kind: 'process-cleanup', ref: `${runId}-application` }); + const workspace = workspaceResource(root, { kind: 'workspace', ref: `${runId}-workspace` }); const group = processGroupState(process); const listeners = listenersReleased(process.runtime_ports); + const workspaceReleased = ownedDirectoryReleaseProven( + workspace.path, workspace.path_identity, config.temp_root, workspace.cleanup_capture_id, + ); + const workerEnvironmentReleased = !process.worker_environment_lease + || workerHomeLedger.releaseProven(root, { + operation_id: runId, + repository: process.repository, + worker_home_ledger_ref: process.worker_home_ledger_ref, + }, process.worker_home_slot_id, process.worker_environment_lease); return { process_group_absent: group.supported === true && group.alive === false, listeners_closed: listeners.supported === true && listeners.released === true, - workspace_absent: !fs.existsSync(config.workspace_root), - worker_environment_absent: !process.worker_environment_lease - || !fs.existsSync(process.worker_environment_lease.home), + workspace_absent: workspaceReleased, + worker_environment_absent: workerEnvironmentReleased, }; } @@ -1146,12 +1300,24 @@ function releaseOwnedProcessResource(resource, timeoutMs) { } const listeners = listenersReleased(resource.runtime_ports); if (!listeners.supported || !listeners.released) fail('process listeners remain after cleanup'); - if (!releaseWorkerEnvironmentLease(resource.worker_environment_lease)) { - fail('process worker environment cleanup failed'); - } return true; } +function blockedCleanupResult(projectRoot, payload, resourceKind, detail = {}) { + const diagnosticRef = `${payload.artifact_root}/diagnostics/cleanup-${sha256(stable(payload.cleanup_ref)).slice(0, 16)}.json`; + artifactWrite(projectRoot, diagnosticRef, { + schema: 'generic-host.cleanup-diagnostic.v1', + status: 'blocked', + resource_kind: resourceKind, + reason: OBJECT_BOUND_CLEANUP_UNAVAILABLE, + }); + return { + status: 'blocked', + diagnostic_ref: { kind: 'artifact', ref: diagnosticRef }, + ...detail, + }; +} + function cleanupResource(projectRoot, payload) { const runId = runIdFor(payload); const root = runRoot(runId); @@ -1169,16 +1335,30 @@ function cleanupResource(projectRoot, payload) { fail('process cleanup workspace ownership differs'); } releaseOwnedProcessResource(resource, Math.max(1, Number(payload.timeout_seconds) || 5) * 1000); + if (!workerHomeLedger.recordPersistedRelease(root, { + ...payload, + repository: resource.repository, + worker_home_ledger_ref: resource.worker_home_ledger_ref, + }, resource.worker_home_slot_id, resource.worker_environment_lease, 'supervised-process-stopped')) { + return blockedCleanupResult(projectRoot, payload, resource.kind); + } } else if (resource.kind === 'workspace') { const process = resourceRecord(root, { kind: 'process-cleanup', ref: `${runId}-application` }); const group = processGroupState(process); if (!group.supported || group.alive) fail('workspace cleanup requires a released process group'); const listeners = listenersReleased(process.runtime_ports); if (!listeners.supported || !listeners.released) fail('workspace cleanup requires released listeners'); - const workspaceState = verifyWorkspace(config, resource); - if (!workspaceState.owned) fail(`workspace cleanup ownership cannot be verified: ${workspaceState.reason}`); - if (!removeOwnedDirectory(resource.path, resource.path_identity, config.temp_root)) { - fail('workspace cleanup did not remove the owned workspace'); + if (!releaseWorkspaceResource(config, resource)) { + return blockedCleanupResult(projectRoot, payload, resource.kind); + } + } else if (resource.kind === 'worker-home-ledger') { + const result = workerHomeLedger.cleanup(root, projectRoot, payload, resource); + if (!result.cleaned) { + return blockedCleanupResult(projectRoot, payload, resource.kind, { + resource_detail_ref: result.resource_detail_ref, + resource_detail_sha256: result.resource_detail_sha256, + remaining_count: result.remaining_count, + }); } } else if (resource.kind === 'ports') { const listeners = listenersReleased(resource.runtime_ports); @@ -1208,13 +1388,14 @@ function environmentEffect(projectRoot, payload, produce) { return result; } -function waitForHttp(url, timeoutSeconds) { +function waitForHttp(url, timeoutSeconds, environment) { const script = [ "const http=require('http'),https=require('https'),url=process.argv[1],end=Date.now()+Number(process.argv[2])*1000;", "function poll(){const client=url.startsWith('https:')?https:http;const req=client.get(url,res=>{res.resume();process.exit(res.statusCode>=200&&res.statusCode<500?0:1)});", "req.on('error',()=>{if(Date.now()>=end)process.exit(1);setTimeout(poll,20)});req.setTimeout(500,()=>req.destroy())}poll();", ].join(''); - return directExec([process.execPath, '-e', script, url, String(timeoutSeconds || 30)], process.cwd(), timeoutSeconds).exit_code === 0; + return directExec([process.execPath, '-e', script, url, String(timeoutSeconds || 30)], + process.cwd(), timeoutSeconds, undefined, environment).exit_code === 0; } function structuredReplayKey(grantId) { @@ -1234,7 +1415,7 @@ function structuredConsumptionKey(receiptId) { return `testing-runner/effect-consumptions/${sha256(stable(receiptId))}`; } -function localHttpRequest(request, timeoutSeconds) { +function localHttpRequest(request, timeoutSeconds, environment) { const script = [ "const http=require('http'),url=process.argv[1],method=process.argv[2],timeout=Number(process.argv[3])*1000;", "const req=http.request(url,{method},res=>{const chunks=[];res.on('data',chunk=>chunks.push(chunk));", @@ -1243,7 +1424,7 @@ function localHttpRequest(request, timeoutSeconds) { "req.setTimeout(timeout,()=>req.destroy(new Error('request-timeout')));req.end();", ].join(''); const executed = directExec([process.execPath, '-e', script, request.url, request.method, - String(timeoutSeconds || 30)], process.cwd(), timeoutSeconds); + String(timeoutSeconds || 30)], process.cwd(), timeoutSeconds, undefined, environment); if (executed.exit_code !== 0) fail('structured HTTP request failed'); try { return JSON.parse(executed.stdout); } catch (_error) { fail('structured HTTP response is malformed'); } } @@ -1802,7 +1983,8 @@ function authorizeEffect(projectRoot, payload, expectedKind) { } } -function dispatch(name, payload, projectRoot) { +function dispatch(name, payload, projectRoot, hooks = {}) { + process.env.FKST_OBJECT_BOUND_CLEANUP_STATE_ROOT = durableRoot(); switch (name) { case 'sha256-bytes': return sha256Bytes(payload); @@ -2061,6 +2243,18 @@ function dispatch(name, payload, projectRoot) { case 'remaining-budget': loadConfig(projectRoot, runIdFor(payload)); return { remaining_seconds: 120 }; + case 'initialize-worker-home-ledger': { + const runId = runIdFor(payload); + const root = runRoot(runId); + const config = loadConfig(projectRoot, runId); + return environmentEffect(projectRoot, payload, () => { + if (payload.operation_id !== runId + || stable(payload.repository) !== stable(config.profile.repository)) { + fail('worker-home ledger initialization binding differs'); + } + return workerHomeLedger.initialize(root, payload); + }); + } case 'checkout': { const runId = runIdFor(payload); const root = runRoot(runId); @@ -2070,26 +2264,85 @@ function dispatch(name, payload, projectRoot) { || payload.working_directory !== config.profile.working_directory) { fail('environment checkout binding differs'); } - const workspaceRoot = path.resolve(config.workspace_root); - const tempRoot = path.resolve(config.temp_root); - if (workspaceRoot === tempRoot || !workspaceRoot.startsWith(`${tempRoot}${path.sep}`)) { - fail('environment checkout workspace escaped the durable temp root'); - } - fs.rmSync(config.workspace_root, { recursive: true, force: true }); - const cloned = directExec(['git', 'clone', '--quiet', config.source_root, config.workspace_root], config.temp_root, - payload.timeout_seconds); - if (cloned.exit_code !== 0) fail('environment checkout clone failed'); - const checkedOut = directExec(['git', 'checkout', '--quiet', config.commit_sha], config.workspace_root, - payload.timeout_seconds); - if (checkedOut.exit_code !== 0) fail('environment checkout revision failed'); - const resolved = directExec(['git', 'rev-parse', 'HEAD'], config.workspace_root, payload.timeout_seconds); - const commit = String(resolved.stdout || '').trim(); - if (resolved.exit_code !== 0 || commit !== config.commit_sha) fail('environment checkout resolved commit differs'); const workspaceRef = { kind: 'workspace', ref: `${runId}-workspace` }; const cleanupRef = { kind: 'workspace-cleanup', ref: `${runId}-workspace` }; + const reservation = durableWorkspaceReservation(root, config); + const recovered = recoverRegisteredCheckout( + projectRoot, config, payload, workspaceRef, cleanupRef, + ); + if (recovered) return recovered; + const workspaceIdentity = prepareReservedWorkspace( + reservation.record, reservation.record.path_identity, { + reservationWasCreated: reservation.created, + hooks: { + afterWorkspaceDirectoryCreated(details) { + if (typeof hooks.afterWorkspaceDirectoryCreated === 'function') { + hooks.afterWorkspaceDirectoryCreated(details); + } + crashBarrier(projectRoot, runId, 'checkout-after-workspace-directory-created', { + reservation_id: reservation.record.reservation_id, + }); + }, + afterWorkspaceResourceRegistered(details) { + if (typeof hooks.afterWorkspaceResourceRegistered === 'function') { + hooks.afterWorkspaceResourceRegistered(details); + } + crashBarrier(projectRoot, runId, 'checkout-after-workspace-allocation-registered', { + reservation_id: reservation.record.reservation_id, + path_identity: details.path_identity, + }); + }, + }, + persistIdentity(identity) { + const next = { + ...reservation.record, + version: reservation.record.version + 1, + reservation_state: 'allocated', + path_identity: identity, + }; + const saved = recordCas(root, reservation.key, next, reservation.record.version); + if (!saved.saved || stable(saved.value) !== stable(next)) { + fail('workspace allocation identity commit conflict'); + } + reservation.record = saved.value; + }, + }, + ); + const commit = workerHomeLedger.withEnvironment(root, + ledgerWorkerRequest(config, payload), 'checkout', config.command_environment || {}, (environment) => { + const cloned = directExec(['git', 'clone', '--quiet', config.source_root, '.'], + config.workspace_root, payload.timeout_seconds, undefined, environment, workspaceIdentity); + if (cloned.exit_code !== 0) fail('environment checkout clone failed'); + const checkedOut = directExec(['git', 'checkout', '--quiet', config.commit_sha], + config.workspace_root, payload.timeout_seconds, undefined, environment, workspaceIdentity); + if (checkedOut.exit_code !== 0) fail('environment checkout revision failed'); + const resolved = directExec(['git', 'rev-parse', 'HEAD'], config.workspace_root, + payload.timeout_seconds, undefined, environment, workspaceIdentity); + const observed = String(resolved.stdout || '').trim(); + if (resolved.exit_code !== 0 || observed !== config.commit_sha) { + fail('environment checkout resolved commit differs'); + } + return observed; + }); + if (typeof hooks.afterSuccessfulCheckout === 'function') { + hooks.afterSuccessfulCheckout({ + reservation: { ...reservation.record }, resolved_commit: commit, + }); + } + crashBarrier(projectRoot, runId, 'checkout-after-successful-checkout', { + reservation_id: reservation.record.reservation_id, resolved_commit: commit, + }); registerWorkspace(projectRoot, { run_id: runId, operation_id: runId, workspace_ref: workspaceRef, cleanup_ref: cleanupRef, path: config.workspace_root, repository: config.repository, + }, reservation.record, workspaceIdentity); + if (typeof hooks.afterFinalWorkspaceResourceRegistered === 'function') { + hooks.afterFinalWorkspaceResourceRegistered({ + reservation: { ...reservation.record }, resolved_commit: commit, + }); + } + crashBarrier(projectRoot, runId, 'checkout-after-workspace-resource-registered', { + reservation_id: reservation.record.reservation_id, resolved_commit: commit, }); return { status: 'passed', resolved_commit: commit, workspace_ref: workspaceRef, cleanup_ref: cleanupRef }; }); @@ -2126,27 +2379,39 @@ function dispatch(name, payload, projectRoot) { case 'run-argv': return environmentEffect(projectRoot, payload, (runId, root) => { const workspace = workspaceResource(root, payload.workspace_ref); + const config = loadConfig(projectRoot, runId); if (payload.mode === 'supervised') { return startApplication(projectRoot, { ...payload, run_id: runId, cleanup_ref: { kind: 'process-cleanup', ref: `${runId}-application` }, }); } - const executed = directExec(payload.argv, workspace.path, payload.timeout_seconds); + const executed = workerHomeLedger.withEnvironment(root, + ledgerWorkerRequest(config, payload), `oneshot:${payload.effect_id}`, + config.command_environment || {}, (environment) => + directExec(payload.argv, workspace.path, payload.timeout_seconds, undefined, + environment, workspace.path_identity)); const result = { status: executed.exit_code === 0 ? 'passed' : 'blocked' }; if (payload.requires_frozen_dependencies) result.frozen_dependencies_enforced = true; return result; }); case 'wait-readiness': - return environmentEffect(projectRoot, payload, (_runId, root) => { - for (const check of payload.checks || []) { + return environmentEffect(projectRoot, payload, (runId, root) => { + const config = loadConfig(projectRoot, runId); + for (const [index, check] of (payload.checks || []).entries()) { + const purpose = `readiness:${index + 1}:${String(check.type || 'unknown')}`; if (check.type === 'http') { - if (!waitForHttp(check.url, payload.timeout_seconds)) return { status: 'blocked' }; + const ready = workerHomeLedger.withEnvironment(root, + ledgerWorkerRequest(config, payload), purpose, config.command_environment || {}, + (environment) => waitForHttp(check.url, payload.timeout_seconds, environment)); + if (!ready) return { status: 'blocked' }; } else if (check.type === 'argv') { const workspace = workspaceResource(root, payload.workspace_ref); - if (directExec(check.argv, workspace.path, payload.timeout_seconds).exit_code !== 0) { - return { status: 'blocked' }; - } + const result = workerHomeLedger.withEnvironment(root, + ledgerWorkerRequest(config, payload), purpose, config.command_environment || {}, + (environment) => directExec(check.argv, workspace.path, payload.timeout_seconds, + undefined, environment, workspace.path_identity)); + if (result.exit_code !== 0) return { status: 'blocked' }; } else { return { status: 'blocked' }; } @@ -2263,8 +2528,13 @@ function dispatch(name, payload, projectRoot) { config, 'structured-execution-consumption', envelope.fence_id, ), }); - const result = directExec(envelope.case.argv, workspace.path, envelope.case.timeout_seconds, - envelope.resource_bounds.output_bytes); + const result = workerHomeLedger.withEnvironment(root, { + operation_id: runId, + effect_id: `target-cli:${receipt.receipt_id}`, + repository: config.profile.repository, + }, `target-cli:${envelope.case.case_id}`, config.command_environment || {}, (environment) => + directExec(envelope.case.argv, workspace.path, envelope.case.timeout_seconds, + envelope.resource_bounds.output_bytes, environment, workspace.path_identity)); const sequence = structuredCaseSequence(projectRoot, request, envelope.case.case_id); const stored = recordImmutable(root, `testing-runner/target-effects/${sha256(stable(payload))}`, { sequence, binding: payload, result, @@ -2323,7 +2593,12 @@ function dispatch(name, payload, projectRoot) { config, 'structured-execution-consumption', envelope.fence_id, ), }); - const result = localHttpRequest(envelope.case.request, envelope.case.timeout_seconds); + const result = workerHomeLedger.withEnvironment(runRoot(runId), { + operation_id: runId, + effect_id: `target-http:${receipt.receipt_id}`, + repository: config.profile.repository, + }, `target-http:${envelope.case.case_id}`, config.command_environment || {}, (environment) => + localHttpRequest(envelope.case.request, envelope.case.timeout_seconds, environment)); const sequence = structuredCaseSequence(projectRoot, request, envelope.case.case_id); const stored = recordImmutable(runRoot(runId), `testing-runner/target-effects/${sha256(stable(payload))}`, { sequence, binding: payload, result, @@ -2510,11 +2785,11 @@ if (require.main === module) main(); module.exports = { assertStructuredGrantDerivation, assertExecutionMatchesClaim, - childProcessEnvironment, hostStructuredGrantValues, + dispatch, materializeImmutableNoReplace, preauthorizationBindingMatches, - releaseOwnedProcessResource, + releaseWorkspaceResource, trustedPreauthorizationRefs, verifyMaterializedImmutable, }; diff --git a/examples/generic-host/bin/worker-home-ledger.js b/examples/generic-host/bin/worker-home-ledger.js new file mode 100644 index 00000000..4ff38e6b --- /dev/null +++ b/examples/generic-host/bin/worker-home-ledger.js @@ -0,0 +1,353 @@ +'use strict'; + +const LEDGER_SCHEMA = 'environment-factory.worker-home-ledger.v1'; +const RETENTION_SCHEMA = 'environment-factory.worker-home-retention.v1'; +const RESOURCE_SCHEMA = 'generic-host.environment-resource.v1'; +const MAX_ENTRIES = 256; +const MAX_LEDGER_BYTES = 1024 * 1024; + +function create(deps) { + const { + artifactWrite, fail, minimalEnvironment, recordCas, recordImmutable, recordRead, + releaseWorkerEnvironmentLease, releaseWorkerEnvironmentReservation, + reservationMatchesLease, resourceKey, sha256, stable, verifyWorkerEnvironmentLease, + workerEnvironmentLease, workerEnvironmentReleaseProven, workerEnvironmentReservation, + } = deps; + + function exactRepository(value) { + if (!value || typeof value.url !== 'string' || value.url === '' + || !/^[0-9a-f]{40}$/.test(String(value.commit_sha || ''))) { + fail('worker-home ledger requires an exact repository identity'); + } + return { url: value.url, commit_sha: value.commit_sha }; + } + + function identity(operationId, repository) { + if (typeof operationId !== 'string' || operationId === '') { + fail('worker-home ledger requires operation_id'); + } + const exact = exactRepository(repository); + const binding = { + schema: 'environment-factory.worker-home-ledger-binding.v1', + operation_id: operationId, + repository: exact, + }; + const ledgerId = sha256(stable(binding)); + return { + operationId, + repository: exact, + ledgerId, + cleanupRef: { + kind: 'resource-cleanup', + ref: `environment-factory-worker-home-ledger-${ledgerId.slice(0, 32)}`, + }, + key: 'environment-factory/worker-home-ledger', + }; + } + + function verifyLedger(value, expected) { + if (!value || value.schema !== LEDGER_SCHEMA || value.ledger_id !== expected.ledgerId + || value.operation_id !== expected.operationId + || stable(value.repository) !== stable(expected.repository) + || value.max_entries !== MAX_ENTRIES || !Number.isInteger(value.version) + || value.version < 1 || !Array.isArray(value.entries) + || value.entries.length > MAX_ENTRIES + || Buffer.byteLength(stable(value)) > MAX_LEDGER_BYTES) { + fail('worker-home ledger binding differs'); + } + return value; + } + + function initialize(root, request) { + const expected = identity(request.operation_id, request.repository); + let ledger = recordRead(root, expected.key); + if (!ledger) { + const initial = { + schema: LEDGER_SCHEMA, + ledger_id: expected.ledgerId, + operation_id: expected.operationId, + repository: expected.repository, + version: 1, + max_entries: MAX_ENTRIES, + entries: [], + }; + const saved = recordCas(root, expected.key, initial, 0); + ledger = saved.saved ? initial : saved.value; + } + verifyLedger(ledger, expected); + const resource = { + schema: RESOURCE_SCHEMA, + kind: 'worker-home-ledger', + operation_id: expected.operationId, + cleanup_ref: expected.cleanupRef, + ledger_id: expected.ledgerId, + repository: expected.repository, + ownership_token: sha256(stable({ ledger_id: expected.ledgerId, cleanup_ref: expected.cleanupRef })), + }; + const stored = recordImmutable(root, resourceKey(expected.cleanupRef), resource); + if (!stored.written && !stored.replayed) fail('worker-home ledger resource binding differs'); + return { status: 'passed', ledger_id: expected.ledgerId, cleanup_ref: expected.cleanupRef }; + } + + function requireContext(root, request) { + const expected = identity(request.operation_id, request.repository); + if (request.worker_home_ledger_ref + && stable(request.worker_home_ledger_ref) !== stable(expected.cleanupRef)) { + fail('worker-home ledger cleanup ref differs'); + } + verifyLedger(recordRead(root, expected.key), expected); + const resource = recordRead(root, resourceKey(expected.cleanupRef)); + if (!resource || resource.schema !== RESOURCE_SCHEMA || resource.kind !== 'worker-home-ledger' + || resource.operation_id !== expected.operationId || resource.ledger_id !== expected.ledgerId + || stable(resource.repository) !== stable(expected.repository) + || stable(resource.cleanup_ref) !== stable(expected.cleanupRef)) { + fail('worker-home ledger resource is unavailable'); + } + return expected; + } + + function updateLedger(root, expected, update) { + for (let attempt = 0; attempt < 32; attempt += 1) { + const current = verifyLedger(recordRead(root, expected.key), expected); + const next = JSON.parse(stable(current)); + const result = update(next); + next.version = current.version + 1; + if (Buffer.byteLength(stable(next)) > MAX_LEDGER_BYTES) { + fail('WORKER_HOME_LEDGER_CAPACITY_EXCEEDED'); + } + const saved = recordCas(root, expected.key, next, current.version); + if (saved.saved) return result; + } + fail('worker-home ledger update did not converge'); + } + + function slotBinding(request, purpose, extra) { + if (typeof request.effect_id !== 'string' || request.effect_id === '' + || typeof purpose !== 'string' || purpose === '' || purpose.length > 180) { + fail('worker-home slot identity is invalid'); + } + return { + schema: 'environment-factory.worker-home-slot-binding.v1', + operation_id: request.operation_id, + effect_id: request.effect_id, + purpose, + repository: exactRepository(request.repository), + environment_sha256: sha256(stable(extra || {})), + }; + } + + function findSlot(ledger, slotId) { + return ledger.entries.find((entry) => entry.slot_id === slotId) || null; + } + + function allocate(root, request, purpose, extra = {}, reservationOverride = null, hooks = {}) { + const expected = requireContext(root, request); + const binding = slotBinding(request, purpose, extra); + const slotId = sha256(stable(binding)); + const reservationId = reservationOverride === null ? slotId.slice(0, 32) : String(reservationOverride); + if (!/^[0-9a-f]{32}$/.test(reservationId)) fail('worker-home slot reservation is invalid'); + const isolation = { + schema: 'environment-factory.ledger-worker-isolation.v1', + ledger_id: expected.ledgerId, + slot_id: slotId, + operation_id: request.operation_id, + effect_id: request.effect_id, + purpose, + repository: expected.repository, + }; + const reservation = workerEnvironmentReservation(extra, isolation, reservationId); + updateLedger(root, expected, (ledger) => { + let entry = findSlot(ledger, slotId); + if (entry && (stable(entry.binding) !== stable(binding) + || entry.reservation_id !== reservationId + || stable(entry.worker_environment_reservation) !== stable(reservation))) { + fail('worker-home slot binding differs'); + } + if (!entry) { + if (ledger.entries.length >= ledger.max_entries) fail('WORKER_HOME_LEDGER_CAPACITY_EXCEEDED'); + entry = { + slot_id: slotId, + reservation_id: reservationId, + generation: 1, + binding, + state: 'reserved', + release_reason: null, + worker_environment_reservation: reservation, + worker_environment_lease: null, + }; + ledger.entries.push(entry); + } else if (entry.state === 'released') { + entry.generation += 1; + entry.state = 'reserved'; + entry.release_reason = null; + entry.worker_environment_reservation = reservation; + entry.worker_environment_lease = null; + } + }); + + const environment = minimalEnvironment(extra, isolation, reservationId, hooks); + if (typeof hooks.afterEnvironmentCreated === 'function') hooks.afterEnvironmentCreated(environment); + const lease = workerEnvironmentLease(environment); + if (!reservationMatchesLease(reservation, lease)) { + fail('worker-home slot lease differs from its durable reservation'); + } + updateLedger(root, expected, (ledger) => { + const entry = findSlot(ledger, slotId); + if (!entry || stable(entry.binding) !== stable(binding) + || entry.reservation_id !== reservationId + || stable(entry.worker_environment_reservation) !== stable(reservation)) { + fail('worker-home slot reservation is unavailable'); + } + if (entry.worker_environment_lease + && stable(entry.worker_environment_lease) !== stable(lease)) { + fail('worker-home slot lease differs'); + } + entry.worker_environment_lease = lease; + entry.state = 'allocated'; + entry.release_reason = null; + }); + verifyWorkerEnvironmentLease(lease); + return { environment, identity: expected, lease, slot_id: slotId }; + } + + function recordRelease(root, allocation, reason = 'effect-complete') { + if (!allocation || !allocation.identity || !allocation.lease + || typeof allocation.slot_id !== 'string') fail('worker-home allocation is invalid'); + let released = false; + let releaseReason = reason; + try { + released = releaseWorkerEnvironmentLease(allocation.lease); + if (!released) releaseReason = 'OBJECT_BOUND_CLEANUP_UNAVAILABLE'; + } catch (_error) { + releaseReason = 'release-verification-failed'; + } + updateLedger(root, allocation.identity, (ledger) => { + const entry = findSlot(ledger, allocation.slot_id); + if (!entry || stable(entry.worker_environment_lease) !== stable(allocation.lease)) { + fail('worker-home release binding differs'); + } + entry.state = released ? 'released' : 'retained'; + entry.release_reason = released ? null : releaseReason; + }); + return released; + } + + function recordPersistedRelease(root, request, slotId, lease, reason) { + return recordRelease(root, { + identity: requireContext(root, request), + lease, + slot_id: slotId, + }, reason); + } + + function verifyPersisted(root, request, purpose, extra, slotId, lease) { + const expected = requireContext(root, request); + const binding = slotBinding(request, purpose, extra || {}); + const expectedSlotId = sha256(stable(binding)); + const ledger = verifyLedger(recordRead(root, expected.key), expected); + const entry = findSlot(ledger, expectedSlotId); + if (slotId !== expectedSlotId || !entry || stable(entry.binding) !== stable(binding) + || stable(entry.worker_environment_lease) !== stable(lease) + || (entry.state !== 'allocated' && entry.state !== 'retained')) { + fail('worker-home persisted slot binding differs'); + } + verifyWorkerEnvironmentLease(lease); + return true; + } + + function releaseProven(root, request, slotId, lease) { + const expected = requireContext(root, request); + const ledger = verifyLedger(recordRead(root, expected.key), expected); + const entry = findSlot(ledger, slotId); + return Boolean(entry && entry.state === 'released' + && stable(entry.worker_environment_lease) === stable(lease) + && workerEnvironmentReleaseProven(lease)); + } + + function withEnvironment(root, request, purpose, extra, callback) { + const allocation = allocate(root, request, purpose, extra); + try { + return callback(allocation.environment, allocation); + } finally { + recordRelease(root, allocation); + } + } + + function publicEntry(entry) { + const lease = entry.worker_environment_lease; + const reservation = entry.worker_environment_reservation; + const value = { + slot_id: entry.slot_id, + lease_id: lease ? lease.lease_id : entry.reservation_id, + effect_id: entry.binding.effect_id, + purpose: entry.binding.purpose, + generation: entry.generation, + state: entry.state, + reason: entry.release_reason || 'allocation-not-complete', + }; + if (lease || reservation) { + value.identity_sha256 = (lease || reservation).identity_sha256; + value.marker_sha256 = (lease || reservation).marker_sha256; + } + return value; + } + + function cleanup(root, projectRoot, request, resource) { + const expected = requireContext(root, { + ...request, + repository: resource.repository, + worker_home_ledger_ref: request.cleanup_ref, + }); + if (resource.ledger_id !== expected.ledgerId) fail('worker-home cleanup ledger differs'); + updateLedger(root, expected, (ledger) => { + for (const entry of ledger.entries) { + if (entry.state === 'released') continue; + if (!entry.worker_environment_lease) { + try { + const released = releaseWorkerEnvironmentReservation(entry.worker_environment_reservation); + entry.state = released ? 'released' : 'retained'; + entry.release_reason = released ? null : 'OBJECT_BOUND_CLEANUP_UNAVAILABLE'; + } catch (_error) { + entry.state = 'retained'; + entry.release_reason = 'release-verification-failed'; + } + continue; + } + try { + const released = releaseWorkerEnvironmentLease(entry.worker_environment_lease); + entry.state = released ? 'released' : 'retained'; + entry.release_reason = released ? null : 'OBJECT_BOUND_CLEANUP_UNAVAILABLE'; + } catch (_error) { + entry.state = 'retained'; + entry.release_reason = 'release-verification-failed'; + } + } + }); + const ledger = verifyLedger(recordRead(root, expected.key), expected); + const remaining = ledger.entries.filter((entry) => entry.state !== 'released').map(publicEntry); + if (remaining.length === 0) return { cleaned: true }; + const ref = `${request.artifact_root}/worker-home-retention-${expected.ledgerId.slice(0, 24)}-v${ledger.version}.json`; + const snapshot = { + schema: RETENTION_SCHEMA, + operation_id: request.operation_id, + ledger_id: expected.ledgerId, + repository: expected.repository, + remaining_count: remaining.length, + entries: remaining, + }; + const written = artifactWrite(projectRoot, ref, snapshot); + return { + cleaned: false, + resource_detail_ref: { kind: 'artifact', ref }, + resource_detail_sha256: written.digest, + remaining_count: remaining.length, + }; + } + + return { + allocate, cleanup, initialize, recordPersistedRelease, recordRelease, releaseProven, + verifyPersisted, withEnvironment, + }; +} + +module.exports = { create }; diff --git a/examples/generic-host/host_canonical_workflow_qa.lua b/examples/generic-host/host_canonical_workflow_qa.lua index 88c58c09..a68817cc 100644 --- a/examples/generic-host/host_canonical_workflow_qa.lua +++ b/examples/generic-host/host_canonical_workflow_qa.lua @@ -148,6 +148,15 @@ function Context:_environment_runtime() } end) end, + initialize_worker_home_ledger = function(request) + return replay(request.effect_id, function() + return { + status = "passed", + ledger_id = sha256_bytes(context.run_id .. "\0worker-home-ledger"), + cleanup_ref = { kind = "resource-cleanup", ref = context.run_id .. "-worker-homes" }, + } + end) + end, checkout = function(request) return replay(request.effect_id, function() remove_tree(context.workspace_root, context.temp_root .. "/") @@ -240,6 +249,8 @@ function Context:_environment_runtime() elseif cleanup.kind == "workspace-cleanup" then remove_tree(context.workspace_root, context.temp_root .. "/") workspaces[context.run_id .. "-workspace"] = nil + elseif cleanup.kind == "resource-cleanup" and cleanup.ref == context.run_id .. "-worker-homes" then + -- This in-memory contract fixture never creates a worker HOME. end return { status = "cleaned" } end) @@ -1008,6 +1019,7 @@ function Context:framework_environment(label, arm_failpoint) local runtime_cli = self.project_root .. "/packages/generic-host/bin/generic-host-runtime.js" local environment = { FKST_RUNTIME_ROOT = self.host_root .. "/framework-runtime-" .. label, + FKST_WORKER_RUNTIME_ROOT = self.host_root .. "/fixture-worker-runtime", FKST_DURABLE_ROOT = self.host_root .. "/framework-durable-" .. label, FKST_GENERIC_HOST_DURABLE_ROOT = self.durable_root, FKST_GENERIC_HOST_PROJECT_ROOT = self.project_root, @@ -1023,6 +1035,16 @@ function Context:framework_environment(label, arm_failpoint) FKST_WORKFLOW_QA_ADAPTER_RUNTIME_CONFIG_REF = self.runtime_config_ref, FKST_MODULE_TEST_LOOP_TEST_RUNTIME = "0", } + local broker = self.project_root + .. "/packages/environment-factory/bin/object-bound-cleanup-broker.py" + local source = read_file(broker) + if source == nil then error("canonical workflow allocation broker is unavailable") end + environment.FKST_OBJECT_BOUND_ALLOCATION_BROKER = broker + environment.FKST_OBJECT_BOUND_ALLOCATION_BROKER_SHA256 = sha256_bytes(source) + if self.object_bound_cleanup_broker ~= false then + environment.FKST_OBJECT_BOUND_CLEANUP_BROKER = broker + environment.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = sha256_bytes(source) + end if arm_failpoint == true and type(self.completed_replay_failpoint) == "table" then environment.FKST_DURABLE_COMPLETED_REPLAY_FAILPOINT = self.completed_replay_failpoint.token elseif type(arm_failpoint) == "string" and type(self.crash_barrier) == "table" @@ -1112,6 +1134,12 @@ function M.new(options) local host_root = temp_root .. "/host" require_exec({ "rm", "-rf", temp_root, absolute(artifact_root) }) require_exec({ "mkdir", "-p", source_root, host_root }) + require_exec({ + "node", "-e", + "const fs=require('fs');fs.chmodSync(process.argv[1],0o700);" + .. "if((fs.statSync(process.argv[1]).mode&0o077)!==0)process.exit(44);", + temp_root, + }) local durable_enabled = options.durable == true or options.durable_root ~= nil local supervisor_project_root = project_root if durable_enabled then @@ -1671,6 +1699,7 @@ function M.new(options) completed_replay_failpoint = completed_replay_failpoint, crash_barrier = crash_barrier, runtime_pep_denial = runtime_pep_denial, + object_bound_cleanup_broker = options.object_bound_cleanup_broker ~= false, pep_mutate_plan_binding = options.pep_mutate_plan_binding == true, fixture_name = fixture_name, fixture_source_root = absolute("examples/generic-host/fixtures/" .. fixture_name), diff --git a/examples/generic-host/host_durable_workflow_qa.lua b/examples/generic-host/host_durable_workflow_qa.lua index e43bbdfb..27bfed4c 100644 --- a/examples/generic-host/host_durable_workflow_qa.lua +++ b/examples/generic-host/host_durable_workflow_qa.lua @@ -296,8 +296,14 @@ local function profile_claim_receipt(config, store, projector, durable_claim) local profile = bound_artifact(store, start.profile_ref.ref, nil, "profile") local approval = bound_artifact(store, start.approval_ref.ref, nil, "profile approval") local validation = bound_artifact(store, start.validation_receipt_ref.ref, nil, "profile validation") + local observed_profile_sha256 = project_profile.profile_sha256( + profile.value, function(body) return store.records:digest(body) end) + local observed_approval_sha256 = project_profile.approval_sha256( + approval.value, function(body) return store.records:digest(body) end) if profile.value.revision ~= config.validation_receipt.profile_revision or approval.value.approval_id ~= config.validation_receipt.approval_id + or observed_profile_sha256 ~= validation.value.profile_sha256 + or observed_approval_sha256 ~= validation.value.approval_sha256 or validation.value.profile_sha256 ~= config.validation_receipt.profile_sha256 or validation.value.approval_sha256 ~= config.validation_receipt.approval_sha256 then error("generic-host durable profile claim source artifacts differ") @@ -442,11 +448,26 @@ function Context:_fixture_effect(name, payload, timeout_seconds) payload.request_id = request_id payload.runtime_config_ref = { kind = "artifact", ref = ".testing/generic-host-runtime.json" } write_file(request_path, json_codec.encode(payload) .. "\n") - local result = direct_exec({ - "env", "FKST_DURABLE_ROOT=" .. self.durable_root, + local command = { + "env", + "FKST_DURABLE_ROOT=" .. self.durable_root, + "FKST_WORKER_RUNTIME_ROOT=" .. self.host_root .. "/fixture-worker-runtime", + } + local broker = self.project_root + .. "/packages/environment-factory/bin/object-bound-cleanup-broker.py" + local source = read_file(broker) + if source == nil then error("generic-host durable allocation broker is unavailable", 0) end + table.insert(command, "FKST_OBJECT_BOUND_ALLOCATION_BROKER=" .. broker) + table.insert(command, "FKST_OBJECT_BOUND_ALLOCATION_BROKER_SHA256=" .. self.records:digest(source)) + if self.object_bound_cleanup_broker ~= false then + table.insert(command, "FKST_OBJECT_BOUND_CLEANUP_BROKER=" .. broker) + table.insert(command, "FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256=" .. self.records:digest(source)) + end + for _, item in ipairs({ "node", self.project_root .. "/packages/generic-host/bin/generic-host-runtime.js", "effect", "--name", name, "--request", request_path, "--response", response_path, - }, self.project_root) + }) do table.insert(command, item) end + local result = direct_exec(command, self.project_root) local response_body = read_file(response_path) local decoded_ok, response = pcall(function() return json.decode(response_body) end) if decoded_ok and type(response) == "table" then @@ -526,6 +547,10 @@ function Context:_environment_runtime() } end) end, + initialize_worker_home_ledger = function(request) + return context:_fixture_effect("initialize-worker-home-ledger", request, + request.timeout_seconds) + end, checkout = function(request) return context:_effect("environment-factory", request.effect_id, request, function() remove_tree(context.workspace_root, context.temp_root .. "/") @@ -586,6 +611,7 @@ function Context:_environment_runtime() argv = copy(request.argv), workspace_ref = copy(request.workspace_ref), cleanup_ref = cleanup_ref, + worker_home_ledger_ref = copy(request.worker_home_ledger_ref), runtime_ports = copy(request.runtime_ports), artifact_root = request.artifact_root, trace_id = request.trace_id, @@ -652,6 +678,7 @@ function Context:_workflow_runtime() local state = request and context.records:read("workflow-qa/state/" .. tostring(request.run_id)) or nil local terminal = request and context.records:read("generic-host/terminal/" .. tostring(request.run_id)) or nil if type(request) == "table" and type(state) == "table" + and state.phase ~= "cleanup-blocked" and (state.phase ~= "terminal" or terminal == nil) then table.insert(pending, copy(request)) if #pending >= limit then break end @@ -1702,8 +1729,10 @@ function M.initialize(context, durable_root) kind = "host-policy", ref = "fixtures/" .. context.fixture_name .. "-target-execution-boundary", }, policy_revision = "generic-host-trusted-fixture-exact-v1", + human_approval_required = false, authorization_capability = false, execution_authorized = false, + promotion_authorized = false, }, profile = copy(context.profile), approval = copy(context.approval), @@ -1721,6 +1750,7 @@ function M.initialize(context, durable_root) completed_replay_failpoint = copy(context.completed_replay_failpoint), crash_barrier = copy(context.crash_barrier), runtime_pep_denial = copy(context.runtime_pep_denial), + object_bound_cleanup_broker = context.object_bound_cleanup_broker ~= false, fixture_name = context.fixture_name, fixture_source_root = context.fixture_source_root, use_local_qa_departments = context.use_local_qa_departments == true, @@ -1810,6 +1840,7 @@ function M.list_pending(project_root, durable_root, limit) local context = M.load(project_root, durable_root, run.run_id) local state = context.workflow_runtime.load_state(context.request.state_ref) if state == nil or (type(state) == "table" + and state.phase ~= "cleanup-blocked" and (state.phase ~= "terminal" or context:terminal_record() == nil)) then table.insert(pending, context) if #pending >= limit then break end diff --git a/examples/generic-host/test_support/durable_workflow_qa_process.lua b/examples/generic-host/test_support/durable_workflow_qa_process.lua index 7a0b5042..b561bdec 100644 --- a/examples/generic-host/test_support/durable_workflow_qa_process.lua +++ b/examples/generic-host/test_support/durable_workflow_qa_process.lua @@ -210,6 +210,19 @@ function M.wait_for_terminal(context) return false end +function M.wait_for_state_phase(context, expected_phase, timeout_seconds) + local state_path = context.durable_run_root .. "/records/workflow-qa/state/" + .. context.run_id .. ".json" + local script = table.concat({ + "const fs=require('fs'),path=process.argv[1],expected=process.argv[2],end=Date.now()+Number(process.argv[3])*1000;", + "function poll(){try{const envelope=JSON.parse(fs.readFileSync(path,'utf8'));", + "const state=envelope&&envelope.value||envelope;if(state&&state.phase===expected)process.exit(0)}catch(_error){}", + "if(Date.now()>=end)process.exit(49);setTimeout(poll,20)}poll();", + }) + return M.exec({ "node", "-e", script, state_path, expected_phase, + tostring(timeout_seconds or 180) }).exit_code == 0 +end + function M.wait_for_noop(context, label) local root = context.host_root .. "/framework-runtime-" .. label .. "/logs/framework-child" for attempt = 1, 4 do diff --git a/examples/generic-host/tests/authorization_lineage_node_validator_test.js b/examples/generic-host/tests/authorization_lineage_node_validator_test.js index 5491213a..020e902f 100644 --- a/examples/generic-host/tests/authorization_lineage_node_validator_test.js +++ b/examples/generic-host/tests/authorization_lineage_node_validator_test.js @@ -7,11 +7,6 @@ const path = require('node:path'); const { stable } = require(path.resolve(__dirname, '../bin/durable-host-store')); const lineage = require(path.resolve(__dirname, '../bin/authorization-lineage')); const runtime = require(path.resolve(__dirname, '../bin/generic-host-runtime')); -const { - releaseWorkerEnvironment, - verifyWorkerEnvironment, - workerEnvironmentLease, -} = require(path.resolve(__dirname, '../../../packages/environment-factory/bin/runtime/common')); const runId = 'node-lineage-validator'; const repository = { url: 'https://example.invalid/testing/fixture.git', commit_sha: '1'.repeat(40) }; @@ -22,12 +17,15 @@ const copy = (value) => JSON.parse(JSON.stringify(value)); const common = (value) => ({ repository: copy(repository), run_id: runId, trace_id: 'trace-node-lineage', dedup_key: runId, recorded_at: '2026-09-10T00:10:00Z', source_max_uses: 1, evidence_role: 'audit-only', - authorization_capability: false, reusable: false, ...value, + human_approval_required: false, + authorization_capability: false, execution_authorized: false, + promotion_authorized: false, reusable: false, ...value, }); const envelopeFields = new Set([ 'schema', 'status', 'receipt_id', 'recorded_at', 'source_max_uses', - 'evidence_role', 'authorization_capability', 'reusable', + 'evidence_role', 'human_approval_required', 'authorization_capability', 'execution_authorized', + 'promotion_authorized', 'reusable', ]); const expected = (value) => Object.fromEntries( Object.entries(copy(value)).filter(([key]) => !envelopeFields.has(key)), @@ -135,7 +133,9 @@ function reseal(state) { name, { ref: artifact.ref, sha256: artifact.sha256 }, ])), lineage_complete: true, source_max_uses: 1, evidence_role: 'audit-only', - authorization_capability: false, reusable: false, + human_approval_required: false, + authorization_capability: false, execution_authorized: false, + promotion_authorized: false, reusable: false, }; } @@ -165,7 +165,10 @@ rejects((values) => { }); rejects((values) => { values.execution_claim.grant_id = 'foreign-grant'; }); rejects((values) => { values.preauthorization_claim.claimed_at = '2026-09-09T23:59:59Z'; }); +rejects((values) => { values.execution_claim.human_approval_required = true; }); rejects((values) => { values.execution_claim.authorization_capability = true; }); +rejects((values) => { values.execution_claim.execution_authorized = true; }); +rejects((values) => { values.execution_claim.promotion_authorized = true; }); for (const invalidTimestamp of [ '2026-02-30T00:00:00Z', '2026-09-10T24:00:00Z', @@ -332,43 +335,3 @@ assert.doesNotThrow(() => runtime.assertStructuredGrantDerivation(structuredConf cliState.grant.value.cli_capabilities[0].argv_prefix = ['fixture-cli']; assert.throws(() => runtime.assertStructuredGrantDerivation(structuredConfig, structuredRequest, cliState.preauthorization, cliState.plan, cliState.environment, cliState.grant)); - -for (const key of [ - 'GH_TOKEN', 'GITHUB_TOKEN', 'SSH_AUTH_SOCK', 'GIT_ASKPASS', 'SSH_ASKPASS', - 'FKST_GENERIC_HOST_DURABLE_ROOT', 'FKST_GENERIC_HOST_PROJECT_ROOT', - 'FKST_STRUCTURED_EXECUTION_RUNTIME_CLI', 'FKST_STRUCTURED_EXECUTION_RUNTIME_CONFIG_REF', - 'FKST_DURABLE_COMPLETED_REPLAY_FAILPOINT', 'FKST_GENERIC_HOST_FIXTURE_CLI_DENY_TOKEN', -]) { - process.env[key] = 'must-not-be-inherited'; -} -const childEnvironment = runtime.childProcessEnvironment(process.cwd()); -for (const key of [ - 'GH_TOKEN', 'GITHUB_TOKEN', 'SSH_AUTH_SOCK', 'GIT_ASKPASS', 'SSH_ASKPASS', - 'FKST_GENERIC_HOST_DURABLE_ROOT', 'FKST_GENERIC_HOST_PROJECT_ROOT', - 'FKST_STRUCTURED_EXECUTION_RUNTIME_CLI', 'FKST_STRUCTURED_EXECUTION_RUNTIME_CONFIG_REF', - 'FKST_DURABLE_COMPLETED_REPLAY_FAILPOINT', 'FKST_GENERIC_HOST_FIXTURE_CLI_DENY_TOKEN', -]) { - assert.equal(Object.prototype.hasOwnProperty.call(childEnvironment, key), false); -} -assert.equal(childEnvironment.GIT_TERMINAL_PROMPT, '0'); -assert.equal(childEnvironment.GIT_CONFIG_NOSYSTEM, '1'); -assert.equal(childEnvironment.GIT_CONFIG_COUNT, '4'); -assert.equal(childEnvironment.GIT_CONFIG_KEY_2, 'core.fsmonitor'); -assert.equal(childEnvironment.GIT_CONFIG_VALUE_2, 'false'); -assert.equal(childEnvironment.GIT_CONFIG_KEY_3, 'core.hooksPath'); -assert.equal(verifyWorkerEnvironment(childEnvironment), true); -const childHome = childEnvironment.HOME; -assert.equal(releaseWorkerEnvironment(childEnvironment), true); -assert.equal(fs.existsSync(childHome), false); - -const exitedEnvironment = runtime.childProcessEnvironment(process.cwd()); -const exitedLease = workerEnvironmentLease(exitedEnvironment); -assert.equal(runtime.releaseOwnedProcessResource({ - pid: 2147483647, - pgid: 2147483647, - process_start_identity: 'naturally-exited-process', - runtime_ports: [], - worker_environment_lease: exitedLease, -}, 100), true); -assert.equal(fs.existsSync(exitedLease.home), false); -assert.equal(releaseWorkerEnvironment(exitedEnvironment), true); diff --git a/examples/generic-host/tests/canonical_browser_walking_skeleton_e2e_test.lua b/examples/generic-host/tests/canonical_browser_walking_skeleton_e2e_test.lua index d40fd48f..961b6228 100644 --- a/examples/generic-host/tests/canonical_browser_walking_skeleton_e2e_test.lua +++ b/examples/generic-host/tests/canonical_browser_walking_skeleton_e2e_test.lua @@ -123,7 +123,7 @@ return { t.eq(context.store:load(compatibility_path), nil) local cleanup = artifact(context, context.terminal.cleanup_receipt_ref).value - t.eq(cleanup.schema, "environment-factory.cleanup-receipt.v1") + t.eq(cleanup.schema, "environment-factory.cleanup-receipt.v2") t.eq(cleanup.status, "complete") t.eq(#cleanup.remaining_resources, 0) local aggregate = artifact(context, context.request.publication.aggregate_report_ref).value diff --git a/examples/generic-host/tests/downstream_local_qa_acceptance_e2e_test.lua b/examples/generic-host/tests/downstream_local_qa_acceptance_e2e_test.lua index 86e23512..fd8eda6c 100644 --- a/examples/generic-host/tests/downstream_local_qa_acceptance_e2e_test.lua +++ b/examples/generic-host/tests/downstream_local_qa_acceptance_e2e_test.lua @@ -22,6 +22,8 @@ local function adapter_marker_counts(context, label) "local-qa-host dept=intake tag=ROUTED run_id=" .. context.run_id), execution_grant = process.count_child_logs(root, "local-qa-host-adapter.execution_grant-", "local-qa-host dept=execution_grant tag=GRANTED"), + execution_grant_replay = process.count_child_logs(root, + "local-qa-host-adapter.execution_grant-", "REPLAY_SCRATCH_BYPASS=enabled"), terminal = process.count_child_logs(root, "local-qa-host-adapter.terminal-", "local-qa-host dept=terminal tag=RECORDED run_id=" .. context.run_id), } @@ -83,6 +85,7 @@ return { process.with_context({ scenario = "downstream-inventory", durable = true, prepare_execution_grant_pending = false, publication_channel = "filesystem-dry-run-v1", arm_completed_replay_failpoint = true, + object_bound_cleanup_broker = false, }, function(context, live_pids) t.eq(#context.commit_sha, 40) t.is_true(context.commit_sha:match("^[0-9a-f]+$") ~= nil) @@ -173,13 +176,15 @@ return { t.eq(surviving.owned, true) t.eq(surviving.pgid, ownership.pgid) t.eq(surviving.ownership_token, ownership.ownership_token) - local adapter_calls = { intake = 0, execution_grant = 0, terminal = 0 } + local adapter_calls = { + intake = 0, execution_grant = 0, execution_grant_replay = 0, terminal = 0, + } add_marker_counts(adapter_calls, adapter_marker_counts(context, "inventory-first")) local second_pid, second_stdout, second_stderr = process.start_supervisor( context, "inventory-second", false, live_pids) - if not process.wait_for_terminal(context) then - error("inventory replacement did not reach terminal\nstdout=" .. tostring(process.read_file(second_stdout)) + if not process.wait_for_state_phase(context, "cleanup-blocked") then + error("inventory replacement did not reach cleanup-blocked\nstdout=" .. tostring(process.read_file(second_stdout)) .. "\nstderr=" .. tostring(process.read_file(second_stderr))) end process.stop_live(second_pid, live_pids) @@ -214,46 +219,38 @@ return { t.eq(preauthorization.capabilities.http[1].methods[1], "GET") t.eq(preauthorization.capabilities.http[1].path_prefixes[1], "/inventory/") - local terminal = recovered:terminal_record() - t.eq(terminal.schema, "workflow-qa.terminal-request.v2") - t.eq(terminal.status, "passed") - for name, expected in pairs({ - planned = 5, executed = 5, passed = 5, failed = 0, skipped = 0, error = 0, blocked = 0, - }) do t.eq(terminal.counts[name], expected) end - local publication = artifact(recovered, terminal.aggregate_publication_receipt_ref) - t.eq(publication.status, "published") - t.eq(publication.channel, "filesystem-dry-run-v1") - t.eq(publication.remote_url, nil) - local aggregate = artifact(recovered, recovered.request.publication.aggregate_report_ref) - t.eq(aggregate.status, "passed") - t.eq(aggregate.counts.passed, 5) - local cleanup = artifact(recovered, terminal.cleanup_receipt_ref) - t.eq(cleanup.status, "complete") - t.eq(#cleanup.remaining_resources, 0) - local report_path = recovered.artifact_root .. "/acceptance-report.md" - local report = artifact(recovered, report_path) - t.is_true(type(report) == "string") - local previous = 0 - for _, expected in ipairs(CASE_IDS) do - local position = assert(report:find(expected, previous + 1, true)) - t.is_true(position > previous) - previous = position - end - t.eq(report:match("([^\n]+)\n$"), "Verdict: downstream business acceptance passed") + local state = recovered.workflow_runtime.load_state(recovered.request.state_ref) + t.eq(state.phase, "cleanup-blocked") + t.eq(state.terminal_status, "blocked") + t.eq(#state.pending_actions, 0) + t.eq(recovered:terminal_record(), nil) + t.eq(recovered.store:load(recovered.request.publication.aggregate_report_ref), nil) + local cleanup = artifact(recovered, state.cleanup_result.cleanup_receipt_ref.ref) + t.eq(cleanup.schema, "environment-factory.cleanup-receipt.v2") + t.eq(cleanup.status, "incomplete") + t.is_true(#cleanup.remaining_resources >= 1) + t.eq(state.cleanup_blocked.reason, "cleanup-incomplete") + t.is_true(#state.cleanup_blocked.worker_home_retention == 1) + local retention = artifact(recovered, + state.cleanup_blocked.worker_home_retention[1].resource_detail_ref.ref) + t.eq(retention.schema, "environment-factory.worker-home-retention.v1") + t.eq(retention.operation_id, context.run_id) + t.eq(retention.repository.commit_sha, context.commit_sha) + t.eq(retention.remaining_count, #retention.entries) local released = recovered:_fixture_effect("fixture-release-status", { run_id = context.run_id, artifact_root = context.artifact_root, }) t.eq(released.process_group_absent, true) t.eq(released.listeners_closed, true) - t.eq(released.workspace_absent, true) - t.eq(released.worker_environment_absent, true) - t.eq(support.read_file(context.workspace_root .. "/state/inventory.json"), nil) + t.eq(released.workspace_absent, false) + t.eq(released.worker_environment_absent, false) + t.eq(support.read_file(context.workspace_root .. "/state/inventory.json"), RESERVED) local before = counts(recovered) local noop_pid, noop_stdout, noop_stderr = process.start_supervisor( context, "inventory-noop", false, live_pids) if not process.wait_for_noop(context, "inventory-noop") then - error("inventory terminal replay was not a no-op\nstdout=" .. tostring(process.read_file(noop_stdout)) + error("inventory cleanup-blocked replay was not a no-op\nstdout=" .. tostring(process.read_file(noop_stdout)) .. "\nstderr=" .. tostring(process.read_file(noop_stderr))) end process.stop_live(noop_pid, live_pids) @@ -261,21 +258,23 @@ return { t.eq(#durable.load(context.project_root, context.durable_root, context.run_id).records:list( "generic-host/local-qa-intake"), 1) t.eq(adapter_calls.intake, 1) - t.eq(adapter_calls.execution_grant, 1) - t.eq(adapter_calls.terminal, 1) + t.is_true(adapter_calls.execution_grant >= 1) + t.eq(adapter_calls.execution_grant_replay, adapter_calls.execution_grant - 1) + t.eq(adapter_calls.terminal, 0) assert_same_counts(before, counts(durable.load(context.project_root, context.durable_root, context.run_id))) t.eq(before.profile, 1) t.eq(before.preauthorization, 1) t.eq(before.replay, 1) + t.eq(#recovered.records:list("testing-runner/grant-verifications"), 1) t.eq(before.authorization, 5) t.eq(before.consumption, 5) t.eq(before.effects, 5) - t.eq(before.publication, 16) + t.is_true(before.publication < 16) local published = durable.load(context.project_root, context.durable_root, context.run_id) - t.eq(publication_stage_count(published, "aggregate-source-case-result-set"), 1) - t.eq(publication_stage_count(published, "aggregate-source-evidence-manifest"), 1) - t.eq(publication_stage_count(published, "aggregate-report"), 1) - t.eq(before.terminal, 1) + t.eq(publication_stage_count(published, "aggregate-source-case-result-set"), 0) + t.eq(publication_stage_count(published, "aggregate-source-evidence-manifest"), 0) + t.eq(publication_stage_count(published, "aggregate-report"), 0) + t.eq(before.terminal, 0) end) end, } diff --git a/examples/generic-host/tests/durable_host_store_test.lua b/examples/generic-host/tests/durable_host_store_test.lua index f1a55f1d..f1267272 100644 --- a/examples/generic-host/tests/durable_host_store_test.lua +++ b/examples/generic-host/tests/durable_host_store_test.lua @@ -260,8 +260,19 @@ return { runtime_cli = context.project_root .. "/packages/generic-host/bin/generic-host-runtime.js", runtime_config_ref = { kind = "artifact", ref = context.runtime_config_ref }, exec_argv = function(request) + local trusted = context:framework_environment("durable-host-store-effect") local command = { "env", "FKST_GENERIC_HOST_DURABLE_ROOT=" .. context.durable_root, + "FKST_RUNTIME_ROOT=" .. trusted.FKST_RUNTIME_ROOT, + "FKST_WORKER_RUNTIME_ROOT=" .. trusted.FKST_WORKER_RUNTIME_ROOT, + "FKST_OBJECT_BOUND_ALLOCATION_BROKER=" + .. trusted.FKST_OBJECT_BOUND_ALLOCATION_BROKER, + "FKST_OBJECT_BOUND_ALLOCATION_BROKER_SHA256=" + .. trusted.FKST_OBJECT_BOUND_ALLOCATION_BROKER_SHA256, + "FKST_OBJECT_BOUND_CLEANUP_BROKER=" + .. trusted.FKST_OBJECT_BOUND_CLEANUP_BROKER, + "FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256=" + .. trusted.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256, "sh", "-c", 'cd "$1" && shift && exec "$@"', "sh", context.project_root, } for _, item in ipairs(request.argv or {}) do table.insert(command, item) end diff --git a/examples/generic-host/tests/durable_workflow_qa_recovery_test.lua b/examples/generic-host/tests/durable_workflow_qa_recovery_test.lua index c994d25f..2bbc92f7 100644 --- a/examples/generic-host/tests/durable_workflow_qa_recovery_test.lua +++ b/examples/generic-host/tests/durable_workflow_qa_recovery_test.lua @@ -46,7 +46,10 @@ local function assert_authorization_lineage(context, recovered) t.is_true(artifact.raw:find('"fence_id"', 1, true) == nil) t.is_true(artifact.raw:find('"mac"', 1, true) == nil) t.is_true(artifact.raw:find(context.workspace_root, 1, true) == nil) + t.is_true(artifact.value.human_approval_required == false) t.is_true(artifact.value.authorization_capability == false) + t.is_true(artifact.value.execution_authorized == false) + t.is_true(artifact.value.promotion_authorized == false) t.is_true(artifact.value.reusable == false) t.eq(artifact.value.source_max_uses, 1) end diff --git a/examples/generic-host/tests/worker_home_ledger_test.js b/examples/generic-host/tests/worker_home_ledger_test.js new file mode 100644 index 00000000..878bc680 --- /dev/null +++ b/examples/generic-host/tests/worker_home_ledger_test.js @@ -0,0 +1,221 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const store = require('../bin/durable-host-store'); +const { releaseWorkspaceResource } = require('../bin/generic-host-runtime'); +const common = require('../../../packages/environment-factory/bin/runtime/common'); +const { create } = require('../bin/worker-home-ledger'); + +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'generic-host-worker-ledger-')); +const durable = path.join(root, 'durable'); +const runtime = path.join(root, 'runtime'); +process.env.FKST_RUNTIME_ROOT = runtime; +process.env.FKST_DURABLE_ROOT = durable; +const cleanupBroker = path.resolve( + __dirname, '..', '..', '..', 'packages', 'environment-factory', 'bin', + 'object-bound-cleanup-broker.py', +); +process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER = cleanupBroker; +process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = common.sha256(fs.readFileSync(cleanupBroker)); +const stable = store.stable; +const sha256 = (value) => crypto.createHash('sha256').update(String(value)).digest('hex'); +const records = new Map(); +const copy = (value) => value == null ? value : JSON.parse(stable(value)); +const read = (key) => copy(records.get(key) || null); +const cas = (key, value, version) => { + const current = records.get(key); + const currentVersion = current ? current.version : 0; + if (currentVersion !== version) { + return { saved: false, stale: true, version: currentVersion, value: copy(current) }; + } + records.set(key, copy(value)); + return { saved: true, stale: false, version: value.version, value: copy(value) }; +}; +const immutable = (key, value) => { + const current = records.get(key); + if (current) return { written: false, replayed: stable(current) === stable(value), value: copy(current) }; + records.set(key, copy(value)); + return { written: true, replayed: false, value: copy(value) }; +}; +const resourceKey = (ref) => `environment-factory/resources/${sha256(stable(ref))}`; +const artifacts = new Map(); +const ledger = create({ + artifactWrite: (_projectRoot, ref, value) => { + const body = `${stable(value)}\n`; + const digest = sha256(body); + const prior = artifacts.get(ref); + if (prior && prior.body !== body) throw new Error('immutable artifact differs'); + artifacts.set(ref, { body, value, digest }); + return { written: true, replayed: Boolean(prior), digest }; + }, + fail: (message) => { throw new Error(message); }, + minimalEnvironment: common.minimalEnvironment, + recordCas: (_root, key, value, version) => cas(key, value, version), + recordImmutable: (_root, key, value) => immutable(key, value), + recordRead: (_root, key) => read(key), + releaseWorkerEnvironmentLease: common.releaseWorkerEnvironmentLease, + releaseWorkerEnvironmentReservation: common.releaseWorkerEnvironmentReservation, + reservationMatchesLease: common.reservationMatchesLease, + resourceKey, + sha256, + stable, + verifyWorkerEnvironmentLease: common.verifyWorkerEnvironmentLease, + workerEnvironmentLease: common.workerEnvironmentLease, + workerEnvironmentReleaseProven: common.workerEnvironmentReleaseProven, + workerEnvironmentReservation: common.workerEnvironmentReservation, +}); + +const repository = { url: 'https://example.invalid/fixture.git', commit_sha: 'a'.repeat(40) }; +const base = { operation_id: 'worker-ledger-test', repository }; +const initialized = ledger.initialize(durable, base); +assert.equal(initialized.status, 'passed'); +assert.match(initialized.ledger_id, /^[0-9a-f]{64}$/); + +for (const key of ['GH_TOKEN', 'GITHUB_TOKEN', 'SSH_AUTH_SOCK', 'GIT_ASKPASS', 'SSH_ASKPASS']) { + process.env[key] = 'must-not-be-inherited'; +} +const checkoutRequest = { + ...base, effect_id: 'checkout-effect', worker_home_ledger_ref: initialized.cleanup_ref, +}; +const checkout = ledger.allocate(durable, checkoutRequest, 'checkout', {}); +const replay = ledger.allocate(durable, checkoutRequest, 'checkout', {}); +const readiness = ledger.allocate(durable, { + ...base, effect_id: 'readiness-effect', worker_home_ledger_ref: initialized.cleanup_ref, +}, 'readiness:1:http', {}); +let interruptedWorkerHome = null; +assert.throws(() => ledger.allocate(durable, { + ...base, effect_id: 'interrupted-allocation', worker_home_ledger_ref: initialized.cleanup_ref, +}, 'interrupted-allocation', {}, null, { + afterEnvironmentCreated(environment) { + interruptedWorkerHome = environment.HOME; + throw new Error('simulated crash after worker HOME creation'); + }, +}), /simulated crash/); +assert.equal(fs.existsSync(interruptedWorkerHome), true); +const externalWorkerHome = path.join(root, 'external-worker-home'); +fs.mkdirSync(path.join(externalWorkerHome, '.config', 'gh'), { recursive: true, mode: 0o700 }); +const externalCredential = path.join(externalWorkerHome, '.config', 'gh', 'hosts.yml'); +fs.writeFileSync(externalCredential, 'external-credential-sentinel\n'); +let displacedWorkerHome = null; +assert.throws(() => ledger.allocate(durable, { + ...base, effect_id: 'displaced-allocation', worker_home_ledger_ref: initialized.cleanup_ref, +}, 'displaced-allocation', {}, null, { + afterHomeDirectoryCreated({ home }) { + displacedWorkerHome = `${home}.displaced`; + fs.renameSync(home, displacedWorkerHome); + fs.symlinkSync(externalWorkerHome, home); + }, +}), /worker environment home identity changed after allocation/); +const displacedExpectedHome = displacedWorkerHome.slice(0, -'.displaced'.length); +assert.equal(fs.lstatSync(displacedExpectedHome).isSymbolicLink(), true); +assert.equal(fs.readFileSync(externalCredential, 'utf8'), 'external-credential-sentinel\n'); +assert.equal(checkout.slot_id, replay.slot_id); +assert.equal(checkout.environment.HOME, replay.environment.HOME); +assert.notEqual(checkout.environment.HOME, readiness.environment.HOME); +for (const environment of [checkout.environment, readiness.environment]) { + for (const key of ['GH_TOKEN', 'GITHUB_TOKEN', 'SSH_AUTH_SOCK', 'GIT_ASKPASS', 'SSH_ASKPASS']) { + assert.equal(Object.hasOwn(environment, key), false); + } + assert.equal(common.verifyWorkerEnvironment(environment), true); +} + +const current = read('environment-factory/worker-home-ledger'); +assert.equal(current.entries.length, 4); +assert.equal(current.entries.filter((entry) => entry.state === 'allocated').length, 2); +assert.equal(current.entries.filter((entry) => entry.state === 'reserved').length, 2); + +const resource = read(resourceKey(initialized.cleanup_ref)); +assert.equal(ledger.releaseProven( + durable, checkoutRequest, checkout.slot_id, checkout.lease, +), false); +delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER; +delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256; +const cleanup = ledger.cleanup(durable, root, { + ...base, + artifact_root: '.testing/runs/worker-ledger-test/environment', + cleanup_ref: initialized.cleanup_ref, +}, resource); +assert.equal(cleanup.cleaned, false); +assert.equal(cleanup.remaining_count >= 1, true); +const snapshot = artifacts.get(cleanup.resource_detail_ref.ref).value; +assert.equal(snapshot.schema, 'environment-factory.worker-home-retention.v1'); +assert.equal(snapshot.operation_id, base.operation_id); +assert.equal(snapshot.ledger_id, initialized.ledger_id); +assert.equal(snapshot.remaining_count, cleanup.remaining_count); +assert.equal(stable(snapshot).includes(root), false); +assert.equal(Object.hasOwn(snapshot.entries[0], 'home'), false); + +process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER = cleanupBroker; +process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = sha256(fs.readFileSync(cleanupBroker)); +fs.unlinkSync(displacedExpectedHome); +const recoveredCleanup = ledger.cleanup(durable, root, { + ...base, + artifact_root: '.testing/runs/worker-ledger-test/environment', + cleanup_ref: initialized.cleanup_ref, +}, resource); +assert.equal(recoveredCleanup.cleaned, false); +const recoveredLedger = read('environment-factory/worker-home-ledger'); +assert.equal(recoveredCleanup.remaining_count, 1, stable(recoveredLedger)); +assert.equal(fs.existsSync(interruptedWorkerHome), false); +assert.equal(fs.existsSync(displacedWorkerHome), true); +assert.equal(fs.readFileSync(externalCredential, 'utf8'), 'external-credential-sentinel\n'); +assert.equal(ledger.releaseProven( + durable, checkoutRequest, checkout.slot_id, checkout.lease, +), true); +const workspaceRoot = path.join(root, 'workspaces'); +const workspace = path.join(workspaceRoot, 'run-workspace'); +fs.mkdirSync(workspace, { recursive: true }); +fs.writeFileSync(path.join(workspace, 'owned.txt'), 'owned\n'); +const workspaceIdentity = common.pathIdentity(workspace); +const workspaceRootIdentity = common.pathIdentity(workspaceRoot); +const cleanupCaptureId = sha256('generic-host-workspace-cleanup-recovery'); +const captureRequest = { + schema: 'environment-factory.object-bound-cleanup-request.v1', + operation: 'capture-delete', + capture_id: cleanupCaptureId, + target: workspaceIdentity.realpath, + target_identity: workspaceIdentity, + containment_root: workspaceRootIdentity.realpath, + containment_root_identity: workspaceRootIdentity, +}; +const captureStateRoot = path.join(durable, 'cleanup-captures'); +fs.mkdirSync(captureStateRoot, { recursive: true, mode: 0o700 }); +fs.writeFileSync(path.join(captureStateRoot, `${cleanupCaptureId}.json`), `${stable({ + schema: 'environment-factory.object-bound-cleanup-capture-state.v1', + capture_id: cleanupCaptureId, + target: workspaceIdentity.realpath, + target_identity: workspaceIdentity, + containment_root: workspaceRootIdentity.realpath, + containment_root_identity: workspaceRootIdentity, + state: 'pending', +})}\n`, { flag: 'wx' }); +const brokerResult = spawnSync('/usr/bin/python3', ['-I', cleanupBroker], { + input: `${stable(captureRequest)}\n`, encoding: 'utf8', env: {}, shell: false, +}); +assert.equal(brokerResult.status, 0, brokerResult.stderr || brokerResult.stdout); +assert.equal(fs.existsSync(workspace), false); +assert.equal(common.ownedDirectoryReleaseProven( + workspace, workspaceIdentity, workspaceRoot, cleanupCaptureId, +), false); +assert.equal(releaseWorkspaceResource({ + run_id: base.operation_id, workspace_root: workspace, temp_root: workspaceRoot, +}, { + operation_id: base.operation_id, + path: workspace, + path_identity: workspaceIdentity, + ownership_token: 'owned-workspace-token', + cleanup_capture_id: cleanupCaptureId, +}), true); +assert.equal(JSON.parse(fs.readFileSync( + path.join(captureStateRoot, `${cleanupCaptureId}.json`), 'utf8', +)).state, 'released'); +assert.equal(common.ownedDirectoryReleaseProven( + workspace, workspaceIdentity, workspaceRoot, cleanupCaptureId, +), true); + +fs.rmSync(root, { recursive: true, force: true }); diff --git a/examples/generic-host/tests/worker_home_ledger_test.lua b/examples/generic-host/tests/worker_home_ledger_test.lua new file mode 100644 index 00000000..3b1bb25d --- /dev/null +++ b/examples/generic-host/tests/worker_home_ledger_test.lua @@ -0,0 +1,18 @@ +local t = fkst.test + +return { + test_generic_host_worker_home_ledger = function() + local candidates = { + "examples/generic-host/tests/worker_home_ledger_test.js", + "packages/generic-host/tests/worker_home_ledger_test.js", + } + local script + for _, candidate in ipairs(candidates) do + local handle = io.open(candidate, "rb") + if handle then handle:close(); script = candidate; break end + end + t.is_true(script ~= nil) + local ok, why, code = os.execute("node " .. script) + t.is_true(ok == true or code == 0, tostring(why) .. ":" .. tostring(code)) + end, +} diff --git a/examples/generic-host/tests/workspace_checkout_recovery_test.js b/examples/generic-host/tests/workspace_checkout_recovery_test.js new file mode 100644 index 00000000..43fd6fcc --- /dev/null +++ b/examples/generic-host/tests/workspace_checkout_recovery_test.js @@ -0,0 +1,123 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const store = require('../bin/durable-host-store'); +const { dispatch } = require('../bin/generic-host-runtime'); +const common = require('../../../packages/environment-factory/bin/runtime/common'); + +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'generic-host-workspace-recovery-')); +const durable = path.join(root, 'durable'); +const runtime = path.join(root, 'runtime'); +const projectRoot = path.join(root, 'project'); +const sourceRoot = path.join(root, 'source'); +fs.mkdirSync(projectRoot); +fs.mkdirSync(sourceRoot); +process.env.FKST_DURABLE_ROOT = durable; +process.env.FKST_GENERIC_HOST_DURABLE_ROOT = durable; +process.env.FKST_RUNTIME_ROOT = runtime; +const cleanupBroker = path.resolve( + __dirname, '..', '..', '..', 'packages', 'environment-factory', 'bin', + 'object-bound-cleanup-broker.py', +); +process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER = cleanupBroker; +process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = common.sha256(fs.readFileSync(cleanupBroker)); + +function git(argv, cwd = sourceRoot) { + const result = spawnSync('git', argv, { cwd, encoding: 'utf8', shell: false }); + assert.equal(result.status, 0, result.stderr || result.stdout); + return String(result.stdout || '').trim(); +} + +git(['init', '--quiet']); +git(['config', 'user.email', 'workspace-recovery@example.invalid']); +git(['config', 'user.name', 'Workspace Recovery']); +fs.writeFileSync(path.join(sourceRoot, 'fixture.txt'), 'immutable fixture\n'); +git(['add', 'fixture.txt']); +git(['commit', '--quiet', '-m', 'fixture']); +const repository = { + url: 'https://example.invalid/testing/generic-host-workspace.git', + commit_sha: git(['rev-parse', 'HEAD']), +}; +const boundary = { + schema: 'testing-host.target-execution-boundary.v1', + mode: 'trusted-fixture-exact', + target_class: 'host-owned-exact-trusted-fixture', + repository, + authority: { kind: 'host-policy', ref: 'fixtures/generic-host-workspace-recovery' }, + policy_revision: 'generic-host-workspace-recovery-v1', + human_approval_required: false, + authorization_capability: false, + execution_authorized: false, + promotion_authorized: false, +}; + +function initializeRun(label) { + const runId = `workspace-recovery-${label}`; + const tempRoot = path.join(runtime, runId); + const workspaceRoot = path.join(tempRoot, 'checkout'); + const runRoot = path.join(durable, 'generic-host', runId); + const config = { + schema: 'generic-host.durable-workflow-qa.v1', + project_root: projectRoot, + run_id: runId, + artifact_root: `.testing/runs/${runId}`, + temp_root: tempRoot, + workspace_root: workspaceRoot, + source_root: sourceRoot, + commit_sha: repository.commit_sha, + repository, + profile: { repository, working_directory: '.' }, + target_execution_boundary: boundary, + command_environment: {}, + }; + const stored = store.execute({ + root: runRoot, operation: 'record-immutable', key: 'generic-host/config', value: config, + }); + assert.equal(stored.written, true); + const base = { + operation_id: runId, + repository, + artifact_root: config.artifact_root, + runtime_config_ref: { kind: 'artifact', ref: '.testing/host/generic-host-runtime.json' }, + timeout_seconds: 20, + }; + const ledger = dispatch('initialize-worker-home-ledger', { + ...base, effect_id: `${runId}/worker-home-ledger`, + }, projectRoot); + return { + config, + payload: { + ...base, + effect_id: `${runId}/checkout`, + worker_home_ledger_ref: ledger.cleanup_ref, + working_directory: '.', + }, + }; +} + +try { + const interruptions = [ + ['afterWorkspaceDirectoryCreated', 'directory-created'], + ['afterWorkspaceResourceRegistered', 'allocation-registered'], + ['afterSuccessfulCheckout', 'checkout-succeeded'], + ['afterFinalWorkspaceResourceRegistered', 'resource-registered'], + ]; + for (const [hookName, label] of interruptions) { + const { config, payload } = initializeRun(label); + assert.throws(() => dispatch('checkout', payload, projectRoot, { + [hookName]() { throw new Error(`simulated checkout interruption: ${label}`); }, + }), new RegExp(`simulated checkout interruption: ${label}`)); + const recovered = dispatch('checkout', payload, projectRoot); + assert.equal(recovered.status, 'passed'); + assert.equal(recovered.resolved_commit, repository.commit_sha); + assert.deepEqual(dispatch('checkout', payload, projectRoot), recovered); + assert.equal(git(['rev-parse', 'HEAD'], config.workspace_root), repository.commit_sha); + assert.equal(git(['status', '--porcelain', '--untracked-files=no'], config.workspace_root), ''); + } +} finally { + fs.rmSync(root, { recursive: true, force: true }); +} diff --git a/examples/generic-host/tests/workspace_checkout_recovery_test.lua b/examples/generic-host/tests/workspace_checkout_recovery_test.lua new file mode 100644 index 00000000..779e216f --- /dev/null +++ b/examples/generic-host/tests/workspace_checkout_recovery_test.lua @@ -0,0 +1,18 @@ +local t = fkst.test + +return { + test_workspace_checkout_recovery = function() + local candidates = { + "examples/generic-host/tests/workspace_checkout_recovery_test.js", + "packages/generic-host/tests/workspace_checkout_recovery_test.js", + } + local command + for _, candidate in ipairs(candidates) do + local file = io.open(candidate, "r") + if file then file:close(); command = candidate; break end + end + if command == nil then error("generic-host workspace recovery test script is unavailable") end + local ok, _, code = os.execute("node " .. command) + t.is_true(ok == true or code == 0) + end, +} diff --git a/libraries/contract/environment_factory.lua b/libraries/contract/environment_factory.lua index cad9d96c..009f5af6 100644 --- a/libraries/contract/environment_factory.lua +++ b/libraries/contract/environment_factory.lua @@ -12,7 +12,9 @@ E.schemas = { interrupt = "environment-factory.interrupt.v1", result = "environment-factory.result.v1", receipt = "environment-factory.receipt.v2", - cleanup_receipt = "environment-factory.cleanup-receipt.v1", + cleanup_receipt = "environment-factory.cleanup-receipt.v2", + cleanup_receipt_v1 = "environment-factory.cleanup-receipt.v1", + worker_home_retention = "environment-factory.worker-home-retention.v1", state = "environment-factory.operation-state.v1", start_binding = "environment-factory.start-binding.v1", readiness_correlation = "environment-factory.browser-readiness-correlation.v1", @@ -588,8 +590,9 @@ local receipt_fields = { function E.validate_cleanup_receipt(value) only_fields(value, cleanup_receipt_fields, "cleanup-receipt") - if value.schema ~= E.schemas.cleanup_receipt then - fail("unknown-schema", "expected " .. E.schemas.cleanup_receipt) + local is_v2 = value.schema == E.schemas.cleanup_receipt + if not is_v2 and value.schema ~= E.schemas.cleanup_receipt_v1 then + fail("unknown-schema", "expected a supported cleanup receipt schema") end require_id(value.operation_id, "operation_id") if value.status ~= "complete" and value.status ~= "incomplete" then @@ -635,11 +638,17 @@ function E.validate_cleanup_receipt(value) end local remaining = {} for index, resource in ipairs(value.remaining_resources) do - only_fields(resource, { + local allowed = { resource_id = true, resource_kind = true, cleanup_ref = true, - }, "remaining-resource") + } + if is_v2 then + allowed.resource_detail_ref = true + allowed.resource_detail_sha256 = true + allowed.remaining_count = true + end + only_fields(resource, allowed, "remaining-resource") local resource_id = require_id(resource.resource_id, "remaining_resources[" .. index .. "].resource_id") if remaining[resource_id] or attempted[resource_id] ~= true or cleaned[resource_id] == true then fail("invalid-remaining-resource", resource_id) @@ -647,6 +656,20 @@ function E.validate_cleanup_receipt(value) remaining[resource_id] = true require_id(resource.resource_kind, "remaining_resources[" .. index .. "].resource_kind") validate_ref(resource.cleanup_ref, "remaining_resources[" .. index .. "].cleanup_ref") + if resource.resource_kind == "worker-home-ledger" then + if not is_v2 or resource.resource_detail_ref == nil then + fail("missing-retention-detail", resource_id) + end + validate_artifact_ref(resource.resource_detail_ref, + "remaining_resources[" .. index .. "].resource_detail_ref") + require_digest(resource.resource_detail_sha256, + "remaining_resources[" .. index .. "].resource_detail_sha256") + require_integer(resource.remaining_count, + "remaining_resources[" .. index .. "].remaining_count", 1, 256) + elseif is_v2 and (resource.resource_detail_ref ~= nil + or resource.resource_detail_sha256 ~= nil or resource.remaining_count ~= nil) then + fail("unexpected-retention-detail", resource_id) + end end for resource_id, _ in pairs(attempted) do if cleaned[resource_id] == true then @@ -669,6 +692,52 @@ function E.validate_cleanup_receipt(value) return value end +function E.validate_worker_home_retention(value) + only_fields(value, { + schema = true, operation_id = true, ledger_id = true, repository = true, + remaining_count = true, entries = true, + }, "worker-home-retention") + if value.schema ~= E.schemas.worker_home_retention then + fail("unknown-schema", "expected " .. E.schemas.worker_home_retention) + end + require_id(value.operation_id, "operation_id") + require_digest(value.ledger_id, "ledger_id") + validate_repository(value.repository) + require_integer(value.remaining_count, "remaining_count", 1, 256) + if not dense_list(value.entries, 256, true) or #value.entries ~= value.remaining_count then + fail("malformed-worker-home-retention", "entries must match remaining_count") + end + local slots = {} + for index, entry in ipairs(value.entries) do + only_fields(entry, { + slot_id = true, lease_id = true, effect_id = true, purpose = true, + generation = true, identity_sha256 = true, marker_sha256 = true, + state = true, reason = true, + }, "worker-home-retention-entry") + require_digest(entry.slot_id, "entries[" .. index .. "].slot_id") + if slots[entry.slot_id] then fail("duplicate-worker-home-slot", entry.slot_id) end + slots[entry.slot_id] = true + if type(entry.lease_id) ~= "string" or entry.lease_id:match("^[0-9a-f]+$") == nil + or #entry.lease_id ~= 32 then + fail("malformed-worker-home-retention", "lease_id must be lowercase 32-hex") + end + require_id(entry.effect_id, "entries[" .. index .. "].effect_id") + require_id(entry.purpose, "entries[" .. index .. "].purpose") + require_integer(entry.generation, "entries[" .. index .. "].generation", 1, 256) + if entry.identity_sha256 ~= nil then + require_digest(entry.identity_sha256, "entries[" .. index .. "].identity_sha256") + end + if entry.marker_sha256 ~= nil then + require_digest(entry.marker_sha256, "entries[" .. index .. "].marker_sha256") + end + if entry.state ~= "reserved" and entry.state ~= "allocated" and entry.state ~= "retained" then + fail("malformed-worker-home-retention", "entry state is invalid") + end + require_bounded(entry.reason, "entries[" .. index .. "].reason", max_string) + end + return value +end + function E.validate_receipt(value) only_fields(value, receipt_fields, "receipt") if value.schema ~= E.schemas.receipt then fail("unknown-schema", "expected " .. E.schemas.receipt) end diff --git a/libraries/contract/execution_authorization_lineage.lua b/libraries/contract/execution_authorization_lineage.lua index 2921e711..81ca3320 100644 --- a/libraries/contract/execution_authorization_lineage.lua +++ b/libraries/contract/execution_authorization_lineage.lua @@ -26,7 +26,8 @@ M.paths = { local common_fields = { schema = true, status = true, receipt_id = true, repository = true, run_id = true, trace_id = true, dedup_key = true, recorded_at = true, source_max_uses = true, - evidence_role = true, authorization_capability = true, reusable = true, + evidence_role = true, human_approval_required = true, authorization_capability = true, + execution_authorized = true, promotion_authorized = true, reusable = true, } local function fail(classification, message) @@ -171,7 +172,9 @@ local function validate_common(value, schema, status, context) timestamp(value.recorded_at, context .. ".recorded_at") repository(value.repository, context .. ".repository") if value.source_max_uses ~= 1 or value.evidence_role ~= "audit-only" - or value.authorization_capability ~= false or value.reusable ~= false then + or value.human_approval_required ~= false + or value.authorization_capability ~= false or value.execution_authorized ~= false + or value.promotion_authorized ~= false or value.reusable ~= false then fail("capability-confusion", context .. " must remain non-reusable audit evidence") end end @@ -372,12 +375,15 @@ function M.validate_lineage_index(value, artifacts, expected) only_fields(value, { schema = true, status = true, repository = true, run_id = true, trace_id = true, dedup_key = true, recorded_at = true, receipts = true, lineage_complete = true, - source_max_uses = true, evidence_role = true, authorization_capability = true, - reusable = true, + source_max_uses = true, evidence_role = true, human_approval_required = true, + authorization_capability = true, execution_authorized = true, + promotion_authorized = true, reusable = true, }, context) if value.schema ~= M.schemas.lineage_index or value.status ~= "complete" or value.lineage_complete ~= true or value.evidence_role ~= "audit-only" + or value.human_approval_required ~= false or value.source_max_uses ~= 1 or value.authorization_capability ~= false + or value.execution_authorized ~= false or value.promotion_authorized ~= false or value.reusable ~= false then fail("capability-confusion", context .. " must be complete non-reusable audit evidence") end diff --git a/libraries/contract/structured_execution.lua b/libraries/contract/structured_execution.lua index 073d6ce4..9dd43714 100644 --- a/libraries/contract/structured_execution.lua +++ b/libraries/contract/structured_execution.lua @@ -1,4 +1,6 @@ local error_facts = require("contract.error_facts") +local canonical_json = require("contract.canonical_json") +local sha256 = require("contract.sha256") local strings = require("contract.strings") local time = require("contract.time") @@ -430,6 +432,19 @@ function M.validate_grant(value, now) return value end +function M.validate_grant_authorization_window(preauthorization, grant, now) + M.validate_preauthorization(preauthorization, now) + M.validate_grant(grant, now) + local parent_issued = time.iso_timestamp_epoch_seconds(preauthorization.issued_at) + local parent_expires = time.iso_timestamp_epoch_seconds(preauthorization.expires_at) + local grant_issued = time.iso_timestamp_epoch_seconds(grant.issued_at) + local grant_expires = time.iso_timestamp_epoch_seconds(grant.expires_at) + if grant_issued < parent_issued or grant_expires > parent_expires then + fail("authorization-window-escalation", "grant validity must be contained by its parent preauthorization") + end + return grant +end + function M.validate_cli_action_envelope(value) only_fields(value, { schema = true, effect_kind = true, capability = true, profile_ref = true, @@ -507,9 +522,7 @@ function M.validate_http_action_envelope(value) fail("malformed-envelope", "HTTP action identity, target, expiry, attempt, or fence is invalid") end validate_case(value.case, {}, false) - if value.case.kind ~= "http" or value.case.skip_reason ~= nil then - fail("unsupported-effect", "the action envelope must contain one executable HTTP case") - end + if value.case.kind ~= "http" or value.case.skip_reason ~= nil then fail("unsupported-effect", "the action envelope must contain one executable HTTP case") end only_fields(value.resource_bounds, { output_bytes = true }, "resource-bounds") if type(value.resource_bounds.output_bytes) ~= "number" or value.resource_bounds.output_bytes ~= math.floor(value.resource_bounds.output_bytes) @@ -558,7 +571,8 @@ function M.validate_effect_authorization_receipt(value, envelope, now) end if envelope ~= nil then M.validate_action_envelope(envelope) - if value.fence_id ~= envelope.fence_id or value.trace_id ~= envelope.trace_id + if value.envelope_sha256 ~= sha256.hex(canonical_json.encode(envelope)) + or value.fence_id ~= envelope.fence_id or value.trace_id ~= envelope.trace_id or value.dedup_key ~= envelope.dedup_key or value.expires_at ~= envelope.expires_at then fail("foreign-receipt", "authorization receipt differs from the action envelope") end @@ -646,7 +660,7 @@ function M.derive_grant(preauthorization, preauthorization_sha256, plan, plan_sh trace_id = request.trace_id, dedup_key = request.dedup_key, } - return M.validate_grant(grant, values.now) + return M.validate_grant_authorization_window(preauthorization, grant, values.now) end local plan_request_fields = { diff --git a/libraries/testing_runtime/authorization_lineage_projection.lua b/libraries/testing_runtime/authorization_lineage_projection.lua index f2e940a3..b4664ca4 100644 --- a/libraries/testing_runtime/authorization_lineage_projection.lua +++ b/libraries/testing_runtime/authorization_lineage_projection.lua @@ -108,7 +108,10 @@ function Projector:write_receipt(name, receipt_id, recorded_at, fields, expected recorded_at = recorded_at, source_max_uses = 1, evidence_role = "audit-only", + human_approval_required = false, authorization_capability = false, + execution_authorized = false, + promotion_authorized = false, reusable = false, } for key, item in pairs(fields) do @@ -158,7 +161,10 @@ function Projector:write_index(recorded_at, artifacts, expected) lineage_complete = true, source_max_uses = 1, evidence_role = "audit-only", + human_approval_required = false, authorization_capability = false, + execution_authorized = false, + promotion_authorized = false, reusable = false, } lineage.validate_lineage_index(value, artifacts, expected) diff --git a/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js b/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js index b82e0b73..44c2db30 100644 --- a/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js +++ b/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js @@ -10,10 +10,8 @@ const { artifactPath, boundedText, isSafeArtifactPath, - minimalEnvironment, parseArgs, readJson, - releaseWorkerEnvironment, sha256, stableStringify, validateArgv, @@ -21,6 +19,10 @@ const { writeJsonAtomic, writeJsonImmutable, } = require('../../../packages/environment-factory/bin/runtime/common'); +const { + allocateDurableWorkerEnvironment, + recordWorkerEnvironmentRelease, +} = require('../../../packages/environment-factory/bin/runtime/worker-home-resource'); const { validateTargetExecutionBoundary, } = require('../../../packages/environment-factory/bin/runtime/target-execution-boundary'); @@ -649,18 +651,78 @@ function receiptTag(config, receipt) { function authorizationReceipt(config, envelope, decision, reasonCode, inputs, now) { const envelopeSha256 = sha256(stableStringify(envelope)); + const boundedIdentity = (value, fallback) => typeof value === 'string' + && value.length > 0 && value.length <= 180 ? value : fallback; + const requestedExpiry = Date.parse(envelope.expires_at); + const expiry = Number.isFinite(requestedExpiry) && requestedExpiry > now.getTime() + ? envelope.expires_at : utcTimestamp(new Date(now.getTime() + 1000)); const receipt = { schema: 'testing-effect-authorization-receipt.v1', decision, reason_code: reasonCode, - receipt_id: `${envelope.effect_kind || 'invalid'}-effect-${envelopeSha256.slice(0, 40)}`, + receipt_id: `${['cli', 'http'].includes(envelope.effect_kind) ? envelope.effect_kind : 'invalid'}-effect-${envelopeSha256.slice(0, 40)}`, envelope_sha256: envelopeSha256, evaluated_input_digests: inputs, - issued_at: utcTimestamp(now), expires_at: envelope.expires_at, - fence_id: envelope.fence_id, trace_id: envelope.trace_id, dedup_key: envelope.dedup_key, + issued_at: utcTimestamp(now), expires_at: expiry, + fence_id: boundedIdentity(envelope.fence_id, 'invalid-fence'), + trace_id: boundedIdentity(envelope.trace_id, 'invalid-trace'), + dedup_key: boundedIdentity(envelope.dedup_key, 'invalid-dedup'), }; receipt.auth_tag = receiptTag(config, receipt); return receipt; } +function persistedDeniedReceipt(config, envelope, reason, inputs, now) { + const expected = authorizationReceipt(config, envelope, 'deny', reason, inputs, now); + const target = authorizationPath(expected.receipt_id); + const release = acquireLock(`${target}.lock`); + try { + const current = fs.existsSync(target) ? readJson(target) : null; + const stored = current && (current.status === 'denied' + ? current.receipt : current.denial_receipt); + if (stored) { + if (stored.schema !== 'testing-effect-authorization-receipt.v1' + || stored.decision !== 'deny' || stored.reason_code !== reason + || stored.receipt_id !== expected.receipt_id + || stored.envelope_sha256 !== expected.envelope_sha256 + || stableStringify(stored.evaluated_input_digests) !== stableStringify(inputs) + || stored.auth_tag !== receiptTag(config, stored)) { + throw new Error('durable effect denial receipt binding differs'); + } + return stored; + } + if (current && !['issued', 'consumed'].includes(current.status)) { + throw new Error('durable effect authorization record is malformed'); + } + if (current) { + writeJsonAtomic(target, { ...current, denial_receipt: expected }); + } else { + writeJsonAtomic(target, { status: 'denied', receipt: expected }); + } + return expected; + } finally { + release(); + } +} + +function reusableIssuedReceipt(config, stored, expected, now) { + if (!stored || stored.status !== 'issued' || !stored.receipt) return null; + const receipt = stored.receipt; + const expectedBinding = { ...expected }; + const observedBinding = { ...receipt }; + delete expectedBinding.issued_at; + delete expectedBinding.auth_tag; + delete observedBinding.issued_at; + delete observedBinding.auth_tag; + const issuedAt = Date.parse(receipt.issued_at); + const expiresAt = Date.parse(receipt.expires_at); + if (stableStringify(observedBinding) !== stableStringify(expectedBinding) + || receipt.auth_tag !== receiptTag(config, receipt) + || !Number.isFinite(issuedAt) || !Number.isFinite(expiresAt) + || issuedAt > now.getTime() || now.getTime() >= expiresAt) { + return null; + } + return receipt; +} + function validateEnvelope(envelope, expectedKind) { const commonFields = [ 'schema', 'effect_kind', 'capability', 'profile_ref', 'profile_artifact_sha256', 'profile_sha256', @@ -780,6 +842,25 @@ function evaluateEnvelope(config, envelope, now, expectedKind) { || profile.value.resource_budgets.output_bytes !== envelope.resource_bounds.output_bytes) { throw new Error('project profile policy denies CLI effect'); } + const parentIssuedAt = Date.parse(preauthorization.value.issued_at); + const parentExpiresAt = Date.parse(preauthorization.value.expires_at); + const grantIssuedAt = Date.parse(grant.value.issued_at); + const grantExpiresAt = Date.parse(grant.value.expires_at); + if (![parentIssuedAt, parentExpiresAt, grantIssuedAt, grantExpiresAt].every(Number.isFinite) + || parentExpiresAt <= parentIssuedAt || grantExpiresAt <= grantIssuedAt) { + throw new Error('execution authorization validity window is malformed'); + } + const nowMs = now.getTime(); + if (nowMs < parentIssuedAt || nowMs >= parentExpiresAt) { + throw new Error('parent preauthorization is expired or not yet valid'); + } + if (grantIssuedAt < parentIssuedAt || grantExpiresAt > parentExpiresAt) { + throw new Error('execution grant validity exceeds parent preauthorization'); + } + if (nowMs < grantIssuedAt || nowMs >= grantExpiresAt + || nowMs >= Date.parse(envelope.expires_at)) { + throw new Error('execution grant or action envelope is expired or not yet valid'); + } const planned = (plan.value.cases || []).find((item) => item.case_id === envelope.case.case_id); if (!planned || stableStringify(planned) !== stableStringify(envelope.case)) { throw new Error('approved plan scope differs'); @@ -799,7 +880,7 @@ function evaluateEnvelope(config, envelope, now, expectedKind) { && sameAuthority(entry.authority, grant.value.authority) && entry.policy_revision === grant.value.policy_revision && samePointer(entry.evidence_ref, grant.value.evidence_ref)); - if (!attested || now >= new Date(grant.value.expires_at) || now >= new Date(envelope.expires_at)) { + if (!attested) { throw new Error('execution grant is unauthenticated or expired'); } const replay = readReplay(config, grant.value.grant_id); @@ -818,6 +899,7 @@ function authorizeEffect(payload, expectedKind) { preauthorization: '0'.repeat(64), environment_receipt: '0'.repeat(64), plan: '0'.repeat(64), grant: '0'.repeat(64), }; + let replayDenied = false; try { validateTargetExecutionBoundary(config.target_execution_boundary, envelope.repository, { runtimeConfigRef: payload.runtime_config_ref, @@ -830,15 +912,21 @@ function authorizeEffect(payload, expectedKind) { try { if (fs.existsSync(target)) { const current = readJson(target); - if (current.status !== 'issued' || stableStringify(current.receipt) !== stableStringify(receipt)) { - return authorizationReceipt(config, envelope, 'deny', 'replayed', inputs, now); - } - return current.receipt; + const reusable = reusableIssuedReceipt(config, current, receipt, now); + if (reusable) return reusable; + replayDenied = true; + } else { + writeJsonAtomic(target, { status: 'issued', receipt }); + return receipt; } - writeJsonAtomic(target, { status: 'issued', receipt }); - return receipt; } finally { release(); } + if (replayDenied) { + return persistedDeniedReceipt(config, envelope, 'replayed', inputs, now); + } + throw new Error('effect authorization replay state is unavailable'); } catch (error) { + if (replayDenied && String(error && error.message || error) + .includes('durable effect denial receipt')) throw error; const message = String(error && error.message || error); const reason = message.includes('digest') ? 'digest-mismatch' : message.includes('expired') ? 'expired' @@ -847,13 +935,7 @@ function authorizeEffect(payload, expectedKind) { : message.includes('profile policy') ? 'profile-policy-denied' : message.includes('fields') || message.includes('malformed') ? 'malformed-envelope' : 'foreign-binding'; - const safeEnvelope = { - expires_at: Number.isFinite(Date.parse(envelope.expires_at)) ? envelope.expires_at : utcTimestamp(new Date(now.getTime() + 1000)), - fence_id: typeof envelope.fence_id === 'string' ? envelope.fence_id : 'invalid-fence', - trace_id: typeof envelope.trace_id === 'string' ? envelope.trace_id : 'invalid-trace', - dedup_key: typeof envelope.dedup_key === 'string' ? envelope.dedup_key : 'invalid-dedup', - }; - return authorizationReceipt(config, safeEnvelope, 'deny', reason, inputs, now); + return persistedDeniedReceipt(config, envelope, reason, inputs, now); } } @@ -906,24 +988,22 @@ async function consumeAuthorizedEffect(payload, expectedKind, execute) { async function execArgv(payload) { return consumeAuthorizedEffect(payload, 'cli', async (envelope, config) => { const workspace = resolveWorkspace({ + effect_id: `structured-workspace:${payload.authorization_receipt.receipt_id}`, operation_id: envelope.operation_id, repository: envelope.repository, environment_receipt_sha256: envelope.environment_receipt_sha256, workspace_ref: envelope.workspace_ref, require_clean: true, }); - const environment = minimalEnvironment(config.command_environment || {}, { - schema: 'testing-runtime.structured-cli-isolation.v1', + const allocation = allocateDurableWorkerEnvironment({ + effect_id: `structured-cli:${payload.authorization_receipt.receipt_id}`, operation_id: envelope.operation_id, - run_id: envelope.run_id, repository: envelope.repository, - case_id: envelope.case.case_id, - attempt: envelope.attempt, - purpose: 'structured-cli', - }); + }, `structured-cli:${envelope.case.case_id}`, config.command_environment || {}); try { - verifyWorkerEnvironment(environment); + verifyWorkerEnvironment(allocation.environment); const result = await runMeasuredCommand(validateArgv(envelope.case.argv), { cwd: workspace.cwd, - env: environment, + cwdIdentity: workspace.cwdIdentity, + env: allocation.environment, timeoutMs: envelope.case.timeout_seconds * 1000, outputBytes: Math.min(boundedOutput(config), envelope.resource_bounds.output_bytes), }); @@ -938,7 +1018,7 @@ async function execArgv(payload) { } return { exit_code: result.exitCode, stdout: result.stdout, stderr: result.stderr }; } finally { - releaseWorkerEnvironment(environment); + recordWorkerEnvironmentRelease(allocation); } }); } diff --git a/libraries/testing_runtime/tests/structured_execution_runtime_test.js b/libraries/testing_runtime/tests/structured_execution_runtime_test.js index c912fed6..b7655ddf 100644 --- a/libraries/testing_runtime/tests/structured_execution_runtime_test.js +++ b/libraries/testing_runtime/tests/structured_execution_runtime_test.js @@ -34,6 +34,10 @@ function copy(value) { return JSON.parse(JSON.stringify(value)); } +function delay(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + function persistJson(ref, value) { const raw = `${stableStringify(value)}\n`; fs.mkdirSync(path.dirname(ref), { recursive: true }); @@ -54,10 +58,17 @@ async function main() { const ambientKeys = [ 'HOME', 'GH_TOKEN', 'GITHUB_TOKEN', 'SSH_AUTH_SOCK', 'GIT_ASKPASS', 'SSH_ASKPASS', 'GIT_CONFIG_GLOBAL', 'GIT_CONFIG_COUNT', 'GIT_CONFIG_KEY_0', 'GIT_CONFIG_VALUE_0', + 'FKST_OBJECT_BOUND_CLEANUP_BROKER', 'FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256', ]; const previousAmbient = Object.fromEntries(ambientKeys.map((key) => [key, process.env[key]])); process.env.FKST_DURABLE_ROOT = path.join(temp, 'durable'); process.env.FKST_RUNTIME_ROOT = path.join(temp, 'runtime'); + const cleanupBroker = path.resolve( + __dirname, '..', '..', '..', 'packages', 'environment-factory', 'bin', + 'object-bound-cleanup-broker.py', + ); + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER = cleanupBroker; + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = sha256(fs.readFileSync(cleanupBroker)); const runId = `structured-runtime-${process.pid}`; const artifactRoot = `.testing/runs/${runId}/execution`; const environmentArtifactRoot = `.testing/runs/${runId}/environment`; @@ -67,6 +78,7 @@ async function main() { const source = path.join(temp, 'source'); let workspace; let checkout; + let workerHomeLedgerRef; const operationId = `${runId}-operation`; let workspaceRef; const repository = { url: 'https://example.invalid/testing/runtime.git', commit_sha: '' }; @@ -121,14 +133,27 @@ async function main() { repository, authority: { kind: 'host-policy', ref: 'fixtures/structured-runtime-target-boundary' }, policy_revision: 'structured-runtime-target-boundary-v1', + human_approval_required: false, authorization_capability: false, execution_authorized: false, + promotion_authorized: false, }, })}\n`); + const workerHomeLedger = await environmentDispatch('initialize-worker-home-ledger', { + effect_id: `${operationId}/worker-home-ledger`, + operation_id: operationId, + repository, + artifact_root: environmentArtifactRoot, + runtime_config_ref: { kind: 'artifact', ref: environmentConfigRef }, + timeout_seconds: 20, + }); + assert.strictEqual(workerHomeLedger.status, 'passed'); + workerHomeLedgerRef = workerHomeLedger.cleanup_ref; checkout = await environmentDispatch('checkout', { effect_id: `${operationId}/checkout`, operation_id: operationId, repository, + worker_home_ledger_ref: workerHomeLedgerRef, working_directory: '.', artifact_root: environmentArtifactRoot, runtime_config_ref: { kind: 'artifact', ref: environmentConfigRef }, @@ -169,7 +194,10 @@ async function main() { const traceId = `${runId}-trace`; const dedupKey = `${runId}-dedup`; + const issuedAt = new Date(Date.now() - 60 * 60 * 1000).toISOString(); + const expiredAt = new Date(Date.now() - 30 * 60 * 1000).toISOString(); const expiresAt = new Date(Date.now() + 60 * 60 * 1000).toISOString(); + const overlongExpiresAt = new Date(Date.now() + 2 * 60 * 60 * 1000).toISOString(); const authorityRoot = `.testing/runs/${runId}/authorization`; const writeAuthority = (name, value) => { const ref = `${authorityRoot}/${name}.json`; @@ -193,6 +221,7 @@ async function main() { cli: [{ argv_prefix: [process.execPath] }], http: [{ origin: `http://127.0.0.1:${address.port}`, methods: ['GET'], path_prefixes: ['/health'] }], }, + issued_at: issuedAt, expires_at: expiresAt, trace_id: traceId, dedup_key: dedupKey, }); const environment = writeAuthority('environment', { @@ -233,7 +262,8 @@ async function main() { origin: `http://127.0.0.1:${address.port}`, methods: ['GET'], path_prefixes: ['/health'], }], authority, policy_revision: 'runtime-test-policy-v1', evidence_ref: evidenceRef, - expires_at: expiresAt, max_uses: 1, trace_id: traceId, dedup_key: dedupKey, + issued_at: issuedAt, expires_at: expiresAt, + max_uses: 1, trace_id: traceId, dedup_key: dedupKey, }); grantSha256 = grant.digest; const httpPlan = writeAuthority('http-plan', { @@ -248,7 +278,21 @@ async function main() { cli_capabilities: [], http_capabilities: [{ origin: `http://127.0.0.1:${address.port}`, methods: ['GET'], path_prefixes: ['/health'], }], authority, policy_revision: 'runtime-test-policy-v1', evidence_ref: evidenceRef, - expires_at: expiresAt, max_uses: 1, trace_id: traceId, dedup_key: dedupKey, + issued_at: issuedAt, expires_at: expiresAt, + max_uses: 1, trace_id: traceId, dedup_key: dedupKey, + }); + const expiredPreauthorization = writeAuthority('expired-preauthorization', { + ...preauthorization.value, expires_at: expiredAt, + }); + const expiredParentGrant = writeAuthority('expired-parent-grant', { + ...grant.value, + grant_id: `${runId}-expired-parent-effect-grant`, + parent_authorization_sha256: expiredPreauthorization.digest, + }); + const overlongGrant = writeAuthority('overlong-grant', { + ...grant.value, + grant_id: `${runId}-overlong-effect-grant`, + expires_at: overlongExpiresAt, }); fs.mkdirSync(path.dirname(configRef), { recursive: true }); @@ -265,14 +309,20 @@ async function main() { repository, authority: { kind: 'host-policy', ref: 'fixtures/structured-runtime-target-boundary' }, policy_revision: 'structured-runtime-target-boundary-v1', + human_approval_required: false, authorization_capability: false, execution_authorized: false, + promotion_authorized: false, }, grant_attestations: [ { grant_sha256: grantSha256, authority, policy_revision: 'runtime-test-policy-v1', evidence_ref: evidenceRef }, { grant_sha256: httpGrant.digest, authority, policy_revision: 'runtime-test-policy-v1', evidence_ref: evidenceRef }, + { grant_sha256: expiredParentGrant.digest, authority, + policy_revision: 'runtime-test-policy-v1', evidence_ref: evidenceRef }, + { grant_sha256: overlongGrant.digest, authority, + policy_revision: 'runtime-test-policy-v1', evidence_ref: evidenceRef }, ], })}\n`); @@ -329,8 +379,13 @@ async function main() { }); assert.strictEqual(authorization.decision, 'allow'); assert.match(authorization.issued_at, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/); + await delay(1_100); + const redeliveredAuthorization = await dispatch('authorize-cli-effect', { + ...common, action_envelope: actionEnvelope, + }); + assert.deepStrictEqual(redeliveredAuthorization, authorization); const cli = await dispatch('exec-argv', { - ...common, action_envelope: actionEnvelope, authorization_receipt: authorization, + ...common, action_envelope: actionEnvelope, authorization_receipt: redeliveredAuthorization, }); assert.strictEqual(cli.exit_code, 0); assert.strictEqual(cli.stdout, fs.realpathSync(workspace)); @@ -338,15 +393,82 @@ async function main() { await assert.rejects(() => dispatch('exec-argv', { ...common, action_envelope: actionEnvelope, authorization_receipt: authorization, }), /replayed or is unavailable/); + const replayDenial = await dispatch('authorize-cli-effect', { + ...common, action_envelope: actionEnvelope, + }); + assert.strictEqual(replayDenial.decision, 'deny'); + assert.strictEqual(replayDenial.reason_code, 'replayed'); + await delay(1_100); + assert.deepStrictEqual(await dispatch('authorize-cli-effect', { + ...common, action_envelope: actionEnvelope, + }), replayDenial); const foreignEnvelope = { ...actionEnvelope, plan_sha256: '0'.repeat(64) }; const denied = await dispatch('authorize-cli-effect', { ...common, action_envelope: foreignEnvelope, }); assert.strictEqual(denied.decision, 'deny'); + assert.strictEqual(denied.envelope_sha256, sha256(stableStringify(foreignEnvelope))); + await delay(1_100); + const redeliveredDenial = await dispatch('authorize-cli-effect', { + ...common, action_envelope: foreignEnvelope, + }); + assert.deepStrictEqual(redeliveredDenial, denied); await assert.rejects(() => dispatch('exec-argv', { ...common, action_envelope: foreignEnvelope, authorization_receipt: denied, }), /missing, denied, malformed, expired, or foreign/); + const expiredParentClaim = await dispatch('replay-guard', { + ...common, grant_id: expiredParentGrant.value.grant_id, + grant_sha256: expiredParentGrant.digest, + parent_authorization_sha256: expiredPreauthorization.digest, + plan_sha256: plan.digest, + environment_receipt_sha256: environment.digest, + }); + const expiredParentEnvelope = { + ...actionEnvelope, + preauthorization_ref: expiredPreauthorization.ref, + preauthorization_sha256: expiredPreauthorization.digest, + grant_ref: expiredParentGrant.ref, + grant_sha256: expiredParentGrant.digest, + fence_id: expiredParentClaim.claim_id, + }; + const expiredParentAuthorization = await dispatch('authorize-cli-effect', { + ...common, action_envelope: expiredParentEnvelope, + }); + assert.strictEqual(expiredParentAuthorization.decision, 'deny'); + assert.strictEqual(expiredParentAuthorization.reason_code, 'expired'); + assert.strictEqual(expiredParentAuthorization.envelope_sha256, + sha256(stableStringify(expiredParentEnvelope))); + await delay(1_100); + assert.deepStrictEqual(await dispatch('authorize-cli-effect', { + ...common, action_envelope: expiredParentEnvelope, + }), expiredParentAuthorization); + + const overlongClaim = await dispatch('replay-guard', { + ...common, grant_id: overlongGrant.value.grant_id, + grant_sha256: overlongGrant.digest, + parent_authorization_sha256: preauthorization.digest, + plan_sha256: plan.digest, + environment_receipt_sha256: environment.digest, + }); + const overlongEnvelope = { + ...actionEnvelope, + grant_ref: overlongGrant.ref, + grant_sha256: overlongGrant.digest, + fence_id: overlongClaim.claim_id, + }; + const overlongAuthorization = await dispatch('authorize-cli-effect', { + ...common, action_envelope: overlongEnvelope, + }); + assert.strictEqual(overlongAuthorization.decision, 'deny'); + assert.strictEqual(overlongAuthorization.reason_code, 'foreign-binding'); + assert.strictEqual(overlongAuthorization.envelope_sha256, + sha256(stableStringify(overlongEnvelope))); + await delay(1_100); + assert.deepStrictEqual(await dispatch('authorize-cli-effect', { + ...common, action_envelope: overlongEnvelope, + }), overlongAuthorization); + const httpEffectClaim = await dispatch('replay-guard', { ...common, grant_id: httpGrant.value.grant_id, grant_sha256: httpGrant.digest, parent_authorization_sha256: preauthorization.digest, plan_sha256: httpPlan.digest, @@ -723,18 +845,23 @@ async function main() { ...common, result_ref: resultRef, result_sha256: replay.result_sha256, }), /digest differs/); + delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER; + delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256; const cleaned = await environmentDispatch('cleanup', { effect_id: `${operationId}/cleanup/workspace`, operation_id: operationId, artifact_root: environmentArtifactRoot, cleanup_ref: checkout.cleanup_ref, + worker_home_ledger_ref: workerHomeLedgerRef, workspace_ref: workspaceRef, working_directory: '.', runtime_config_ref: { kind: 'artifact', ref: environmentConfigRef }, timeout_seconds: 10, }); - assert.strictEqual(cleaned.status, 'cleaned'); - assert.strictEqual(fs.existsSync(workspace), false); + assert.strictEqual(cleaned.status, 'blocked'); + assert.strictEqual(fs.existsSync(workspace), true); + const cleanupDiagnostic = JSON.parse(fs.readFileSync(cleaned.diagnostic_ref.ref, 'utf8')); + assert.strictEqual(cleanupDiagnostic.reason, 'OBJECT_BOUND_CLEANUP_UNAVAILABLE'); await assert.rejects(() => dispatch('exec-argv', { ...common, }), /fields are invalid|malformed/); diff --git a/libraries/testing_runtime/workflow_qa_host_adapter.lua b/libraries/testing_runtime/workflow_qa_host_adapter.lua index cafe421f..d8c86984 100644 --- a/libraries/testing_runtime/workflow_qa_host_adapter.lua +++ b/libraries/testing_runtime/workflow_qa_host_adapter.lua @@ -85,17 +85,13 @@ function M.new(options) end if grant.plan_sha256 ~= request.plan_sha256 or grant.environment_receipt_sha256 ~= request.environment_receipt_sha256 - or grant.parent_authorization_sha256 ~= request.preauthorization_sha256 then - fail("structured grant binding differs from request") - end + or grant.parent_authorization_sha256 ~= request.preauthorization_sha256 then fail("structured grant binding differs from request") end return grant end browser_control.validate_grant(grant) if grant.reviewed_plan_sha256 ~= request.plan_sha256 or grant.environment_receipt_sha256 ~= request.environment_receipt_sha256 - or grant.parent_authorization_sha256 ~= request.preauthorization_sha256 then - fail("browser grant binding differs from request") - end + or grant.parent_authorization_sha256 ~= request.preauthorization_sha256 then fail("browser grant binding differs from request") end return grant end diff --git a/packages/environment-factory/bin/environment-factory-runtime.js b/packages/environment-factory/bin/environment-factory-runtime.js index 73f136bc..fe1d8059 100644 --- a/packages/environment-factory/bin/environment-factory-runtime.js +++ b/packages/environment-factory/bin/environment-factory-runtime.js @@ -9,9 +9,10 @@ const path = require('path'); const { loadAuthorizationBundle } = require('./runtime/authorization'); const { createBudgetRuntime } = require('./runtime/budgets'); const { - DEFAULT_OUTPUT_BYTES, acquireLock, artifactPath, boundedText, commandResult, - isSafeArtifactPath, minimalEnvironment, parseArgs, readJson, releaseWorkerEnvironment, - releaseWorkerEnvironmentLease, processStartIdentity, runtimeConfig, sameArray, sha256, stableStringify, validateArgv, + DEFAULT_OUTPUT_BYTES, OBJECT_BOUND_CLEANUP_UNAVAILABLE, acquireLock, artifactPath, boundedText, commandResult, + isSafeArtifactPath, parseArgs, readJson, + processStartIdentity, pathEntryExists, pathIdentity, removeOwnedDirectory, + requireOwnedDirectory, runtimeConfig, sameArray, samePathIdentity, sha256, stableStringify, validateArgv, verifyWorkerEnvironment, verifyWorkerEnvironmentLease, writeJsonAtomic, writeJsonImmutable, } = require('./runtime/common'); const { validateTargetExecutionBoundary } = require('./runtime/target-execution-boundary'); @@ -21,23 +22,21 @@ const { listenersOwnedByProcessGroup, processGroupUsage } = require('./runtime/p const { startOrRecoverSupervisedProcess } = require('./runtime/supervised-process'); const { loadState, saveState } = require('./runtime/state'); const { readResource: readWorkspaceResource, resolveWorkspace } = require('./runtime/workspace'); +const { prepareReservedWorkspace } = require('./runtime/workspace-reservation'); +const { + allocateDurableWorkerEnvironment, + cleanupWorkerHomeLedger, + initializeWorkerHomeLedger, + recordPersistedWorkerEnvironmentRelease, + verifyPersistedWorkerEnvironment, + withDurableWorkerEnvironment, +} = require('./runtime/worker-home-resource'); function durableRoot() { return path.resolve(process.env.FKST_DURABLE_ROOT || path.join('.testing', 'durable')); } function executionRoot() { return path.resolve(process.env.FKST_RUNTIME_ROOT || path.join('.testing', 'runtime')); } -function workerIsolationIdentity(payload, purpose) { - const identity = { - schema: 'environment-factory.worker-isolation-identity.v1', - operation_id: payload.operation_id, - effect_id: payload.effect_id || purpose, - purpose, - }; - if (payload.repository) identity.repository = payload.repository; - return identity; -} - function targetExecutionConfig(payload, repository = payload && payload.repository) { const config = runtimeConfig(payload); validateTargetExecutionBoundary(config.target_execution_boundary, repository, { @@ -139,10 +138,6 @@ function writeDiagnostic(request, label, value) { return ref; } -function workspaceCwd(request) { - return resolveWorkspace(request).cwd; -} - function relativeWorkspacePath(cwd, value, field) { if (typeof value !== 'string' || value === '' || path.isAbsolute(value) || value.includes('\\') || value.split('/').some((segment) => segment === '' || segment === '.' || segment === '..')) { @@ -212,84 +207,153 @@ function remainingTimeoutMs(deadline) { return Math.max(1, deadline - Date.now()); } -async function checkout(payload) { +function workspaceReservation(payload, repository, workspacePath, workspaceContainer, resourceRef) { + const workspaceRef = { kind: 'workspace', ref: resourceRef }; + const staticBinding = { + schema: 'environment-factory.resource.v1', + reservation_schema: 'environment-factory.workspace-reservation.v1', + reservation_id: resourceRef, + kind: 'workspace', + operation_id: payload.operation_id, + ref: resourceRef, + path: workspacePath, + containment_root: workspaceContainer, + containment_root_identity: pathIdentity(workspaceContainer), + workspace_ref: workspaceRef, + repository: { url: repository.url, commit_sha: repository.commit_sha }, + working_directory: payload.working_directory, + cleaned: false, + }; + const current = readIfExists(resourcePath(resourceRef)); + if (current) { + const observedBinding = { ...current }; + delete observedBinding.ownership_token; + delete observedBinding.cleanup_capture_id; + delete observedBinding.path_identity; + delete observedBinding.reservation_state; + if (stableStringify(observedBinding) !== stableStringify(staticBinding) + || typeof current.ownership_token !== 'string' + || !/^[0-9a-f]{64}$/.test(current.ownership_token) + || typeof current.cleanup_capture_id !== 'string' + || !/^[0-9a-f]{64}$/.test(current.cleanup_capture_id)) { + throw new Error('workspace reservation binding differs'); + } + if (!['reserved', 'allocated'].includes(current.reservation_state) + || (current.reservation_state === 'allocated' && !current.path_identity) + || (current.reservation_state === 'reserved' && current.path_identity !== null)) { + throw new Error('workspace reservation state is malformed'); + } + return { record: current, created: false }; + } + if (pathEntryExists(workspacePath)) { + throw new Error('workspace path already exists before durable reservation'); + } + const record = { + ...staticBinding, + ownership_token: crypto.randomBytes(32).toString('hex'), + cleanup_capture_id: crypto.randomBytes(32).toString('hex'), + path_identity: null, + reservation_state: 'reserved', + }; + writeJsonAtomic(resourcePath(resourceRef), record); + return { record, created: true }; +} + +async function checkout(payload, hooks = {}) { return withEffect(payload, async () => { const config = targetExecutionConfig(payload); const deadline = effectDeadline(payload); const repository = payload.repository || {}; const source = config.repository_mirrors && config.repository_mirrors[repository.url]; if (typeof source !== 'string' || source === '') throw new Error('repository mirror is unavailable'); + const sourcePath = path.resolve(source); if (!/^[0-9a-f]{40}$/.test(String(repository.commit_sha || ''))) throw new Error('exact commit is required'); - const workspacePath = path.join(executionRoot(), 'environment-factory', sha256(payload.operation_id).slice(0, 24), 'checkout'); + fs.mkdirSync(executionRoot(), { recursive: true, mode: 0o700 }); + const runtimeRoot = requireOwnedDirectory(executionRoot(), { privateDirectory: true }); + const factoryRoot = requireOwnedDirectory( + path.join(runtimeRoot, 'environment-factory'), { privateDirectory: true }, + ); + const workspaceContainer = requireOwnedDirectory( + path.join(factoryRoot, sha256(payload.operation_id).slice(0, 24)), + { privateDirectory: true }, + ); + const workspacePath = path.join(workspaceContainer, 'checkout'); const resourceRef = `environment-factory-resource-${sha256(`workspace\0${payload.operation_id}\0${workspacePath}`).slice(0, 32)}`; const workspaceRef = { kind: 'workspace', ref: resourceRef }; const cleanupRef = { kind: 'resource-cleanup', ref: resourceRef }; - fs.rmSync(workspacePath, { recursive: true, force: true }); - fs.mkdirSync(workspacePath, { recursive: true }); - writeJsonAtomic(resourcePath(resourceRef), { - schema: 'environment-factory.resource.v1', - kind: 'workspace', - operation_id: payload.operation_id, - ref: resourceRef, - path: workspacePath, - workspace_ref: workspaceRef, - repository: { - url: repository.url, - commit_sha: repository.commit_sha, + fs.mkdirSync(workspaceContainer, { recursive: true }); + const reservation = workspaceReservation( + payload, repository, workspacePath, workspaceContainer, resourceRef, + ); + const workspaceIdentity = prepareReservedWorkspace( + reservation.record, reservation.record.path_identity, { + reservationWasCreated: reservation.created, + hooks, + persistIdentity(identity) { + reservation.record = { + ...reservation.record, + path_identity: identity, + reservation_state: 'allocated', + }; + writeJsonAtomic(resourcePath(resourceRef), reservation.record); + }, }, - working_directory: payload.working_directory, - cleaned: false, - }); - const commandEnvironment = minimalEnvironment( - config.command_environment || {}, workerIsolationIdentity(payload, 'checkout'), ); - try { - const clone = await executeBudgetedCommand(payload, - ['git', 'clone', '--quiet', '--no-checkout', source, workspacePath], { + const result = await withDurableWorkerEnvironment( + payload, 'checkout', config.command_environment || {}, async (commandEnvironment) => { + const clone = await executeBudgetedCommand(payload, + ['git', 'clone', '--quiet', '--no-checkout', sourcePath, '.'], { + cwd: workspacePath, + cwdIdentity: workspaceIdentity, + env: commandEnvironment, + timeoutMs: remainingTimeoutMs(deadline), + workspacePath, + }); + if (clone.reason !== null) { + return { + status: 'blocked', + workspace_ref: workspaceRef, + cleanup_ref: cleanupRef, + diagnostic_ref: writeDiagnostic(payload, 'checkout', { + status: 'blocked', reason: clone.reason, stderr: boundedText(clone.stderr, payload.output_bytes), + }), + }; + } + const checkoutResult = await executeBudgetedCommand(payload, + ['git', 'checkout', '--quiet', '--detach', repository.commit_sha], { + cwd: workspacePath, + cwdIdentity: workspaceIdentity, + env: commandEnvironment, + timeoutMs: remainingTimeoutMs(deadline), + workspacePath, + }); + const resolved = await executeBudgetedCommand(payload, ['git', 'rev-parse', 'HEAD'], { + cwd: workspacePath, + cwdIdentity: workspaceIdentity, env: commandEnvironment, timeoutMs: remainingTimeoutMs(deadline), workspacePath, }); - if (clone.reason !== null) { + const resolvedCommit = String(resolved.stdout || '').trim(); + const passed = checkoutResult.reason === null && resolved.reason === null + && resolvedCommit === repository.commit_sha; return { - status: 'blocked', + status: passed ? 'passed' : 'blocked', + resolved_commit: resolvedCommit || null, workspace_ref: workspaceRef, cleanup_ref: cleanupRef, diagnostic_ref: writeDiagnostic(payload, 'checkout', { - status: 'blocked', reason: clone.reason, stderr: boundedText(clone.stderr, payload.output_bytes), + status: passed ? 'passed' : 'blocked', + resolved_commit: resolvedCommit, + stderr: boundedText(`${checkoutResult.stderr} ${resolved.stderr}`, 1024), }), }; - } - const checkoutResult = await executeBudgetedCommand(payload, - ['git', 'checkout', '--quiet', '--detach', repository.commit_sha], { - cwd: workspacePath, - env: commandEnvironment, - timeoutMs: remainingTimeoutMs(deadline), - workspacePath, - }); - const resolved = await executeBudgetedCommand(payload, ['git', 'rev-parse', 'HEAD'], { - cwd: workspacePath, - env: commandEnvironment, - timeoutMs: remainingTimeoutMs(deadline), - workspacePath, - }); - const resolvedCommit = String(resolved.stdout || '').trim(); - const passed = checkoutResult.reason === null && resolved.reason === null - && resolvedCommit === repository.commit_sha; - return { - status: passed ? 'passed' : 'blocked', - resolved_commit: resolvedCommit || null, - workspace_ref: workspaceRef, - cleanup_ref: cleanupRef, - diagnostic_ref: writeDiagnostic(payload, 'checkout', { - status: passed ? 'passed' : 'blocked', - resolved_commit: resolvedCommit, - stderr: boundedText(`${checkoutResult.stderr} ${resolved.stderr}`, 1024), - }), - }; - } finally { - releaseWorkerEnvironment(commandEnvironment); + }, + ); + if (result.status === 'passed' && typeof hooks.afterSuccessfulCheckout === 'function') { + hooks.afterSuccessfulCheckout({ reservation: { ...reservation.record }, result: { ...result } }); } + return result; }); } @@ -422,19 +486,14 @@ async function runArgvEffect(payload) { const workspaceResource = readWorkspaceResource(payload.workspace_ref); const config = targetExecutionConfig(payload, workspaceResource.repository); const argv = validateArgv(payload.argv); - const cwd = workspaceCwd(payload); + const workspace = resolveWorkspace(payload); + const cwd = workspace.cwd; const timeoutMs = Math.max(1, Number(payload.timeout_seconds) || 1) * 1000; const outputBytes = Number(payload.output_bytes) || DEFAULT_OUTPUT_BYTES; if (payload.mode !== 'oneshot' && payload.mode !== 'supervised') { throw new Error('unsupported argv mode'); } - let baseEnv = null; - let retainWorkerHome = false; - try { if (payload.mode === 'oneshot') { - baseEnv = minimalEnvironment( - config.command_environment || {}, workerIsolationIdentity(payload, 'run-argv'), - ); const frozen = payload.requires_frozen_dependencies === true; let frozenProof = null; if (frozen) { @@ -451,50 +510,57 @@ async function runArgvEffect(payload) { }; } } - const result = await executeBudgetedCommand(payload, argv, { - cwd, - env: baseEnv, - timeoutMs, - workspacePath: resolveWorkspace(payload).workspaceRoot, - }); - let frozenEnforced = false; - if (frozen && result.reason === null) { - try { - const afterLock = sha256(fs.readFileSync(frozenProof.lockfilePath)); - const afterManifest = sha256(fs.readFileSync(frozenProof.manifestPath)); - frozenProof.lockfileSha256After = afterLock; - frozenProof.manifestSha256After = afterManifest; - frozenEnforced = afterLock === frozenProof.lockfileSha256 - && afterManifest === frozenProof.manifestSha256; - } catch (_error) { - frozenEnforced = false; - } - if (!frozenEnforced) result.reason = 'frozen-lockfile-changed'; - } - return { - status: result.reason === null ? 'passed' : 'blocked', - ...(frozen ? { frozen_dependencies_enforced: frozenEnforced } : {}), - diagnostic_ref: writeDiagnostic(payload, sha256(payload.effect_id).slice(0, 16), { - schema: 'environment-factory.command-diagnostic.v1', - status: result.reason === null ? 'passed' : 'blocked', - reason: result.reason, - exit_code: result.exitCode, - stderr: boundedText(result.stderr, outputBytes), - cpu_millis: result.cpuMillis ?? null, - max_rss_bytes: result.maxRssBytes ?? null, - max_processes: result.maxProcesses ?? null, - frozen_dependency_proof: frozenProof && { - policy_revision: frozenProof.revision, - manifest_sha256_before: frozenProof.manifestSha256, - manifest_sha256_after: frozenProof.manifestSha256After ?? null, - lockfile_sha256_before: frozenProof.lockfileSha256, - lockfile_sha256_after: frozenProof.lockfileSha256After ?? null, - }, - }), - }; + const workerRequest = { ...payload, repository: workspaceResource.repository }; + return withDurableWorkerEnvironment( + workerRequest, `oneshot:${payload.effect_id}`, config.command_environment || {}, + async (environment) => { + const result = await executeBudgetedCommand(payload, argv, { + cwd, + cwdIdentity: workspace.cwdIdentity, + env: environment, + timeoutMs, + workspacePath: workspace.workspaceRoot, + }); + let frozenEnforced = false; + if (frozen && result.reason === null) { + try { + const afterLock = sha256(fs.readFileSync(frozenProof.lockfilePath)); + const afterManifest = sha256(fs.readFileSync(frozenProof.manifestPath)); + frozenProof.lockfileSha256After = afterLock; + frozenProof.manifestSha256After = afterManifest; + frozenEnforced = afterLock === frozenProof.lockfileSha256 + && afterManifest === frozenProof.manifestSha256; + } catch (_error) { + frozenEnforced = false; + } + if (!frozenEnforced) result.reason = 'frozen-lockfile-changed'; + } + return { + status: result.reason === null ? 'passed' : 'blocked', + ...(frozen ? { frozen_dependencies_enforced: frozenEnforced } : {}), + diagnostic_ref: writeDiagnostic(payload, sha256(payload.effect_id).slice(0, 16), { + schema: 'environment-factory.command-diagnostic.v1', + status: result.reason === null ? 'passed' : 'blocked', + reason: result.reason, + exit_code: result.exitCode, + stderr: boundedText(result.stderr, outputBytes), + cpu_millis: result.cpuMillis ?? null, + max_rss_bytes: result.maxRssBytes ?? null, + max_processes: result.maxProcesses ?? null, + frozen_dependency_proof: frozenProof && { + policy_revision: frozenProof.revision, + manifest_sha256_before: frozenProof.manifestSha256, + manifest_sha256_after: frozenProof.manifestSha256After ?? null, + lockfile_sha256_before: frozenProof.lockfileSha256, + lockfile_sha256_after: frozenProof.lockfileSha256After ?? null, + }, + }), + }; + }, + ); } resourceBudgets(payload); - const workspacePath = resolveWorkspace(payload).workspaceRoot; + const workspacePath = workspace.workspaceRoot; const before = enforceCurrentBudgets(payload, workspacePath); if (!before.passed) { return { @@ -510,7 +576,9 @@ async function runArgvEffect(payload) { const cleanupRef = { kind: 'resource-cleanup', ref: resourceRef }; const processResourcePath = resourcePath(resourceRef); const startupClaimPath = `${processResourcePath}.startup`; - const startupClaimExisted = fs.existsSync(startupClaimPath); + const workerRequest = { ...payload, repository: workspaceResource.repository }; + const workerPurpose = `supervised:${payload.effect_id}`; + const commandEnvironment = config.command_environment || {}; const starting = { schema: 'environment-factory.resource.v1', kind: 'process', @@ -532,7 +600,8 @@ async function runArgvEffect(payload) { if (existing) { const volatile = new Set([ 'startup_state', 'startup_token_sha256', 'pid', 'pgid', - 'process_start_identity', 'worker_environment_lease', + 'process_start_identity', 'worker_environment_lease', 'worker_home_ledger_ref', + 'worker_home_slot_id', ]); const existingBinding = Object.fromEntries( Object.entries(existing).filter(([key]) => !volatile.has(key)), @@ -540,7 +609,14 @@ async function runArgvEffect(payload) { if (stableStringify(existingBinding) !== stableStringify(starting)) { throw new Error('supervised process resource binding differs'); } - verifyWorkerEnvironmentLease(existing.worker_environment_lease); + if (stableStringify(existing.worker_home_ledger_ref) !== stableStringify(payload.worker_home_ledger_ref) + || typeof existing.worker_home_slot_id !== 'string') { + throw new Error('supervised worker-home ledger binding differs'); + } + verifyPersistedWorkerEnvironment( + workerRequest, workerPurpose, commandEnvironment, + existing.worker_home_slot_id, existing.worker_environment_lease, + ); const state = processGroupState(existing); const running = state.supported && state.alive && state.foreign !== true; return { @@ -559,19 +635,24 @@ async function runArgvEffect(payload) { const listenerNames = inheritedListenerNames(payload); const inheritedStdio = listenerNames.map((_name, index) => 3 + index); const createSupervisedEnvironment = (reservation) => { - const environment = minimalEnvironment( - config.command_environment || {}, workerIsolationIdentity(payload, 'run-argv'), + const allocation = allocateDurableWorkerEnvironment( + workerRequest, + workerPurpose, + commandEnvironment, reservation && reservation.reservation_id, ); + const environment = allocation.environment; environment.FKST_LISTEN_FDS = String(listenerNames.length); environment.FKST_LISTEN_FDNAMES = listenerNames.join(':'); - return environment; + return { environment, worker_home_slot_id: allocation.slot_id }; }; + let launchResource = null; try { const launch = startOrRecoverSupervisedProcess({ claimPath: startupClaimPath, argv, cwd, + cwdIdentity: workspace.cwdIdentity, createEnvironment: createSupervisedEnvironment, inheritedStdio, binding: starting, @@ -579,8 +660,16 @@ async function runArgvEffect(payload) { if (launch.interrupted || !launch.resource) { throw new Error('supervised process startup was interrupted before registration'); } - retainWorkerHome = launch.environment_retained === true; const resource = launch.resource; + launchResource = resource; + resource.worker_home_ledger_ref = payload.worker_home_ledger_ref; + verifyPersistedWorkerEnvironment( + workerRequest, + workerPurpose, + commandEnvironment, + resource.worker_home_slot_id, + resource.worker_environment_lease, + ); writeJsonAtomic(processResourcePath, resource); const live = processGroupState(resource); const running = launch.state === 'running' && live.supported && live.alive && live.foreign !== true; @@ -606,7 +695,7 @@ async function runArgvEffect(payload) { }), }; } catch (error) { - if (!startupClaimExisted && fs.existsSync(startupClaimPath)) retainWorkerHome = true; + if (launchResource) await stopProcess(launchResource, Date.now() + 500); return { status: 'blocked', early_exit: true, @@ -617,9 +706,6 @@ async function runArgvEffect(payload) { }), }; } - } finally { - if (!retainWorkerHome) releaseWorkerEnvironment(baseEnv); - } }); } @@ -653,27 +739,21 @@ function checkHttp(check, timeoutMs) { }); } -async function readinessCheck(check, payload, deadline, config) { +async function readinessCheck(check, payload, deadline, environment) { const remaining = deadline - Date.now(); if (remaining <= 0) return false; if (check.type === 'tcp') return checkTcp(check, Math.min(500, remaining)); if (check.type === 'http') return checkHttp(check, Math.min(500, remaining)); if (check.type === 'argv') { const workspace = resolveWorkspace(payload); - const environment = minimalEnvironment( - config.command_environment || {}, workerIsolationIdentity(payload, 'readiness-argv'), - ); - try { - const result = await executeBudgetedCommand(payload, validateArgv(check.argv), { - cwd: workspace.cwd, - env: environment, - timeoutMs: Math.min(2_000, remaining), - workspacePath: workspace.workspaceRoot, - }); - return result.reason === null && Date.now() <= deadline; - } finally { - releaseWorkerEnvironment(environment); - } + const result = await executeBudgetedCommand(payload, validateArgv(check.argv), { + cwd: workspace.cwd, + cwdIdentity: workspace.cwdIdentity, + env: environment, + timeoutMs: Math.min(2_000, remaining), + workspacePath: workspace.workspaceRoot, + }); + return result.reason === null && Date.now() <= deadline; } return false; } @@ -701,44 +781,54 @@ async function waitReadiness(payload) { throw new Error('readiness process ownership binding is invalid'); } const workspacePath = resolveWorkspace(payload).workspaceRoot; - const deadline = Date.now() + Math.max(1, Number(payload.timeout_seconds) || 1) * 1000; - let { attempts, probes, reason } = initialReadinessState(budgets, checks); - let ready = false; - while (reason === null && Date.now() < deadline) { - attempts += 1; - const budget = enforceCurrentBudgets(payload, workspacePath); - if (!budget.passed) { reason = budget.reason; break; } - const results = []; - for (const check of checks) { - if (check.type === 'tcp' || check.type === 'http') { - const consumption = consumeNetworkRequest(payload); - if (!consumption.accepted) { - reason = 'network-request-budget-exceeded'; - break; + const usesArgv = checks.some((check) => check.type === 'argv'); + const executeChecks = async (environment) => { + const deadline = Date.now() + Math.max(1, Number(payload.timeout_seconds) || 1) * 1000; + let { attempts, probes, reason } = initialReadinessState(budgets, checks); + let ready = false; + while (reason === null && Date.now() < deadline) { + attempts += 1; + const budget = enforceCurrentBudgets(payload, workspacePath); + if (!budget.passed) { reason = budget.reason; break; } + const results = []; + for (const check of checks) { + if (check.type === 'tcp' || check.type === 'http') { + const consumption = consumeNetworkRequest(payload); + if (!consumption.accepted) { + reason = 'network-request-budget-exceeded'; + break; + } + probes += 1; } - probes += 1; + results.push(await readinessCheck(check, payload, deadline, environment)); + if (Date.now() > deadline) { reason = 'readiness-timeout'; break; } } - results.push(await readinessCheck(check, payload, deadline, config)); - if (Date.now() > deadline) { reason = 'readiness-timeout'; break; } - } - if (reason === 'network-request-budget-exceeded' || reason === 'readiness-timeout') break; - if (results.length > 0 && results.every(Boolean)) { - const ownership = listenersOwnedByProcessGroup(ports, processResource.pgid || processResource.pid); - if (!ownership.supported) { reason = 'listener-ownership-unavailable'; break; } - if (!ownership.owned) { reason = ownership.reason; break; } - ready = true; - reason = null; - break; + if (reason === 'network-request-budget-exceeded' || reason === 'readiness-timeout') break; + if (results.length > 0 && results.every(Boolean)) { + const ownership = listenersOwnedByProcessGroup(ports, processResource.pgid || processResource.pid); + if (!ownership.supported) { reason = 'listener-ownership-unavailable'; break; } + if (!ownership.owned) { reason = ownership.reason; break; } + ready = true; + reason = null; + break; + } + await new Promise((resolve) => setTimeout(resolve, 25)); } - await new Promise((resolve) => setTimeout(resolve, 25)); - } - if (!ready && reason === null) reason = 'readiness-timeout'; - return { - status: ready ? 'ready' : 'blocked', - diagnostic_ref: writeDiagnostic(payload, sha256(payload.effect_id).slice(0, 16), { - status: ready ? 'ready' : 'blocked', attempts, network_requests: probes, reason, - }), + if (!ready && reason === null) reason = 'readiness-timeout'; + return { + status: ready ? 'ready' : 'blocked', + diagnostic_ref: writeDiagnostic(payload, sha256(payload.effect_id).slice(0, 16), { + status: ready ? 'ready' : 'blocked', attempts, network_requests: probes, reason, + }), + }; }; + if (!usesArgv) return executeChecks(undefined); + return withDurableWorkerEnvironment( + { ...payload, repository: workspaceResource.repository }, + `readiness:${payload.effect_id}`, + config.command_environment || {}, + executeChecks, + ); }); } @@ -794,14 +884,14 @@ async function stopProcess(resource, deadline) { } } -function resourceIsReleased(resource) { +function resourceIsReleased(resource, workerHomeReleased = false) { if (resource.kind === 'process') { const state = processGroupState(resource); const homeReleased = !resource.worker_environment_lease - || !fs.existsSync(resource.worker_environment_lease.home); + || workerHomeReleased === true || resource.cleaned === true; return state.supported && (!state.alive || state.foreign === true) && homeReleased; } - if (resource.kind === 'workspace') return typeof resource.path === 'string' && !fs.existsSync(resource.path); + if (resource.kind === 'workspace') return typeof resource.path === 'string' && !pathEntryExists(resource.path); if (resource.kind === 'ports') { return exactPortList(resource.ports).every((item) => { const owner = readIfExists(ledgerPath('ports', item.port)); @@ -845,35 +935,63 @@ async function cleanup(payload) { const deadline = effectDeadline(payload); let cleaned = true; + let blockedReason = null; if (resource.kind === 'process') { + let workerHomeReleased = !resource.worker_environment_lease; const config = targetExecutionConfig(payload, resource.repository); if (Array.isArray(payload.argv) && payload.argv.length > 0) { - const environment = minimalEnvironment( - config.command_environment || {}, workerIsolationIdentity(payload, 'cleanup-argv'), - ); - try { - verifyWorkerEnvironment(environment); - const result = await runMeasuredCommand(validateArgv(payload.argv), { - cwd: workspaceCwd(payload), + const cleanupWorkspace = resolveWorkspace(payload); + const result = await withDurableWorkerEnvironment( + { ...payload, repository: resource.repository }, + `cleanup-argv:${resource.ref}`, + config.command_environment || {}, + async (environment) => runMeasuredCommand(validateArgv(payload.argv), { + cwd: cleanupWorkspace.cwd, + cwdIdentity: cleanupWorkspace.cwdIdentity, env: environment, timeoutMs: remainingTimeoutMs(deadline), outputBytes: payload.output_bytes, - }); - cleaned = result.exitCode === 0 && result.timedOut !== true && result.outputExceeded !== true; - } finally { - releaseWorkerEnvironment(environment); - } + }), + ); + cleaned = result.exitCode === 0 && result.timedOut !== true && result.outputExceeded !== true; } await stopProcess(resource, deadline); const stopped = processGroupState(resource); if (stopped.supported && (!stopped.alive || stopped.foreign === true) && resource.worker_environment_lease) { - releaseWorkerEnvironmentLease(resource.worker_environment_lease); + workerHomeReleased = recordPersistedWorkerEnvironmentRelease({ + ...payload, + repository: resource.repository, + worker_home_ledger_ref: resource.worker_home_ledger_ref, + }, resource.worker_home_slot_id, resource.worker_environment_lease, 'supervised-process-stopped'); + if (!workerHomeReleased) { + blockedReason = OBJECT_BOUND_CLEANUP_UNAVAILABLE; + } + } + cleaned = cleaned && resourceIsReleased(resource, workerHomeReleased); + if (!cleaned && blockedReason === null && resource.worker_environment_lease + && !workerHomeReleased) { + blockedReason = OBJECT_BOUND_CLEANUP_UNAVAILABLE; } - cleaned = cleaned && resourceIsReleased(resource); } else if (resource.kind === 'workspace') { - fs.rmSync(resource.path, { recursive: true, force: true }); - cleaned = !fs.existsSync(resource.path); + if (typeof resource.containment_root !== 'string' + || !samePathIdentity(pathIdentity(resource.containment_root), resource.containment_root_identity)) { + throw new Error('workspace containment root identity changed'); + } + cleaned = removeOwnedDirectory( + resource.path, resource.path_identity, resource.containment_root, + resource.cleanup_capture_id, + ); + if (!cleaned) blockedReason = OBJECT_BOUND_CLEANUP_UNAVAILABLE; + } else if (resource.kind === 'worker-home-ledger') { + const result = cleanupWorkerHomeLedger(payload, resource); + cleaned = result.cleaned; + if (!cleaned) { + blockedReason = OBJECT_BOUND_CLEANUP_UNAVAILABLE; + resource.resource_detail_ref = result.resource_detail_ref; + resource.resource_detail_sha256 = result.resource_detail_sha256; + resource.remaining_count = result.remaining_count; + } } else if (resource.kind === 'ports') { const lockPath = path.join(durableRoot(), 'environment-factory', 'claim.lock'); fs.mkdirSync(path.dirname(lockPath), { recursive: true }); @@ -895,7 +1013,16 @@ async function cleanup(payload) { writeJsonAtomic(resourcePath(resource.ref), resource); const outcome = { status: cleaned ? 'cleaned' : 'blocked', - diagnostic_ref: writeDiagnostic(payload, sha256(payload.effect_id).slice(0, 16), { status: cleaned ? 'cleaned' : 'blocked', resource_kind: resource.kind }), + ...(resource.resource_detail_ref ? { + resource_detail_ref: resource.resource_detail_ref, + resource_detail_sha256: resource.resource_detail_sha256, + remaining_count: resource.remaining_count, + } : {}), + diagnostic_ref: writeDiagnostic(payload, sha256(payload.effect_id).slice(0, 16), { + status: cleaned ? 'cleaned' : 'blocked', + resource_kind: resource.kind, + reason: cleaned ? null : (blockedReason || 'resource-release-incomplete'), + }), }; if (cleaned) { writeJsonAtomic(effectPath(payload.effect_id), { @@ -963,6 +1090,9 @@ async function dispatch(name, payload) { } if (name === 'plan-listener-claim') return listenerClaimPlan(payload); if (name === 'authorize-claim-ports') return claimPorts(payload); + if (name === 'initialize-worker-home-ledger') { + return withEffect(payload, async () => initializeWorkerHomeLedger(payload)); + } if (name === 'checkout') return checkout(payload); if (name === 'remaining-budget') return remainingBudget(payload); if (name === 'create-readiness-attempt') return createReadinessAttempt(payload); @@ -995,10 +1125,12 @@ if (require.main === module) { } module.exports = { + checkout, commandResult, dispatch, initialReadinessState, isSafeArtifactPath, + resourceIsReleased, sha256, stableStringify, }; diff --git a/packages/environment-factory/bin/object-bound-cleanup-broker.py b/packages/environment-factory/bin/object-bound-cleanup-broker.py new file mode 100644 index 00000000..290879cf --- /dev/null +++ b/packages/environment-factory/bin/object-bound-cleanup-broker.py @@ -0,0 +1,999 @@ +#!/usr/bin/env python3 +import ctypes +import json +import os +import secrets +import stat +import sys + + +MAX_REQUEST_BYTES = 32 * 1024 +MAX_ENTRIES, MAX_DEPTH = 100_000, 128 +REQUEST_SCHEMA = "environment-factory.object-bound-cleanup-request.v1" +RECEIPT_SCHEMA = "environment-factory.object-bound-cleanup-receipt.v1" +ALLOCATION_REQUEST_SCHEMA = "environment-factory.object-bound-directory-allocation-request.v1" +ALLOCATION_RECEIPT_SCHEMA = "environment-factory.object-bound-directory-allocation-receipt.v1" +MARKER_RETIREMENT_REQUEST_SCHEMA = "environment-factory.object-bound-marker-retirement-request.v1" +MARKER_RETIREMENT_RECEIPT_SCHEMA = "environment-factory.object-bound-marker-retirement-receipt.v1" +ALLOCATION_STAGE_PREFIX = ".fkst-object-allocation-" +QUARANTINE_PREFIX = ".fkst-object-cleanup-" +QUARANTINE_SLOT, CAPTURE_MARKER = "entry", "capture.json" +CAPTURE_CLEANED_MARKER = "captured-cleaned" +CAPTURE_RETAINED_MARKER = "captured-retained" +CAPTURE_FINALIZED_MARKER = "finalized" +CAPTURE_MARKER_SCHEMA = "environment-factory.object-bound-cleanup-capture.v1" +RENAME_NOREPLACE, RENAME_EXCL = 1, 0x00000004 + + +class CleanupBlocked(Exception): + pass + + +def same_object(left, right): + return left.st_dev == right.st_dev and left.st_ino == right.st_ino + + +def identity_matches(observed, expected): + return ( + isinstance(expected, dict) + and str(observed.st_dev) == expected.get("device") + and str(observed.st_ino) == expected.get("inode") + ) + + +def valid_digest(value): + return isinstance(value, str) and len(value) == 64 and all( + char in "0123456789abcdef" for char in value + ) + + +def valid_identity(value): + return isinstance(value, dict) and set(value) == {"realpath", "device", "inode"} and all( + isinstance(value.get(key), str) and value[key] for key in value + ) + + +def valid_absolute_path(value): + return isinstance(value, str) and os.path.isabs(value) and "\x00" not in value + + +def read_cleanup_request(value): + expected_keys = { + "schema", + "operation", + "capture_id", + "target", + "target_identity", + "containment_root", + "containment_root_identity", + } + if set(value) != expected_keys or value.get("schema") != REQUEST_SCHEMA: + raise CleanupBlocked("request-invalid") + if value.get("operation") not in ("capture-delete", "finalize", "release-proof"): + raise CleanupBlocked("operation-invalid") + if not valid_digest(value.get("capture_id")): + raise CleanupBlocked("capture-id-invalid") + for name in ("target", "containment_root"): + if not valid_absolute_path(value.get(name)): + raise CleanupBlocked("path-invalid") + for name in ("target_identity", "containment_root_identity"): + if not valid_identity(value.get(name)): + raise CleanupBlocked("identity-invalid") + return value + + +def valid_relative_directory(value): + if not isinstance(value, str) or not value or "\x00" in value or os.path.isabs(value): + return False + segments = value.split("/") + return all(segment not in ("", ".", "..") and os.sep not in segment for segment in segments) + + +def read_allocation_request(value): + expected_keys = { + "schema", + "operation", + "allocation_id", + "target", + "containment_root", + "containment_root_identity", + "marker_name", + "marker_body", + "child_directories", + } + if set(value) != expected_keys or value.get("schema") != ALLOCATION_REQUEST_SCHEMA: + raise CleanupBlocked("allocation-request-invalid") + allocation_id = value.get("allocation_id") + if value.get("operation") != "allocate-directory" or not valid_digest(allocation_id): + raise CleanupBlocked("allocation-binding-invalid") + for name in ("target", "containment_root"): + if not valid_absolute_path(value.get(name)): + raise CleanupBlocked("allocation-path-invalid") + identity = value.get("containment_root_identity") + if not valid_identity(identity): + raise CleanupBlocked("allocation-root-identity-invalid") + marker_name = value.get("marker_name") + marker_body = value.get("marker_body") + children = value.get("child_directories") + if ( + not isinstance(marker_name, str) + or marker_name in ("", ".", "..") + or os.sep in marker_name + or "\x00" in marker_name + or not isinstance(marker_body, str) + or not marker_body.endswith("\n") + or len(marker_body.encode("utf-8")) > 16 * 1024 + or not isinstance(children, list) + or len(children) > 16 + or len(set(children)) != len(children) + or not all(valid_relative_directory(item) for item in children) + ): + raise CleanupBlocked("allocation-content-invalid") + return value + + +def read_marker_retirement_request(value): + expected_keys = { + "schema", + "operation", + "allocation_id", + "target", + "target_identity", + "containment_root", + "containment_root_identity", + "marker_name", + "marker_body", + } + if set(value) != expected_keys or value.get("schema") != MARKER_RETIREMENT_REQUEST_SCHEMA: + raise CleanupBlocked("marker-retirement-request-invalid") + allocation_id = value.get("allocation_id") + if value.get("operation") != "retire-marker" or not valid_digest(allocation_id): + raise CleanupBlocked("marker-retirement-binding-invalid") + for name in ("target", "containment_root"): + if not valid_absolute_path(value.get(name)): + raise CleanupBlocked("marker-retirement-path-invalid") + for name in ("target_identity", "containment_root_identity"): + if not valid_identity(value.get(name)): + raise CleanupBlocked("marker-retirement-identity-invalid") + marker_name = value.get("marker_name") + marker_body = value.get("marker_body") + if ( + not isinstance(marker_name, str) + or marker_name in ("", ".", "..") + or os.sep in marker_name + or "\x00" in marker_name + or not isinstance(marker_body, str) + or not marker_body.endswith("\n") + or len(marker_body.encode("utf-8")) > 16 * 1024 + ): + raise CleanupBlocked("marker-retirement-content-invalid") + return value + + +def read_request(): + body = sys.stdin.buffer.read(MAX_REQUEST_BYTES + 1) + if len(body) > MAX_REQUEST_BYTES: + raise CleanupBlocked("request-too-large") + try: + value = json.loads(body) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise CleanupBlocked("request-invalid") from error + if not isinstance(value, dict): + raise CleanupBlocked("request-invalid") + if value.get("schema") == ALLOCATION_REQUEST_SCHEMA: + return read_allocation_request(value) + if value.get("schema") == MARKER_RETIREMENT_REQUEST_SCHEMA: + return read_marker_retirement_request(value) + return read_cleanup_request(value) + + +def open_directory(name, parent_fd=None): + flags = os.O_RDONLY | os.O_DIRECTORY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + return os.open(name, flags, dir_fd=parent_fd) + + +def rename_noreplace(source_name, source_fd, destination_name, destination_fd): + libc = ctypes.CDLL(None, use_errno=True) + encoded_source = os.fsencode(source_name) + encoded_destination = os.fsencode(destination_name) + if sys.platform.startswith("linux"): + primitive = getattr(libc, "renameat2", None) + flags = RENAME_NOREPLACE + elif sys.platform == "darwin": + primitive = getattr(libc, "renameatx_np", None) + flags = RENAME_EXCL + else: + raise CleanupBlocked("atomic-capture-unsupported") + if primitive is None: + raise CleanupBlocked("atomic-capture-unsupported") + primitive.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] + primitive.restype = ctypes.c_int + ctypes.set_errno(0) + if primitive(source_fd, encoded_source, destination_fd, encoded_destination, flags) != 0: + error_number = ctypes.get_errno() + raise OSError(error_number, os.strerror(error_number), source_name) + + +def stat_entry(directory_fd, name): + return os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + + +def create_private_quarantine(parent_fd, filesystem_device): + for _ in range(16): + name = QUARANTINE_PREFIX + secrets.token_hex(16) + try: + os.mkdir(name, mode=0o700, dir_fd=parent_fd) + except FileExistsError: + continue + directory_fd = None + linked = None + try: + linked = stat_entry(parent_fd, name) + directory_fd = open_directory(name, parent_fd) + opened = os.fstat(directory_fd) + if ( + not stat.S_ISDIR(linked.st_mode) + or not same_object(linked, opened) + or opened.st_dev != filesystem_device + or opened.st_uid != os.geteuid() + or opened.st_mode & 0o077 + ): + raise CleanupBlocked("quarantine-invalid") + return name, directory_fd, opened + except Exception: + if directory_fd is not None: + os.close(directory_fd) + try: + current = stat_entry(parent_fd, name) + if linked is not None and same_object(current, linked): + os.rmdir(name, dir_fd=parent_fd) + except OSError: + pass + raise + raise CleanupBlocked("quarantine-allocation-failed") + + +def capture_marker_body(request): + return ( + json.dumps( + { + "schema": CAPTURE_MARKER_SCHEMA, + "capture_id": request["capture_id"], + "target": request["target"], + "target_identity": request["target_identity"], + "containment_root": request["containment_root"], + "containment_root_identity": request["containment_root_identity"], + }, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + + +def write_exclusive_file(directory_fd, name, body): + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(name, flags, 0o600, dir_fd=directory_fd) + try: + offset = 0 + while offset < len(body): + written = os.write(descriptor, body[offset:]) + if written <= 0: + raise CleanupBlocked("capture-marker-write-failed") + offset += written + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def read_bound_file(directory_fd, name, maximum=4096): + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(name, flags, dir_fd=directory_fd) + try: + linked = stat_entry(directory_fd, name) + opened = os.fstat(descriptor) + if ( + not stat.S_ISREG(linked.st_mode) + or not same_object(linked, opened) + or opened.st_uid != os.geteuid() + or opened.st_mode & 0o077 + or opened.st_size > maximum + ): + raise CleanupBlocked("capture-marker-invalid") + chunks = [] + remaining = maximum + 1 + while remaining > 0: + chunk = os.read(descriptor, remaining) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + body = b"".join(chunks) + if len(body) > maximum: + raise CleanupBlocked("capture-marker-invalid") + return body + finally: + os.close(descriptor) + + +def marker_exists(directory_fd, name): + try: + stat_entry(directory_fd, name) + except FileNotFoundError: + return False + return True + + +def capture_quarantine_name(request): + return QUARANTINE_PREFIX + request["capture_id"] + + +def open_capture_quarantine(bound, request, create): + name = capture_quarantine_name(request) + created = False + if create: + try: + os.mkdir(name, mode=0o700, dir_fd=bound["parent_fd"]) + created = True + except FileExistsError: + pass + elif not marker_exists(bound["parent_fd"], name): + return None + if not created and not marker_exists(bound["parent_fd"], name): + if not create: + return None + raise CleanupBlocked("capture-quarantine-missing") + directory_fd = open_directory(name, bound["parent_fd"]) + try: + linked = stat_entry(bound["parent_fd"], name) + opened = os.fstat(directory_fd) + if ( + not stat.S_ISDIR(linked.st_mode) + or not same_object(linked, opened) + or opened.st_dev != bound["root_stat"].st_dev + or opened.st_uid != os.geteuid() + or opened.st_mode & 0o077 + ): + raise CleanupBlocked("capture-quarantine-invalid") + expected_marker = capture_marker_body(request) + if created: + write_exclusive_file(directory_fd, CAPTURE_MARKER, expected_marker) + os.fsync(directory_fd) + if read_bound_file(directory_fd, CAPTURE_MARKER) != expected_marker: + raise CleanupBlocked("capture-marker-binding-mismatch") + return {"name": name, "fd": directory_fd, "stat": opened} + except Exception: + os.close(directory_fd) + if created: + try: + os.rmdir(name, dir_fd=bound["parent_fd"]) + except OSError: + pass + raise + + +def entry_is_absent(directory_fd, name): + try: + stat_entry(directory_fd, name) + except FileNotFoundError: + return True + return False + + +def remove_empty_quarantine(parent_fd, name, directory_fd, expected): + if os.listdir(directory_fd): + return False + try: + current = stat_entry(parent_fd, name) + if not same_object(current, expected): + return False + os.rmdir(name, dir_fd=parent_fd) + return True + except OSError: + return False + + +def restore_captured_entry(source_fd, source_name, quarantine_fd, captured): + current = stat_entry(quarantine_fd, QUARANTINE_SLOT) + if not same_object(current, captured): + raise CleanupBlocked("captured-entry-changed") + try: + rename_noreplace(QUARANTINE_SLOT, quarantine_fd, source_name, source_fd) + except OSError as error: + raise CleanupBlocked("captured-entry-retained") from error + restored = stat_entry(source_fd, source_name) + if not same_object(restored, captured): + raise CleanupBlocked("restored-entry-changed") + + +def capture_observed_entry(source_fd, source_name, observed, quarantine_fd): + rename_noreplace(source_name, source_fd, QUARANTINE_SLOT, quarantine_fd) + captured = stat_entry(quarantine_fd, QUARANTINE_SLOT) + if not same_object(captured, observed): + restore_captured_entry(source_fd, source_name, quarantine_fd, captured) + raise CleanupBlocked("entry-changed-before-capture") + return captured + + +def clear_directory(directory_fd, filesystem_device, budget, depth=0): + if depth > MAX_DEPTH: + raise CleanupBlocked("depth-exceeded") + quarantine_name, quarantine_fd, quarantine_stat = create_private_quarantine( + directory_fd, filesystem_device, + ) + try: + names = [name for name in os.listdir(directory_fd) if name != quarantine_name] + for name in names: + if name in (".", "..") or os.sep in name or "\x00" in name: + raise CleanupBlocked("entry-invalid") + budget[0] += 1 + if budget[0] > MAX_ENTRIES: + raise CleanupBlocked("entry-limit-exceeded") + observed = stat_entry(directory_fd, name) + if observed.st_dev != filesystem_device: + raise CleanupBlocked("filesystem-boundary-crossed") + captured = capture_observed_entry(directory_fd, name, observed, quarantine_fd) + if stat.S_ISDIR(captured.st_mode): + child_fd = open_directory(QUARANTINE_SLOT, quarantine_fd) + try: + opened = os.fstat(child_fd) + if not same_object(opened, captured): + raise CleanupBlocked("captured-directory-changed") + clear_directory(child_fd, filesystem_device, budget, depth + 1) + current = stat_entry(quarantine_fd, QUARANTINE_SLOT) + if not same_object(current, opened): + raise CleanupBlocked("captured-directory-changed") + os.rmdir(QUARANTINE_SLOT, dir_fd=quarantine_fd) + finally: + os.close(child_fd) + else: + current = stat_entry(quarantine_fd, QUARANTINE_SLOT) + if not same_object(current, captured): + raise CleanupBlocked("captured-entry-changed") + os.unlink(QUARANTINE_SLOT, dir_fd=quarantine_fd) + if not entry_is_absent(directory_fd, name): + raise CleanupBlocked("source-entry-reappeared") + remaining = [name for name in os.listdir(directory_fd) if name != quarantine_name] + if remaining: + raise CleanupBlocked("directory-changed-during-cleanup") + finally: + os.close(quarantine_fd) + cleanup_fd = None + try: + cleanup_fd = open_directory(quarantine_name, directory_fd) + cleanup_stat = os.fstat(cleanup_fd) + if same_object(cleanup_stat, quarantine_stat): + remove_empty_quarantine(directory_fd, quarantine_name, cleanup_fd, quarantine_stat) + except OSError: + pass + finally: + if cleanup_fd is not None: + os.close(cleanup_fd) + + +def open_bound_root(request): + root = os.path.normpath(request["containment_root"]) + target = os.path.normpath(request["target"]) + if root != request["containment_root_identity"]["realpath"]: + raise CleanupBlocked("root-path-mismatch") + if target != request["target_identity"]["realpath"]: + raise CleanupBlocked("target-path-mismatch") + if os.path.dirname(target) != root or os.path.basename(target) in ("", ".", ".."): + raise CleanupBlocked("target-must-be-direct-child") + root_parent = os.path.dirname(root) + root_name = os.path.basename(root) + if not root_parent or root_name in ("", ".", ".."): + raise CleanupBlocked("containment-invalid") + + parent_fd = open_directory(root_parent) + opened = [parent_fd] + try: + parent_stat = os.fstat(parent_fd) + linked_root = stat_entry(parent_fd, root_name) + root_fd = open_directory(root_name, parent_fd) + opened.append(root_fd) + root_stat = os.fstat(root_fd) + if ( + not same_object(linked_root, root_stat) + or not identity_matches(root_stat, request["containment_root_identity"]) + ): + raise CleanupBlocked("root-identity-mismatch") + return { + "opened": opened, + "parent_fd": parent_fd, + "parent_stat": parent_stat, + "root_name": root_name, + "root_fd": root_fd, + "root_stat": root_stat, + "target_name": os.path.basename(target), + } + except Exception: + for descriptor in reversed(opened): + os.close(descriptor) + raise + + +def open_allocation_root(request): + root = os.path.normpath(request["containment_root"]) + target = os.path.normpath(request["target"]) + if root != request["containment_root_identity"]["realpath"]: + raise CleanupBlocked("allocation-root-path-mismatch") + if os.path.dirname(target) != root or os.path.basename(target) in ("", ".", ".."): + raise CleanupBlocked("allocation-target-must-be-direct-child") + root_parent = os.path.dirname(root) + root_name = os.path.basename(root) + if not root_parent or root_name in ("", ".", ".."): + raise CleanupBlocked("allocation-containment-invalid") + + parent_fd = open_directory(root_parent) + opened = [parent_fd] + try: + parent_stat = os.fstat(parent_fd) + linked_root = stat_entry(parent_fd, root_name) + root_fd = open_directory(root_name, parent_fd) + opened.append(root_fd) + root_stat = os.fstat(root_fd) + if ( + not same_object(linked_root, root_stat) + or not identity_matches(root_stat, request["containment_root_identity"]) + or root_stat.st_uid != os.geteuid() + or root_stat.st_mode & 0o077 + ): + raise CleanupBlocked("allocation-root-identity-mismatch") + return { + "opened": opened, + "parent_fd": parent_fd, + "parent_stat": parent_stat, + "root_name": root_name, + "root_fd": root_fd, + "root_stat": root_stat, + "target_name": os.path.basename(target), + } + except Exception: + for descriptor in reversed(opened): + os.close(descriptor) + raise + + +def create_bound_child_directories(root_fd, values): + for relative in sorted(values, key=lambda item: (item.count("/"), item)): + current_fd = os.dup(root_fd) + try: + for segment in relative.split("/"): + try: + os.mkdir(segment, mode=0o700, dir_fd=current_fd) + except FileExistsError: + pass + child_fd = open_directory(segment, current_fd) + child_stat = os.fstat(child_fd) + linked = stat_entry(current_fd, segment) + if ( + not stat.S_ISDIR(linked.st_mode) + or not same_object(linked, child_stat) + or child_stat.st_uid != os.geteuid() + or child_stat.st_mode & 0o077 + ): + os.close(child_fd) + raise CleanupBlocked("allocation-child-invalid") + os.close(current_fd) + current_fd = child_fd + os.fsync(current_fd) + finally: + os.close(current_fd) + + +def validate_allocated_directory(directory_fd, directory_stat, request): + if ( + not stat.S_ISDIR(directory_stat.st_mode) + or directory_stat.st_uid != os.geteuid() + or directory_stat.st_mode & 0o077 + ): + raise CleanupBlocked("allocated-directory-invalid") + if read_bound_file(directory_fd, request["marker_name"], 16 * 1024) != request["marker_body"].encode("utf-8"): + raise CleanupBlocked("allocated-directory-marker-differs") + expected = {"": {request["marker_name"]}} + for relative in request["child_directories"]: + parent = "" + for segment in relative.split("/"): + expected.setdefault(parent, set()).add(segment) + parent = segment if parent == "" else parent + "/" + segment + expected.setdefault(parent, set()) + for relative, names in expected.items(): + current_fd = os.dup(directory_fd) + try: + if relative: + for segment in relative.split("/"): + child_fd = open_directory(segment, current_fd) + os.close(current_fd) + current_fd = child_fd + if set(os.listdir(current_fd)) != names: + raise CleanupBlocked("allocated-directory-contents-differ") + finally: + os.close(current_fd) + + +def allocation_receipt(request, bound, target_stat): + return { + "schema": ALLOCATION_RECEIPT_SCHEMA, + "status": "allocated", + "allocation_id": request["allocation_id"], + "target_realpath": request["target"], + "target_device": str(target_stat.st_dev), + "target_inode": str(target_stat.st_ino), + "containment_root_device": str(bound["root_stat"].st_dev), + "containment_root_inode": str(bound["root_stat"].st_ino), + } + + +def allocate_directory(request): + bound = open_allocation_root(request) + stage_fd = None + stage_name = None + try: + try: + existing = stat_entry(bound["root_fd"], bound["target_name"]) + except FileNotFoundError: + existing = None + if existing is not None: + target_fd = open_directory(bound["target_name"], bound["root_fd"]) + try: + opened = os.fstat(target_fd) + if not same_object(existing, opened) or opened.st_dev != bound["root_stat"].st_dev: + raise CleanupBlocked("allocated-target-changed") + validate_allocated_directory(target_fd, opened, request) + if not root_still_bound(bound): + raise CleanupBlocked("allocation-root-moved") + return allocation_receipt(request, bound, opened) + finally: + os.close(target_fd) + + for _ in range(16): + candidate = ALLOCATION_STAGE_PREFIX + secrets.token_hex(16) + try: + os.mkdir(candidate, mode=0o700, dir_fd=bound["root_fd"]) + stage_name = candidate + break + except FileExistsError: + continue + if stage_name is None: + raise CleanupBlocked("allocation-stage-unavailable") + stage_fd = open_directory(stage_name, bound["root_fd"]) + linked_stage = stat_entry(bound["root_fd"], stage_name) + stage_stat = os.fstat(stage_fd) + if ( + not same_object(linked_stage, stage_stat) + or stage_stat.st_dev != bound["root_stat"].st_dev + or stage_stat.st_uid != os.geteuid() + or stage_stat.st_mode & 0o077 + ): + raise CleanupBlocked("allocation-stage-invalid") + write_exclusive_file( + stage_fd, request["marker_name"], request["marker_body"].encode("utf-8"), + ) + create_bound_child_directories(stage_fd, request["child_directories"]) + validate_allocated_directory(stage_fd, stage_stat, request) + os.fsync(stage_fd) + if not root_still_bound(bound): + raise CleanupBlocked("allocation-root-moved-before-publish") + rename_noreplace(stage_name, bound["root_fd"], bound["target_name"], bound["root_fd"]) + stage_name = None + os.fsync(bound["root_fd"]) + published = stat_entry(bound["root_fd"], bound["target_name"]) + if not same_object(published, stage_stat) or not root_still_bound(bound): + raise CleanupBlocked("allocated-target-changed-after-publish") + return allocation_receipt(request, bound, published) + finally: + if stage_fd is not None: + os.close(stage_fd) + if stage_name is not None: + try: + os.rmdir(stage_name, dir_fd=bound["root_fd"]) + except OSError: + pass + for descriptor in reversed(bound["opened"]): + os.close(descriptor) + + +def retire_directory_marker(request): + bound = open_bound_root(request) + target_fd = None + try: + target_fd, target_stat = open_expected_target(bound, request) + expected = request["marker_body"].encode("utf-8") + if read_bound_file(target_fd, request["marker_name"], 16 * 1024) != expected: + raise CleanupBlocked("retired-marker-binding-differs") + os.unlink(request["marker_name"], dir_fd=target_fd) + os.fsync(target_fd) + current = stat_entry(bound["root_fd"], bound["target_name"]) + if not same_object(current, target_stat) or not root_still_bound(bound): + raise CleanupBlocked("retired-marker-target-changed") + return { + "schema": MARKER_RETIREMENT_RECEIPT_SCHEMA, + "status": "retired", + "allocation_id": request["allocation_id"], + "target_device": str(target_stat.st_dev), + "target_inode": str(target_stat.st_ino), + "containment_root_device": str(bound["root_stat"].st_dev), + "containment_root_inode": str(bound["root_stat"].st_ino), + } + finally: + if target_fd is not None: + os.close(target_fd) + for descriptor in reversed(bound["opened"]): + os.close(descriptor) + + +def root_still_bound(bound): + try: + return ( + same_object(os.fstat(bound["parent_fd"]), bound["parent_stat"]) + and same_object(stat_entry(bound["parent_fd"], bound["root_name"]), bound["root_stat"]) + ) + except OSError: + return False + + +def open_expected_target(bound, request): + linked_target = stat_entry(bound["root_fd"], bound["target_name"]) + if not stat.S_ISDIR(linked_target.st_mode): + raise CleanupBlocked("target-not-directory") + target_fd = open_directory(bound["target_name"], bound["root_fd"]) + try: + target_stat = os.fstat(target_fd) + if ( + not same_object(linked_target, target_stat) + or target_stat.st_dev != bound["root_stat"].st_dev + or not identity_matches(target_stat, request["target_identity"]) + ): + raise CleanupBlocked("target-identity-mismatch") + return target_fd, target_stat + except Exception: + os.close(target_fd) + raise + + +def capture_receipt(request, bound, status): + return { + "schema": RECEIPT_SCHEMA, + "status": status, + "capture_id": request["capture_id"], + "target_removed": True, + "target_device": request["target_identity"]["device"], + "target_inode": request["target_identity"]["inode"], + "containment_root_device": str(bound["root_stat"].st_dev), + "containment_root_inode": str(bound["root_stat"].st_ino), + } + + +def marker_token(request, state): + return (state + ":" + request["capture_id"] + "\n").encode("ascii") + + +def mark_capture_retained(quarantine_fd, request): + if marker_exists(quarantine_fd, CAPTURE_CLEANED_MARKER): + return + if not marker_exists(quarantine_fd, CAPTURE_RETAINED_MARKER): + write_exclusive_file( + quarantine_fd, + CAPTURE_RETAINED_MARKER, + marker_token(request, "retained"), + ) + os.fsync(quarantine_fd) + + +def validate_capture_entries(quarantine_fd): + names = set(os.listdir(quarantine_fd)) + allowed = { + CAPTURE_MARKER, + CAPTURE_CLEANED_MARKER, + CAPTURE_RETAINED_MARKER, + CAPTURE_FINALIZED_MARKER, + QUARANTINE_SLOT, + } + if not names.issubset(allowed): + raise CleanupBlocked("capture-quarantine-contains-unknown-entry") + return names + + +def capture_delete(request): + bound = open_bound_root(request) + quarantine = None + target_fd = None + captured = None + delete_started = False + try: + quarantine = open_capture_quarantine(bound, request, True) + names = validate_capture_entries(quarantine["fd"]) + if CAPTURE_RETAINED_MARKER in names: + if read_bound_file(quarantine["fd"], CAPTURE_RETAINED_MARKER) != marker_token(request, "retained"): + raise CleanupBlocked("capture-retained-marker-invalid") + raise CleanupBlocked("captured-resource-retained") + if CAPTURE_CLEANED_MARKER in names: + if ( + QUARANTINE_SLOT in names + or read_bound_file(quarantine["fd"], CAPTURE_CLEANED_MARKER) + != marker_token(request, "cleaned") + ): + raise CleanupBlocked("capture-cleaned-marker-invalid") + return capture_receipt(request, bound, "captured-cleaned") + + if QUARANTINE_SLOT in names: + captured = stat_entry(quarantine["fd"], QUARANTINE_SLOT) + if ( + not stat.S_ISDIR(captured.st_mode) + or not identity_matches(captured, request["target_identity"]) + ): + raise CleanupBlocked("captured-target-changed") + target_fd = open_directory(QUARANTINE_SLOT, quarantine["fd"]) + if not same_object(os.fstat(target_fd), captured): + raise CleanupBlocked("captured-target-changed") + else: + target_fd, observed = open_expected_target(bound, request) + captured = capture_observed_entry( + bound["root_fd"], bound["target_name"], observed, quarantine["fd"], + ) + if not same_object(os.fstat(target_fd), captured): + raise CleanupBlocked("captured-target-changed") + + if not root_still_bound(bound): + raise CleanupBlocked("root-moved-before-capture-completed") + if not entry_is_absent(bound["root_fd"], bound["target_name"]): + raise CleanupBlocked("target-reappeared-before-delete") + current = stat_entry(quarantine["fd"], QUARANTINE_SLOT) + if not same_object(current, captured) or not same_object(current, os.fstat(target_fd)): + raise CleanupBlocked("captured-target-changed") + + delete_started = True + clear_directory(target_fd, captured.st_dev, [0]) + current = stat_entry(quarantine["fd"], QUARANTINE_SLOT) + if not same_object(current, captured): + raise CleanupBlocked("captured-target-changed") + os.rmdir(QUARANTINE_SLOT, dir_fd=quarantine["fd"]) + if not root_still_bound(bound): + raise CleanupBlocked("root-moved-during-cleanup") + write_exclusive_file( + quarantine["fd"], + CAPTURE_CLEANED_MARKER, + marker_token(request, "cleaned"), + ) + os.fsync(quarantine["fd"]) + return capture_receipt(request, bound, "captured-cleaned") + except Exception: + if quarantine is not None: + if captured is not None and not delete_started: + try: + if entry_is_absent(bound["root_fd"], bound["target_name"]): + restore_captured_entry( + bound["root_fd"], bound["target_name"], quarantine["fd"], captured, + ) + captured = None + except (CleanupBlocked, OSError): + pass + try: + mark_capture_retained(quarantine["fd"], request) + except (CleanupBlocked, OSError): + pass + raise + finally: + if target_fd is not None: + os.close(target_fd) + if quarantine is not None: + os.close(quarantine["fd"]) + for descriptor in reversed(bound["opened"]): + os.close(descriptor) + + +def finalize_capture(request): + bound = open_bound_root(request) + quarantine = None + try: + quarantine = open_capture_quarantine(bound, request, False) + if quarantine is None: + raise CleanupBlocked("capture-proof-missing") + names = validate_capture_entries(quarantine["fd"]) + expected_names = {CAPTURE_MARKER, CAPTURE_CLEANED_MARKER} + if CAPTURE_FINALIZED_MARKER in names: + expected_names.add(CAPTURE_FINALIZED_MARKER) + if names != expected_names: + raise CleanupBlocked("capture-not-finalizable") + if ( + read_bound_file(quarantine["fd"], CAPTURE_CLEANED_MARKER) + != marker_token(request, "cleaned") + ): + raise CleanupBlocked("capture-cleaned-marker-invalid") + if CAPTURE_FINALIZED_MARKER in names: + if ( + read_bound_file(quarantine["fd"], CAPTURE_FINALIZED_MARKER) + != marker_token(request, "finalized") + ): + raise CleanupBlocked("capture-finalized-marker-invalid") + else: + write_exclusive_file( + quarantine["fd"], + CAPTURE_FINALIZED_MARKER, + marker_token(request, "finalized"), + ) + os.fsync(quarantine["fd"]) + return capture_receipt(request, bound, "finalized") + finally: + if quarantine is not None: + os.close(quarantine["fd"]) + for descriptor in reversed(bound["opened"]): + os.close(descriptor) + + +def release_capture_proof(request): + bound = open_bound_root(request) + quarantine = None + try: + quarantine = open_capture_quarantine(bound, request, False) + if quarantine is None: + return capture_receipt(request, bound, "released") + names = validate_capture_entries(quarantine["fd"]) + if names != {CAPTURE_MARKER, CAPTURE_CLEANED_MARKER, CAPTURE_FINALIZED_MARKER}: + raise CleanupBlocked("capture-proof-not-releasable") + if ( + read_bound_file(quarantine["fd"], CAPTURE_CLEANED_MARKER) + != marker_token(request, "cleaned") + or read_bound_file(quarantine["fd"], CAPTURE_FINALIZED_MARKER) + != marker_token(request, "finalized") + ): + raise CleanupBlocked("capture-proof-marker-invalid") + for name in (CAPTURE_FINALIZED_MARKER, CAPTURE_CLEANED_MARKER, CAPTURE_MARKER): + os.unlink(name, dir_fd=quarantine["fd"]) + os.fsync(quarantine["fd"]) + linked = stat_entry(bound["parent_fd"], quarantine["name"]) + if not same_object(linked, quarantine["stat"]): + raise CleanupBlocked("capture-quarantine-changed") + os.rmdir(quarantine["name"], dir_fd=bound["parent_fd"]) + os.fsync(bound["parent_fd"]) + return capture_receipt(request, bound, "released") + finally: + if quarantine is not None: + os.close(quarantine["fd"]) + for descriptor in reversed(bound["opened"]): + os.close(descriptor) + + +def cleanup(request): + if request["operation"] == "capture-delete": + return capture_delete(request) + if request["operation"] == "finalize": + return finalize_capture(request) + if request["operation"] == "release-proof": + return release_capture_proof(request) + raise CleanupBlocked("operation-invalid") + + +def execute(request): + if request["schema"] == ALLOCATION_REQUEST_SCHEMA: + return allocate_directory(request) + if request["schema"] == MARKER_RETIREMENT_REQUEST_SCHEMA: + return retire_directory_marker(request) + return cleanup(request) + + +def emit(value): + sys.stdout.write(json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n") + + +def main(): + try: + emit(execute(read_request())) + except (CleanupBlocked, FileNotFoundError, NotADirectoryError, PermissionError, OSError): + emit({ + "schema": RECEIPT_SCHEMA, + "status": "blocked", + "reason": "OBJECT_BOUND_DIRECTORY_OPERATION_FAILED", + }) + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/packages/environment-factory/bin/runtime/budgets.js b/packages/environment-factory/bin/runtime/budgets.js index 50a5ca70..085e66bd 100644 --- a/packages/environment-factory/bin/runtime/budgets.js +++ b/packages/environment-factory/bin/runtime/budgets.js @@ -112,6 +112,7 @@ function createBudgetRuntime(deps) { } const result = await runMeasuredCommand(argv, { cwd: options.cwd, + cwdIdentity: options.cwdIdentity, env: options.env, timeoutMs: options.timeoutMs, outputBytes: payload.output_bytes, diff --git a/packages/environment-factory/bin/runtime/common.js b/packages/environment-factory/bin/runtime/common.js index 1b8adf2d..ce43b693 100644 --- a/packages/environment-factory/bin/runtime/common.js +++ b/packages/environment-factory/bin/runtime/common.js @@ -6,8 +6,12 @@ const path = require('path'); const { spawn, spawnSync } = require('child_process'); const MAX_JSON_BYTES = 2 * 1024 * 1024; +const MAX_LOCK_METADATA_BYTES = 8 * 1024; +const MAX_CLEANUP_BROKER_BYTES = 256 * 1024; const DEFAULT_OUTPUT_BYTES = 64 * 1024; const LOCK_TIMEOUT_MS = 10_000; +const OBJECT_BOUND_CLEANUP_UNAVAILABLE = 'OBJECT_BOUND_CLEANUP_UNAVAILABLE'; +const CLEANUP_CAPTURE_SCHEMA = 'environment-factory.object-bound-cleanup-capture-state.v1'; const sleepCell = new Int32Array(new SharedArrayBuffer(4)); const WORKER_ENVIRONMENT_LEASE = Symbol('fkst.worker-environment-lease'); @@ -56,6 +60,51 @@ function boundedText(value, limit = 512) { return String(value || '').replace(/[\x00-\x1f\x7f]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, limit); } +function readBoundedRegularFile(filePath, maximumBytes = MAX_JSON_BYTES) { + if (!Number.isInteger(maximumBytes) || maximumBytes < 1 || maximumBytes > MAX_JSON_BYTES) { + throw new Error('bounded file limit is invalid'); + } + const flags = fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0); + const fd = fs.openSync(filePath, flags); + try { + const before = fs.fstatSync(fd, { bigint: true }); + const linkedBefore = fs.lstatSync(filePath, { bigint: true }); + if (!before.isFile() || !linkedBefore.isFile() || linkedBefore.isSymbolicLink() + || before.dev !== linkedBefore.dev || before.ino !== linkedBefore.ino + || before.size > BigInt(maximumBytes)) { + throw new Error(`bounded regular file is invalid: ${filePath}`); + } + const realpath = fs.realpathSync(filePath); + const size = Number(before.size); + const buffer = Buffer.alloc(size); + let offset = 0; + while (offset < size) { + const count = fs.readSync(fd, buffer, offset, size - offset, offset); + if (count === 0) break; + offset += count; + } + const after = fs.fstatSync(fd, { bigint: true }); + const linkedAfter = fs.lstatSync(filePath, { bigint: true }); + if (offset !== size || after.dev !== before.dev || after.ino !== before.ino + || after.size !== before.size || after.mtimeNs !== before.mtimeNs || after.ctimeNs !== before.ctimeNs + || linkedAfter.dev !== before.dev || linkedAfter.ino !== before.ino + || linkedAfter.isSymbolicLink() || !linkedAfter.isFile() + || fs.realpathSync(filePath) !== realpath) { + throw new Error(`bounded regular file changed while reading: ${filePath}`); + } + return { + body: buffer.toString('utf8'), + identity: { + realpath, + device: String(before.dev), + inode: String(before.ino), + }, + }; + } finally { + fs.closeSync(fd); + } +} + function readJson(filePath) { const stat = fs.statSync(filePath); if (stat.size > MAX_JSON_BYTES) throw new Error(`JSON input exceeds ${MAX_JSON_BYTES} bytes`); @@ -137,28 +186,348 @@ function samePathIdentity(left, right) { && left.device === right.device && left.inode === right.inode); } -function removeOwnedDirectory(target, expectedIdentity, containmentRoot) { +function pathEntryExists(target) { + try { + fs.lstatSync(target); + return true; + } catch (error) { + if (error.code === 'ENOENT') return false; + throw error; + } +} + +function configuredObjectBoundBroker(pathKey, digestKey, label) { + const sourcePath = process.env[pathKey]; + const expectedSha256 = process.env[digestKey]; + if (sourcePath === undefined && expectedSha256 === undefined) return null; + if (typeof sourcePath !== 'string' || !path.isAbsolute(sourcePath) + || typeof expectedSha256 !== 'string' || !/^[0-9a-f]{64}$/.test(expectedSha256)) { + throw new Error(`object-bound ${label} broker configuration is invalid`); + } + const source = readBoundedRegularFile(sourcePath, MAX_CLEANUP_BROKER_BYTES).body; + if (sha256(source) !== expectedSha256) { + throw new Error(`object-bound ${label} broker digest differs`); + } + return source; +} + +function configuredObjectBoundCleanupBroker() { + return configuredObjectBoundBroker( + 'FKST_OBJECT_BOUND_CLEANUP_BROKER', 'FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256', 'cleanup', + ); +} + +function configuredObjectBoundAllocationBroker() { + const configured = configuredObjectBoundBroker( + 'FKST_OBJECT_BOUND_ALLOCATION_BROKER', 'FKST_OBJECT_BOUND_ALLOCATION_BROKER_SHA256', + 'allocation', + ); + return configured === null ? configuredObjectBoundCleanupBroker() : configured; +} + +function invokeObjectBoundCleanupBroker( + target, targetIdentity, containmentRoot, rootIdentity, captureId, operation, +) { + const source = configuredObjectBoundCleanupBroker(); + if (source === null) return null; + const python = process.platform === 'win32' ? null : '/usr/bin/python3'; + if (python === null || !fs.existsSync(python)) return null; + const request = { + schema: 'environment-factory.object-bound-cleanup-request.v1', + operation, + capture_id: captureId, + target, + target_identity: targetIdentity, + containment_root: containmentRoot, + containment_root_identity: rootIdentity, + }; + const environment = Object.create(null); + for (const key of ['LANG', 'LC_ALL']) { + if (typeof process.env[key] === 'string') environment[key] = process.env[key]; + } + const result = spawnSync(python, ['-I', '-c', source], { + input: `${stableStringify(request)}\n`, + env: environment, + shell: false, + encoding: 'utf8', + timeout: 30_000, + maxBuffer: MAX_LOCK_METADATA_BYTES, + }); + if (result.status !== 0) return null; + let receipt; + try { receipt = JSON.parse(String(result.stdout || '')); } catch (_error) { return null; } + const expectedStatus = { + 'capture-delete': 'captured-cleaned', + finalize: 'finalized', + 'release-proof': 'released', + }[operation]; + if (expectedStatus === undefined) return null; + return receipt + && receipt.schema === 'environment-factory.object-bound-cleanup-receipt.v1' + && receipt.status === expectedStatus + && receipt.capture_id === captureId + && receipt.target_removed === true + && String(receipt.target_device) === targetIdentity.device + && String(receipt.target_inode) === targetIdentity.inode + && String(receipt.containment_root_device) === rootIdentity.device + && String(receipt.containment_root_inode) === rootIdentity.inode + ? receipt : null; +} + +function allocateOwnedDirectory( + target, containmentRoot, rootIdentity, allocationId, markerName, markerBody, + childDirectories = [], +) { + if (typeof target !== 'string' || !path.isAbsolute(target) + || typeof containmentRoot !== 'string' || !path.isAbsolute(containmentRoot) + || path.dirname(target) !== containmentRoot + || !samePathIdentity(pathIdentity(containmentRoot), rootIdentity) + || !/^[0-9a-f]{64}$/.test(String(allocationId || '')) + || typeof markerName !== 'string' || markerName === '' || path.basename(markerName) !== markerName + || typeof markerBody !== 'string' || !markerBody.endsWith('\n') + || !Array.isArray(childDirectories)) { + throw new Error('object-bound directory allocation binding is invalid'); + } + const source = configuredObjectBoundAllocationBroker(); + const python = process.platform === 'win32' ? null : '/usr/bin/python3'; + if (source === null || python === null || !fs.existsSync(python)) { + throw new Error('OBJECT_BOUND_DIRECTORY_ALLOCATION_UNAVAILABLE'); + } + const request = { + schema: 'environment-factory.object-bound-directory-allocation-request.v1', + operation: 'allocate-directory', + allocation_id: allocationId, + target: path.join(rootIdentity.realpath, path.basename(target)), + containment_root: rootIdentity.realpath, + containment_root_identity: rootIdentity, + marker_name: markerName, + marker_body: markerBody, + child_directories: childDirectories, + }; + const environment = Object.create(null); + for (const key of ['LANG', 'LC_ALL']) { + if (typeof process.env[key] === 'string') environment[key] = process.env[key]; + } + const result = spawnSync(python, ['-I', '-c', source], { + input: `${stableStringify(request)}\n`, + env: environment, + shell: false, + encoding: 'utf8', + timeout: 30_000, + maxBuffer: MAX_LOCK_METADATA_BYTES, + }); + if (result.status !== 0) throw new Error('OBJECT_BOUND_DIRECTORY_ALLOCATION_FAILED'); + let receipt; + try { receipt = JSON.parse(String(result.stdout || '')); } catch (_error) { + throw new Error('OBJECT_BOUND_DIRECTORY_ALLOCATION_FAILED'); + } + const identity = receipt && { + realpath: receipt.target_realpath, + device: String(receipt.target_device), + inode: String(receipt.target_inode), + }; + if (!receipt + || receipt.schema !== 'environment-factory.object-bound-directory-allocation-receipt.v1' + || receipt.status !== 'allocated' || receipt.allocation_id !== allocationId + || receipt.target_realpath !== request.target + || String(receipt.containment_root_device) !== rootIdentity.device + || String(receipt.containment_root_inode) !== rootIdentity.inode + || !samePathIdentity(pathIdentity(target), identity) + || !samePathIdentity(pathIdentity(containmentRoot), rootIdentity)) { + throw new Error('OBJECT_BOUND_DIRECTORY_ALLOCATION_FAILED'); + } + return identity; +} + +function retireOwnedDirectoryMarker( + target, targetIdentity, containmentRoot, rootIdentity, allocationId, markerName, markerBody, +) { + if (!samePathIdentity(pathIdentity(target), targetIdentity) + || !samePathIdentity(pathIdentity(containmentRoot), rootIdentity)) { + throw new Error('object-bound marker retirement identity changed'); + } + const source = configuredObjectBoundAllocationBroker(); + const python = process.platform === 'win32' ? null : '/usr/bin/python3'; + if (source === null || python === null || !fs.existsSync(python)) { + throw new Error('OBJECT_BOUND_MARKER_RETIREMENT_UNAVAILABLE'); + } + const request = { + schema: 'environment-factory.object-bound-marker-retirement-request.v1', + operation: 'retire-marker', + allocation_id: allocationId, + target: targetIdentity.realpath, + target_identity: targetIdentity, + containment_root: rootIdentity.realpath, + containment_root_identity: rootIdentity, + marker_name: markerName, + marker_body: markerBody, + }; + const environment = Object.create(null); + for (const key of ['LANG', 'LC_ALL']) { + if (typeof process.env[key] === 'string') environment[key] = process.env[key]; + } + const result = spawnSync(python, ['-I', '-c', source], { + input: stableStringify(request) + '\n', + env: environment, + shell: false, + encoding: 'utf8', + timeout: 30_000, + maxBuffer: MAX_LOCK_METADATA_BYTES, + }); + if (result.status !== 0) throw new Error('OBJECT_BOUND_MARKER_RETIREMENT_FAILED'); + let receipt; + try { receipt = JSON.parse(String(result.stdout || '')); } catch (_error) { + throw new Error('OBJECT_BOUND_MARKER_RETIREMENT_FAILED'); + } + if (!receipt || receipt.schema !== 'environment-factory.object-bound-marker-retirement-receipt.v1' + || receipt.status !== 'retired' || receipt.allocation_id !== allocationId + || String(receipt.target_device) !== targetIdentity.device + || String(receipt.target_inode) !== targetIdentity.inode + || String(receipt.containment_root_device) !== rootIdentity.device + || String(receipt.containment_root_inode) !== rootIdentity.inode + || !samePathIdentity(pathIdentity(target), targetIdentity) + || !samePathIdentity(pathIdentity(containmentRoot), rootIdentity)) { + throw new Error('OBJECT_BOUND_MARKER_RETIREMENT_FAILED'); + } +} + +function cleanupCaptureStatePath(captureId) { + const durableRoot = requireOwnedDirectory(path.resolve( + process.env.FKST_OBJECT_BOUND_CLEANUP_STATE_ROOT + || process.env.FKST_DURABLE_ROOT + || path.join('.testing', 'durable'), + )); + const stateRoot = requireOwnedDirectory( + path.join(durableRoot, 'cleanup-captures'), + { privateDirectory: true }, + ); + return path.join(stateRoot, `${captureId}.json`); +} + +function cleanupCaptureBinding(target, expectedIdentity, containmentRoot, rootIdentity, captureId) { + return { + schema: CLEANUP_CAPTURE_SCHEMA, + capture_id: captureId, + target, + target_identity: expectedIdentity, + containment_root: containmentRoot, + containment_root_identity: rootIdentity, + }; +} + +function ownedDirectoryReleaseProven(target, expectedIdentity, containmentRoot, captureId) { + if (typeof target !== 'string' || typeof containmentRoot !== 'string' + || !/^[0-9a-f]{64}$/.test(String(captureId || ''))) return false; + try { + const root = fs.realpathSync(containmentRoot); + const requestedTarget = path.resolve(target); + const targetParent = fs.realpathSync(path.dirname(requestedTarget)); + const canonicalTarget = path.join(targetParent, path.basename(requestedTarget)); + if (!expectedIdentity || targetParent !== root || expectedIdentity.realpath !== canonicalTarget + || canonicalTarget === root || !canonicalTarget.startsWith(`${root}${path.sep}`)) return false; + const binding = cleanupCaptureBinding( + canonicalTarget, expectedIdentity, root, pathIdentity(root), captureId, + ); + const state = readOptionalJson(cleanupCaptureStatePath(captureId)); + return state !== null + && stableStringify(state) === stableStringify({ ...binding, state: 'released' }) + && !pathEntryExists(canonicalTarget); + } catch (_error) { + return false; + } +} + +function readOptionalJson(filePath) { + try { + return readJson(filePath); + } catch (error) { + if (error.code === 'ENOENT') return null; + throw error; + } +} + +function removeOwnedDirectory(target, expectedIdentity, containmentRoot, captureId) { if (typeof target !== 'string' || typeof containmentRoot !== 'string') { throw new Error('owned directory paths are invalid'); } + if (!/^[0-9a-f]{64}$/.test(String(captureId || ''))) { + throw new Error('owned directory cleanup capture identity is invalid'); + } const root = fs.realpathSync(containmentRoot); - const identity = pathIdentity(target); - if (!samePathIdentity(identity, expectedIdentity)) throw new Error('owned directory identity changed'); - if (identity.realpath === root || !identity.realpath.startsWith(`${root}${path.sep}`)) { + const requestedTarget = path.resolve(target); + const targetParent = fs.realpathSync(path.dirname(requestedTarget)); + const targetName = path.basename(requestedTarget); + const canonicalTarget = path.join(targetParent, targetName); + if (!expectedIdentity || targetParent !== root || expectedIdentity.realpath !== canonicalTarget) { + throw new Error('owned directory cleanup identity is malformed'); + } + if (expectedIdentity.realpath === root || !expectedIdentity.realpath.startsWith(`${root}${path.sep}`)) { throw new Error('owned directory escaped containment root'); } - fs.rmSync(identity.realpath, { recursive: true, force: false }); - return !fs.existsSync(identity.realpath); + const rootIdentity = pathIdentity(root); + target = canonicalTarget; + const binding = cleanupCaptureBinding(target, expectedIdentity, root, rootIdentity, captureId); + const statePath = cleanupCaptureStatePath(captureId); + const release = acquireLock(`${statePath}.lock`); + try { + let state = readOptionalJson(statePath); + if (state === null) { + if (!pathEntryExists(target)) return false; + const identity = pathIdentity(target); + if (!samePathIdentity(identity, expectedIdentity)) { + throw new Error('owned directory identity changed'); + } + state = { ...binding, state: 'pending' }; + writeJsonAtomic(statePath, state); + } else { + const expected = { ...binding, state: state.state }; + if (stableStringify(state) !== stableStringify(expected) + || !['pending', 'captured-cleaned', 'finalized', 'released'].includes(state.state)) { + throw new Error('owned directory cleanup capture binding differs'); + } + } + if (state.state === 'pending') { + const captured = invokeObjectBoundCleanupBroker( + binding.target, binding.target_identity, binding.containment_root, + binding.containment_root_identity, captureId, 'capture-delete', + ); + if (captured === null) return false; + state = { ...binding, state: 'captured-cleaned' }; + writeJsonAtomic(statePath, state); + } + if (state.state === 'captured-cleaned') { + const finalized = invokeObjectBoundCleanupBroker( + binding.target, binding.target_identity, binding.containment_root, + binding.containment_root_identity, captureId, 'finalize', + ); + if (finalized === null) return false; + state = { ...binding, state: 'finalized' }; + writeJsonAtomic(statePath, state); + } + if (state.state === 'finalized') { + const released = invokeObjectBoundCleanupBroker( + binding.target, binding.target_identity, binding.containment_root, + binding.containment_root_identity, captureId, 'release-proof', + ); + if (released === null) return false; + state = { ...binding, state: 'released' }; + writeJsonAtomic(statePath, state); + } + return state.state === 'released' && !pathEntryExists(target); + } finally { + release(); + } } function readLockOwner(lockPath) { try { + const lockIdentity = lockPathIdentity(lockPath); const lockStat = fs.lstatSync(lockPath); - if (lockStat.isSymbolicLink()) return null; const ownerPath = lockStat.isDirectory() ? path.join(lockPath, 'owner.json') : lockPath; - const stat = fs.lstatSync(ownerPath); - if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_JSON_BYTES) return null; - const owner = JSON.parse(fs.readFileSync(ownerPath, 'utf8')); + const metadata = readBoundedRegularFile(ownerPath, MAX_LOCK_METADATA_BYTES); + if (!lockPathStillMatches(lockPath, lockIdentity) + || (!lockStat.isDirectory() && !samePathIdentity(lockIdentity, metadata.identity))) return null; + const owner = JSON.parse(metadata.body); if (!owner || owner.schema !== 'environment-factory.lock-owner.v1' || !Number.isInteger(owner.pid) || owner.pid < 1 || typeof owner.process_start_identity !== 'string' || owner.process_start_identity === '' @@ -197,13 +566,82 @@ function lockPathStillMatches(lockPath, expectedIdentity) { } } -function removeLockPath(lockPath, expectedIdentity) { - if (!samePathIdentity(lockPathIdentity(lockPath), expectedIdentity)) { - throw new Error(`lock path identity changed: ${lockPath}`); +function sameObjectIdentity(left, right) { + return Boolean(left && right && left.device === right.device && left.inode === right.inode); +} + +function createLockQuarantine(filePath) { + for (let attempt = 0; attempt < 8; attempt += 1) { + const root = `${filePath}.retired-${process.pid}-${crypto.randomBytes(16).toString('hex')}`; + try { + fs.mkdirSync(root, { mode: 0o700 }); + return { root, entry: path.join(root, 'entry') }; + } catch (error) { + if (error.code !== 'EEXIST') throw error; + } + } + throw new Error(`could not allocate private lock quarantine: ${filePath}`); +} + +function restoreQuarantinedLockEntry(filePath, quarantine) { + const movedStat = fs.lstatSync(quarantine.entry); + if (!movedStat.isFile() || movedStat.isSymbolicLink()) { + throw new Error(`lock replacement retained in quarantine: ${quarantine.entry}`); + } + try { + fs.linkSync(quarantine.entry, filePath); + } catch (error) { + if (error.code === 'EEXIST') { + throw new Error(`lock replacement could not be restored; retained in quarantine: ${quarantine.entry}`); + } + throw error; + } + const restored = pathIdentity(filePath); + const moved = pathIdentity(quarantine.entry); + if (!sameObjectIdentity(restored, moved)) { + throw new Error(`restored lock replacement identity differs; retained in quarantine: ${quarantine.entry}`); + } + fs.unlinkSync(quarantine.entry); + fs.rmdirSync(quarantine.root); +} + +function retireObservedLockFile(filePath, observed) { + const quarantine = createLockQuarantine(filePath); + try { + fs.renameSync(filePath, quarantine.entry); + } catch (error) { + fs.rmdirSync(quarantine.root); + if (error.code === 'ENOENT') return false; + throw error; + } + + let moved; + try { + moved = readBoundedRegularFile(quarantine.entry, MAX_LOCK_METADATA_BYTES); + } catch (_error) { + restoreQuarantinedLockEntry(filePath, quarantine); + return false; + } + if (!sameObjectIdentity(moved.identity, observed.identity) || moved.body !== observed.body) { + restoreQuarantinedLockEntry(filePath, quarantine); + return false; } + fs.unlinkSync(quarantine.entry); + fs.rmdirSync(quarantine.root); + return true; +} + +function removeLockPath(lockPath, expectedIdentity, expectedOwner) { const stat = fs.lstatSync(lockPath); - if (stat.isDirectory()) fs.rmSync(expectedIdentity.realpath, { recursive: true, force: true }); - else fs.unlinkSync(expectedIdentity.realpath); + if (stat.isDirectory()) { + const error = new Error(`legacy lock directory cleanup requires an object-bound broker: ${lockPath}`); + error.code = 'OBJECT_BOUND_CLEANUP_UNAVAILABLE'; + throw error; + } + const observed = readBoundedRegularFile(lockPath, MAX_LOCK_METADATA_BYTES); + if (!sameObjectIdentity(observed.identity, expectedIdentity) + || observed.body !== lockOwnerBody(expectedOwner)) return false; + return retireObservedLockFile(lockPath, observed); } function lockOwnerBody(owner) { @@ -217,11 +655,10 @@ function pendingLockOwnerPath(lockPath, owner) { function removeMatchingPendingLockOwner(lockPath, owner) { const pendingPath = pendingLockOwnerPath(lockPath, owner); try { - const stat = fs.lstatSync(pendingPath); - if (!stat.isFile() || stat.isSymbolicLink() - || fs.readFileSync(pendingPath, 'utf8') !== lockOwnerBody(owner)) return false; - fs.unlinkSync(pendingPath); - return true; + const metadata = readBoundedRegularFile(pendingPath, MAX_LOCK_METADATA_BYTES); + if (metadata.body !== lockOwnerBody(owner) + || !samePathIdentity(pathIdentity(pendingPath), metadata.identity)) return false; + return retireObservedLockFile(pendingPath, metadata); } catch (error) { if (error.code === 'ENOENT') return false; throw error; @@ -246,14 +683,14 @@ function createAtomicLock(lockPath, identity) { } const lockIdentity = lockPathIdentity(lockPath); if (readLockOwner(lockPath) === null || fs.readFileSync(lockPath, 'utf8') !== ownerBody) { - try { removeLockPath(lockPath, lockIdentity); } catch (_cleanupError) {} + try { removeLockPath(lockPath, lockIdentity, owner); } catch (_cleanupError) {} try { fs.unlinkSync(pendingPath); } catch (_cleanupError) {} throw new Error(`atomic lock publication failed: ${lockPath}`); } try { fs.unlinkSync(pendingPath); } catch (error) { - try { removeLockPath(lockPath, lockIdentity); } catch (_cleanupError) {} + try { removeLockPath(lockPath, lockIdentity, owner); } catch (_cleanupError) {} throw error; } let released = false; @@ -265,7 +702,7 @@ function createAtomicLock(lockPath, identity) { const recorded = readLockOwner(lockPath); if (sameLockOwner(recorded, owner) && samePathIdentity(lockPathIdentity(lockPath), lockIdentity)) { - removeLockPath(lockPath, lockIdentity); + removeLockPath(lockPath, lockIdentity, owner); } released = true; }, @@ -294,6 +731,12 @@ function acquireTakeoverGuard(lockPath, timeoutMs) { for (const key of ['LANG', 'LC_ALL', 'PATH', 'SystemRoot', 'WINDIR']) { if (typeof process.env[key] === 'string') environment[key] = process.env[key]; } + for (const key of [ + 'FKST_OBJECT_BOUND_CLEANUP_BROKER', + 'FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256', + ]) { + if (typeof process.env[key] === 'string') environment[key] = process.env[key]; + } let argv; if (process.platform === 'linux') { const flock = executablePath(['/usr/bin/flock', '/bin/flock']); @@ -315,7 +758,7 @@ function acquireTakeoverGuard(lockPath, timeoutMs) { const currentIdentity = processStartIdentity(child.pid); if (currentIdentity === null) return false; try { - const marker = JSON.parse(fs.readFileSync(markerPath, 'utf8')); + const marker = JSON.parse(readBoundedRegularFile(markerPath, MAX_LOCK_METADATA_BYTES).body); if (marker.pid !== child.pid || marker.token !== token) return false; if (holderIdentity === null) holderIdentity = currentIdentity; return currentIdentity === holderIdentity; @@ -383,8 +826,14 @@ function acquireLock(lockPath, timeoutMs = LOCK_TIMEOUT_MS, options = {}) { if (error.code === 'ENOENT') continue; throw error; } - const observedOwner = readLockOwner(lockPath); + let observedOwner = readLockOwner(lockPath); if (observedOwner === null) { + if (!lockPathStillMatches(lockPath, observedIdentity)) continue; + sleep(10); + if (!lockPathStillMatches(lockPath, observedIdentity)) continue; + observedOwner = readLockOwner(lockPath); + if (observedOwner !== null) continue; + if (!lockPathStillMatches(lockPath, observedIdentity)) continue; throw new Error(`ownerless or malformed lock cannot be recovered safely: ${lockPath}`); } if (lockOwnerIsStale(observedOwner)) { @@ -402,7 +851,7 @@ function acquireLock(lockPath, timeoutMs = LOCK_TIMEOUT_MS, options = {}) { if (sameLockOwner(confirmedOwner, observedOwner) && lockPathStillMatches(lockPath, observedIdentity) && lockOwnerIsStale(confirmedOwner)) { - removeLockPath(lockPath, observedIdentity); + if (!removeLockPath(lockPath, observedIdentity, confirmedOwner)) continue; removeMatchingPendingLockOwner(lockPath, confirmedOwner); while (true) { try { @@ -500,6 +949,9 @@ function forbiddenWorkerEnvironmentKey(key) { const exact = new Set([ 'HOME', 'USERPROFILE', 'HOMEDRIVE', 'HOMEPATH', 'APPDATA', 'LOCALAPPDATA', 'XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'XDG_STATE_HOME', 'GH_CONFIG_DIR', + 'FKST_WORKER_RUNTIME_ROOT', + 'FKST_OBJECT_BOUND_ALLOCATION_BROKER', 'FKST_OBJECT_BOUND_ALLOCATION_BROKER_SHA256', + 'FKST_OBJECT_BOUND_CLEANUP_BROKER', 'FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256', ]); return exact.has(upper) || upper.startsWith('GH_') || upper.startsWith('GITHUB_') || upper.startsWith('GIT_') || upper.startsWith('SSH_') @@ -530,7 +982,7 @@ function requireOwnedDirectory(directory, { privateDirectory = false } = {}) { return fs.realpathSync(directory); } -function minimalEnvironment(extra = {}, isolationKey = 'shared-runtime-command', reservationId = null) { +function workerEnvironmentInputs(extra, isolationKey) { if (!extra || typeof extra !== 'object' || Array.isArray(extra)) { throw new Error('command environment must be an object'); } @@ -550,66 +1002,143 @@ function minimalEnvironment(extra = {}, isolationKey = 'shared-runtime-command', if (!identity || typeof identity !== 'object' || Array.isArray(identity)) { throw new Error('worker isolation identity is invalid'); } - const identityBody = stableStringify(identity); - if (reservationId !== null && !/^[0-9a-f]{32}$/.test(String(reservationId))) { - throw new Error('worker environment reservation is invalid'); - } + return { env, identityBody: stableStringify(identity) }; +} + +function workerEnvironmentRoots() { const configuredRuntimeRoot = path.resolve( - process.env.FKST_RUNTIME_ROOT || path.join('.testing', 'runtime'), + process.env.FKST_WORKER_RUNTIME_ROOT + || process.env.FKST_RUNTIME_ROOT + || path.join('.testing', 'runtime'), ); - fs.mkdirSync(configuredRuntimeRoot, { recursive: true }); - const runtimeRoot = requireOwnedDirectory(configuredRuntimeRoot); + fs.mkdirSync(configuredRuntimeRoot, { recursive: true, mode: 0o700 }); + const runtimeRoot = requireOwnedDirectory(configuredRuntimeRoot, { privateDirectory: true }); const homesRoot = requireOwnedDirectory(path.join(runtimeRoot, 'worker-homes'), { privateDirectory: true }); - const identitySha256 = sha256(identityBody); - const leaseId = reservationId || crypto.randomBytes(16).toString('hex'); - const home = reservationId === null - ? fs.mkdtempSync(path.join(homesRoot, `${identitySha256.slice(0, 24)}-`)) - : path.join(homesRoot, `reserved-${reservationId}`); - let homeCreated = reservationId === null; - if (reservationId !== null) { - try { - fs.mkdirSync(home, { mode: 0o700 }); - homeCreated = true; - } catch (error) { - if (error.code !== 'EEXIST') throw error; - requireOwnedDirectory(home, { privateDirectory: true }); - } - } - if (process.platform !== 'win32') fs.chmodSync(home, 0o700); - const marker = `${stableStringify({ + return { runtimeRoot, homesRoot }; +} + +function workerHomeMarker(identitySha256, leaseId) { + return `${stableStringify({ schema: 'fkst.worker-home-identity.v1', identity_sha256: identitySha256, lease_id: leaseId, })}\n`; - const markerPath = path.join(home, '.fkst-worker-home.json'); - const pendingMarkerPath = `${markerPath}.pending`; - if (!homeCreated) { - if (fs.existsSync(markerPath)) { - const stat = fs.lstatSync(markerPath); - if (!stat.isFile() || stat.isSymbolicLink() || fs.readFileSync(markerPath, 'utf8') !== marker) { - throw new Error('worker environment reservation binding changed'); - } - } else { - const entries = fs.readdirSync(home); - if (entries.some((entry) => entry !== path.basename(pendingMarkerPath))) { - throw new Error('worker environment reservation is not recoverable'); - } - if (fs.existsSync(pendingMarkerPath)) { - const pending = fs.lstatSync(pendingMarkerPath); - if (!pending.isFile() || pending.isSymbolicLink()) { - throw new Error('worker environment reservation marker is invalid'); - } - fs.unlinkSync(pendingMarkerPath); - } - } +} + +function buildWorkerEnvironmentReservation(identityBody, homesRoot, reservationId) { + if (reservationId !== null && !/^[0-9a-f]{32}$/.test(String(reservationId))) { + throw new Error('worker environment reservation is invalid'); } - if (!fs.existsSync(markerPath)) { - fs.writeFileSync(pendingMarkerPath, marker, { flag: 'wx', mode: 0o600 }); - fs.renameSync(pendingMarkerPath, markerPath); + const identitySha256 = sha256(identityBody); + const leaseId = reservationId || crypto.randomBytes(16).toString('hex'); + const home = path.join(homesRoot, `reserved-${leaseId}`); + const marker = workerHomeMarker(identitySha256, leaseId); + return { + schema: 'fkst.worker-home-reservation.v1', + lease_id: leaseId, + home, + homes_root: homesRoot, + homes_root_identity: pathIdentity(homesRoot), + marker_sha256: sha256(marker), + identity_sha256: identitySha256, + cleanup_capture_id: sha256(`worker-home-cleanup\0${marker}`), + }; +} + +function workerEnvironmentReservation(extra, isolationKey, reservationId) { + if (!/^[0-9a-f]{32}$/.test(String(reservationId || ''))) { + throw new Error('durable worker environment reservation is invalid'); + } + const { identityBody } = workerEnvironmentInputs(extra, isolationKey); + const { homesRoot } = workerEnvironmentRoots(); + return buildWorkerEnvironmentReservation(identityBody, homesRoot, reservationId); +} + +function reservationMatchesLease(reservation, lease) { + return Boolean(reservation && lease + && reservation.schema === 'fkst.worker-home-reservation.v1' + && lease.schema === 'fkst.worker-home-lease.v1' + && reservation.lease_id === lease.lease_id + && reservation.home === lease.home + && reservation.homes_root === lease.homes_root + && samePathIdentity(reservation.homes_root_identity, lease.homes_root_identity) + && reservation.marker_sha256 === lease.marker_sha256 + && reservation.identity_sha256 === lease.identity_sha256 + && reservation.cleanup_capture_id === lease.cleanup_capture_id); +} + +function verifyWorkerEnvironmentReservation(reservation) { + if (!reservation || reservation.schema !== 'fkst.worker-home-reservation.v1' + || !/^[0-9a-f]{32}$/.test(String(reservation.lease_id || '')) + || typeof reservation.home !== 'string' || !path.isAbsolute(reservation.home) + || typeof reservation.homes_root !== 'string' || !path.isAbsolute(reservation.homes_root) + || !/^[0-9a-f]{64}$/.test(String(reservation.marker_sha256 || '')) + || !/^[0-9a-f]{64}$/.test(String(reservation.identity_sha256 || '')) + || !/^[0-9a-f]{64}$/.test(String(reservation.cleanup_capture_id || '')) + || path.dirname(reservation.home) !== reservation.homes_root + || path.basename(reservation.home) !== `reserved-${reservation.lease_id}` + || !samePathIdentity(pathIdentity(reservation.homes_root), reservation.homes_root_identity)) { + throw new Error('worker environment reservation binding changed'); + } + const marker = workerHomeMarker(reservation.identity_sha256, reservation.lease_id); + if (sha256(marker) !== reservation.marker_sha256 + || sha256(`worker-home-cleanup\0${marker}`) !== reservation.cleanup_capture_id) { + throw new Error('worker environment reservation marker binding changed'); + } + return reservation; +} + +function releaseWorkerEnvironmentReservation(reservation) { + verifyWorkerEnvironmentReservation(reservation); + // A missing pathname is not proof that allocation never published. The + // directory may have been displaced before its inode-bearing lease was + // durably recorded, so cleanup must retain the reservation for audit. + if (!pathEntryExists(reservation.home)) return false; + const marker = workerHomeMarker(reservation.identity_sha256, reservation.lease_id); + const markerPath = path.join(reservation.home, '.fkst-worker-home.json'); + if (!pathEntryExists(markerPath) + || readBoundedRegularFile(markerPath, MAX_LOCK_METADATA_BYTES).body !== marker) { + throw new Error('worker environment reservation is not recoverable'); + } + const lease = { + schema: 'fkst.worker-home-lease.v1', + lease_id: reservation.lease_id, + home: reservation.home, + home_identity: pathIdentity(reservation.home), + homes_root: reservation.homes_root, + homes_root_identity: reservation.homes_root_identity, + marker_sha256: reservation.marker_sha256, + identity_sha256: reservation.identity_sha256, + cleanup_capture_id: reservation.cleanup_capture_id, + }; + verifyWorkerEnvironmentLease(lease); + return releaseWorkerEnvironmentLease(lease); +} + +function minimalEnvironment(extra = {}, isolationKey = 'shared-runtime-command', reservationId = null, hooks = {}) { + const { env, identityBody } = workerEnvironmentInputs(extra, isolationKey); + const { homesRoot } = workerEnvironmentRoots(); + const reservation = buildWorkerEnvironmentReservation(identityBody, homesRoot, reservationId); + const identitySha256 = reservation.identity_sha256; + const leaseId = reservation.lease_id; + const home = reservation.home; + const marker = workerHomeMarker(identitySha256, leaseId); + const homeIdentity = allocateOwnedDirectory( + home, + reservation.homes_root, + reservation.homes_root_identity, + reservation.cleanup_capture_id, + '.fkst-worker-home.json', + marker, + ['.config', '.config/gh'], + ); + if (typeof hooks.afterHomeDirectoryCreated === 'function') { + hooks.afterHomeDirectoryCreated({ home, reservation: { ...reservation } }); + } + if (!samePathIdentity(pathIdentity(home), homeIdentity)) { + throw new Error('worker environment home identity changed after allocation'); } const configHome = path.join(home, '.config'); - requireOwnedDirectory(configHome, { privateDirectory: true }); - requireOwnedDirectory(path.join(configHome, 'gh'), { privateDirectory: true }); const nullDevice = process.platform === 'win32' ? 'NUL' : '/dev/null'; Object.assign(env, { HOME: home, @@ -640,11 +1169,12 @@ function minimalEnvironment(extra = {}, isolationKey = 'shared-runtime-command', schema: 'fkst.worker-home-lease.v1', lease_id: leaseId, home, - home_identity: pathIdentity(home), + home_identity: homeIdentity, homes_root: homesRoot, homes_root_identity: pathIdentity(homesRoot), marker_sha256: sha256(marker), identity_sha256: identitySha256, + cleanup_capture_id: reservation.cleanup_capture_id, released: false, }, }); @@ -665,6 +1195,7 @@ function workerEnvironmentLease(environment) { homes_root_identity: { ...lease.homes_root_identity }, marker_sha256: lease.marker_sha256, identity_sha256: lease.identity_sha256, + cleanup_capture_id: lease.cleanup_capture_id, }; } @@ -672,17 +1203,20 @@ function verifyWorkerEnvironmentLease(lease) { if (!lease || lease.schema !== 'fkst.worker-home-lease.v1' || typeof lease.lease_id !== 'string' || !/^[0-9a-f]{32}$/.test(lease.lease_id) || typeof lease.home !== 'string' || typeof lease.homes_root !== 'string' - || !samePathIdentity(pathIdentity(lease.homes_root), lease.homes_root_identity) - || !samePathIdentity(pathIdentity(lease.home), lease.home_identity)) { + || !/^[0-9a-f]{64}$/.test(String(lease.cleanup_capture_id || ''))) { throw new Error('worker environment lease identity changed'); } + let identitiesMatch = false; + try { + identitiesMatch = samePathIdentity(pathIdentity(lease.homes_root), lease.homes_root_identity) + && samePathIdentity(pathIdentity(lease.home), lease.home_identity); + } catch (_error) {} + if (!identitiesMatch) throw new Error('worker environment lease identity changed'); const root = fs.realpathSync(lease.homes_root); const home = fs.realpathSync(lease.home); if (!home.startsWith(`${root}${path.sep}`)) throw new Error('worker environment home escaped its lease root'); const markerPath = path.join(home, '.fkst-worker-home.json'); - const markerStat = fs.lstatSync(markerPath); - if (!markerStat.isFile() || markerStat.isSymbolicLink()) throw new Error('worker environment marker is invalid'); - const marker = fs.readFileSync(markerPath, 'utf8'); + const marker = readBoundedRegularFile(markerPath, MAX_LOCK_METADATA_BYTES).body; if (sha256(marker) !== lease.marker_sha256) throw new Error('worker environment marker changed'); const value = JSON.parse(marker); if (value.schema !== 'fkst.worker-home-identity.v1' @@ -690,6 +1224,9 @@ function verifyWorkerEnvironmentLease(lease) { || value.lease_id !== lease.lease_id) { throw new Error('worker environment marker binding changed'); } + if (lease.cleanup_capture_id !== sha256(`worker-home-cleanup\0${marker}`)) { + throw new Error('worker environment cleanup capture binding changed'); + } for (const directory of [path.join(home, '.config'), path.join(home, '.config', 'gh')]) { const stat = fs.lstatSync(directory); if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error('worker environment config directory changed'); @@ -711,9 +1248,17 @@ function releaseWorkerEnvironmentLease(lease) { if (!lease || lease.schema !== 'fkst.worker-home-lease.v1') { throw new Error('worker environment lease is invalid'); } - if (!fs.existsSync(lease.home)) return true; - verifyWorkerEnvironmentLease(lease); - return removeOwnedDirectory(lease.home, lease.home_identity, lease.homes_root); + if (pathEntryExists(lease.home)) verifyWorkerEnvironmentLease(lease); + return removeOwnedDirectory( + lease.home, lease.home_identity, lease.homes_root, lease.cleanup_capture_id, + ); +} + +function workerEnvironmentReleaseProven(lease) { + if (!lease || lease.schema !== 'fkst.worker-home-lease.v1') return false; + return ownedDirectoryReleaseProven( + lease.home, lease.home_identity, lease.homes_root, lease.cleanup_capture_id, + ); } function releaseWorkerEnvironment(environment) { @@ -728,27 +1273,18 @@ function releaseWorkerEnvironment(environment) { function commandResult(argv, options = {}) { validateArgv(argv); const outputBytes = Math.max(1, Math.min(Number(options.outputBytes) || DEFAULT_OUTPUT_BYTES, MAX_JSON_BYTES)); - const ownedEnvironment = options.env === undefined; - const environment = options.env || minimalEnvironment(); - try { - verifyWorkerEnvironment(environment); - const result = spawnSync(argv[0], argv.slice(1), { - cwd: options.cwd, - env: environment, - shell: false, - encoding: 'utf8', - timeout: Math.max(1, Number(options.timeoutMs) || 30_000), - maxBuffer: outputBytes, - }); - return { - exitCode: Number.isInteger(result.status) ? result.status : -1, - stdout: String(result.stdout || '').slice(0, outputBytes), - stderr: boundedText(result.stderr || (result.error && result.error.message), outputBytes), - error: result.error, - }; - } finally { - if (ownedEnvironment) releaseWorkerEnvironment(environment); - } + if (options.env === undefined) throw new Error('commandResult requires a durable worker environment'); + verifyWorkerEnvironment(options.env); + const result = spawnSync(argv[0], argv.slice(1), { + cwd: options.cwd, env: options.env, shell: false, encoding: 'utf8', + timeout: Math.max(1, Number(options.timeoutMs) || 30_000), maxBuffer: outputBytes, + }); + return { + exitCode: Number.isInteger(result.status) ? result.status : -1, + stdout: String(result.stdout || '').slice(0, outputBytes), + stderr: boundedText(result.stderr || (result.error && result.error.message), outputBytes), + error: result.error, + }; } function sameArray(left, right) { @@ -758,21 +1294,30 @@ function sameArray(left, right) { module.exports = { DEFAULT_OUTPUT_BYTES, MAX_JSON_BYTES, + OBJECT_BOUND_CLEANUP_UNAVAILABLE, acquireLock, + allocateOwnedDirectory, artifactPath, authorizationArtifact, boundedText, commandResult, isSafeArtifactPath, minimalEnvironment, + ownedDirectoryReleaseProven, parseArgs, + pathEntryExists, pathIdentity, processAlive, processStartIdentity, readJson, + readBoundedRegularFile, + requireOwnedDirectory, removeOwnedDirectory, + retireOwnedDirectoryMarker, releaseWorkerEnvironment, releaseWorkerEnvironmentLease, + releaseWorkerEnvironmentReservation, + reservationMatchesLease, runtimeConfig, sameArray, samePathIdentity, @@ -783,7 +1328,10 @@ module.exports = { validateArgv, verifyWorkerEnvironment, verifyWorkerEnvironmentLease, + verifyWorkerEnvironmentReservation, + workerEnvironmentReservation, workerEnvironmentLease, + workerEnvironmentReleaseProven, writeJsonAtomic, writeJsonImmutable, }; diff --git a/packages/environment-factory/bin/runtime/lock-holder.js b/packages/environment-factory/bin/runtime/lock-holder.js index d7810aec..9b10b503 100644 --- a/packages/environment-factory/bin/runtime/lock-holder.js +++ b/packages/environment-factory/bin/runtime/lock-holder.js @@ -2,11 +2,39 @@ const fs = require('fs'); +const MAX_MARKER_BYTES = 8 * 1024; + function fail(message) { process.stderr.write(`lock-holder: ${message}\n`); process.exit(2); } +function readMarker(markerPath) { + const fd = fs.openSync(markerPath, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)); + try { + const before = fs.fstatSync(fd, { bigint: true }); + const linked = fs.lstatSync(markerPath, { bigint: true }); + if (!before.isFile() || !linked.isFile() || linked.isSymbolicLink() + || before.dev !== linked.dev || before.ino !== linked.ino + || before.size > BigInt(MAX_MARKER_BYTES)) fail('takeover marker is invalid'); + const buffer = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < buffer.length) { + const count = fs.readSync(fd, buffer, offset, buffer.length - offset, offset); + if (count === 0) break; + offset += count; + } + const after = fs.fstatSync(fd, { bigint: true }); + if (offset !== buffer.length || after.size !== before.size + || after.mtimeNs !== before.mtimeNs || after.ctimeNs !== before.ctimeNs) { + fail('takeover marker changed while reading'); + } + return JSON.parse(buffer.toString('utf8')); + } finally { + fs.closeSync(fd); + } +} + function removeStaleMarker(markerPath) { try { const stat = fs.lstatSync(markerPath); @@ -47,7 +75,7 @@ function cleanup() { if (cleaned) return; cleaned = true; try { - const marker = JSON.parse(fs.readFileSync(markerPath, 'utf8')); + const marker = readMarker(markerPath); if (marker.pid === process.pid && marker.token === token) fs.unlinkSync(markerPath); } catch (_error) {} if (darwinGuard !== null) fs.closeSync(darwinGuard); diff --git a/packages/environment-factory/bin/runtime/measured-command.js b/packages/environment-factory/bin/runtime/measured-command.js index aa39c634..cc559b7b 100644 --- a/packages/environment-factory/bin/runtime/measured-command.js +++ b/packages/environment-factory/bin/runtime/measured-command.js @@ -5,6 +5,11 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); const { spawn } = require('child_process'); +const { + closeDirectoryAnchor, + objectBoundExec, + openDirectoryAnchor, +} = require('./object-bound-exec'); const { processGroupUsage } = require('./platform'); function parseMetrics(stderr) { @@ -54,13 +59,27 @@ function runMeasuredCommand(argv, options = {}) { let timedOut = false; let settled = false; let maxProcesses = 0; - const child = spawn(timed[0], timed[1], { - cwd: options.cwd, - env: options.env, - shell: false, - detached: true, - stdio: ['ignore', 'pipe', 'pipe'], - }); + let anchor; + let child; + try { + anchor = openDirectoryAnchor(options.cwd, options.cwdIdentity); + if (typeof options.afterCwdAnchored === 'function') options.afterCwdAnchored(anchor); + const launch = objectBoundExec(anchor, [timed[0], ...timed[1]], 3); + child = spawn(launch.command, launch.argv, { + cwd: '/', + env: options.env, + shell: false, + detached: true, + stdio: ['ignore', 'pipe', 'pipe', anchor.descriptor], + }); + closeDirectoryAnchor(anchor); + } catch (error) { + closeDirectoryAnchor(anchor); + fs.rmSync(metricsPath, { force: true }); + resolve({ exitCode: -1, metricsSupported: false, processMetricsSupported: false, + stdout: '', stderr: String(error.message || error), error }); + return; + } const stop = () => { if (!Number.isInteger(child.pid)) return; try { process.kill(-child.pid, 'SIGKILL'); } catch (_error) { diff --git a/packages/environment-factory/bin/runtime/object-bound-exec.js b/packages/environment-factory/bin/runtime/object-bound-exec.js new file mode 100644 index 00000000..036d38b6 --- /dev/null +++ b/packages/environment-factory/bin/runtime/object-bound-exec.js @@ -0,0 +1,77 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const PYTHON = '/usr/bin/python3'; +const EXEC_SOURCE = [ + 'import os, stat, sys', + 'fd = int(sys.argv[1])', + 'expected_device, expected_inode = sys.argv[2], sys.argv[3]', + 'info = os.fstat(fd)', + 'if not stat.S_ISDIR(info.st_mode): raise SystemExit(126)', + 'if str(info.st_dev) != expected_device or str(info.st_ino) != expected_inode: raise SystemExit(126)', + 'argv = sys.argv[4:]', + 'if not argv: raise SystemExit(126)', + 'os.fchdir(fd)', + 'os.close(fd)', + 'os.execvpe(argv[0], argv, os.environ)', +].join('\n'); + +function sameObjectIdentity(left, right) { + return Boolean(left && right + && String(left.device) === String(right.device) + && String(left.inode) === String(right.inode)); +} + +function openDirectoryAnchor(cwd, expectedIdentity) { + if (!['darwin', 'linux'].includes(process.platform) || !fs.existsSync(PYTHON)) { + throw new Error('object-bound process cwd is unsupported on this platform'); + } + const target = path.resolve(cwd || process.cwd()); + const flags = fs.constants.O_RDONLY | (fs.constants.O_DIRECTORY || 0) + | (fs.constants.O_NOFOLLOW || 0); + const descriptor = fs.openSync(target, flags); + try { + const stat = fs.fstatSync(descriptor); + if (!stat.isDirectory()) throw new Error('process cwd is not a directory'); + const identity = { + device: String(stat.dev), inode: String(stat.ino), size: stat.size, mode: stat.mode, + }; + if (expectedIdentity && !sameObjectIdentity(identity, expectedIdentity)) { + throw new Error('process cwd object identity changed'); + } + return { descriptor, identity, target }; + } catch (error) { + fs.closeSync(descriptor); + throw error; + } +} + +function objectBoundExec(anchor, argv, childDescriptor) { + if (!anchor || !Number.isInteger(anchor.descriptor) || !anchor.identity + || !Array.isArray(argv) || argv.length === 0 + || !Number.isInteger(childDescriptor) || childDescriptor < 3) { + throw new Error('object-bound process launch is invalid'); + } + return { + command: PYTHON, + argv: [ + '-I', '-c', EXEC_SOURCE, String(childDescriptor), + anchor.identity.device, anchor.identity.inode, ...argv, + ], + }; +} + +function closeDirectoryAnchor(anchor) { + if (!anchor || !Number.isInteger(anchor.descriptor)) return; + try { fs.closeSync(anchor.descriptor); } catch (_error) {} + anchor.descriptor = null; +} + +module.exports = { + closeDirectoryAnchor, + objectBoundExec, + openDirectoryAnchor, + sameObjectIdentity, +}; diff --git a/packages/environment-factory/bin/runtime/supervised-process.js b/packages/environment-factory/bin/runtime/supervised-process.js index 57faa537..0d9d02d4 100644 --- a/packages/environment-factory/bin/runtime/supervised-process.js +++ b/packages/environment-factory/bin/runtime/supervised-process.js @@ -7,6 +7,7 @@ const { spawn } = require('child_process'); const { acquireLock, boundedText, + pathEntryExists, processAlive, processStartIdentity, releaseWorkerEnvironmentLease, @@ -19,6 +20,12 @@ const { workerEnvironmentLease, writeJsonAtomic, } = require('./common'); +const { + closeDirectoryAnchor, + objectBoundExec, + openDirectoryAnchor, + sameObjectIdentity, +} = require('./object-bound-exec'); const CLAIM_SCHEMA = 'fkst.supervised-process-startup.v1'; const SPEC_SCHEMA = 'fkst.supervised-process-launch.v1'; @@ -113,8 +120,11 @@ function validClaim(claim, bindingSha256) { })) && (leasePending ? claim.worker_environment_lease === null && claim.worker_environment_lease_sha256 === null + && (claim.worker_home_slot_id === null || claim.worker_home_slot_id === undefined) : claim.worker_environment_lease - && /^[0-9a-f]{64}$/.test(String(claim.worker_environment_lease_sha256 || ''))) + && /^[0-9a-f]{64}$/.test(String(claim.worker_environment_lease_sha256 || '')) + && (claim.worker_home_slot_id == null + || /^[0-9a-f]{64}$/.test(String(claim.worker_home_slot_id)))) && ['allocating', 'preparing', 'prepared', 'registered', 'running', 'exited', 'failed', 'revoked'].includes(state) && (state === 'allocating' || state === 'preparing' || sameFileIdentityShape(claim.launch_spec_identity))); } @@ -141,12 +151,6 @@ function sameFileIdentity(left, right) { && left.size === right.size && left.mode === right.mode; } -function sameNodeIdentity(left, right) { - return sameFileIdentityShape(left) && sameFileIdentityShape(right) - && left.device === right.device && left.inode === right.inode - && (left.mode & fs.constants.S_IFMT) === (right.mode & fs.constants.S_IFMT); -} - function descriptorIdentities(descriptors) { return descriptors.map((descriptor) => fileIdentity(fs.fstatSync(descriptor))); } @@ -227,6 +231,7 @@ function resourceFromClaim(binding, claim) { pgid: Number.isInteger(claim.pgid) ? claim.pgid : null, process_start_identity: typeof claim.process_start_identity === 'string' ? claim.process_start_identity : null, + worker_home_slot_id: claim.worker_home_slot_id || null, worker_environment_lease: claim.worker_environment_lease, }; } @@ -244,6 +249,7 @@ function launchInvariantSha256(claim) { inherited_fd_count: claim.inherited_fd_count, inherited_fd_identities: claim.inherited_fd_identities, worker_environment_reservation: claim.worker_environment_reservation, + worker_home_slot_id: claim.worker_home_slot_id || null, worker_environment_lease_sha256: claim.worker_environment_lease_sha256, })); } @@ -257,6 +263,9 @@ function startOrRecoverSupervisedProcess(options) { if (!stat.isDirectory()) throw new Error('supervised process cwd is not a directory'); return fileIdentity(stat); })(); + if (options.cwdIdentity && !sameObjectIdentity(cwdIdentity, options.cwdIdentity)) { + throw new Error('supervised process cwd object identity changed'); + } const binding = options.binding; if (!binding || typeof binding !== 'object' || Array.isArray(binding)) { throw new Error('supervised process binding is invalid'); @@ -319,6 +328,7 @@ function startOrRecoverSupervisedProcess(options) { inherited_fd_count: inheritedStdio.length, inherited_fd_identities: inheritedFdIdentities, worker_environment_reservation: reservation, + worker_home_slot_id: null, worker_environment_lease: null, worker_environment_lease_sha256: null, pid: null, @@ -333,10 +343,21 @@ function startOrRecoverSupervisedProcess(options) { if (options.environment || typeof options.createEnvironment !== 'function') { throw new Error('recoverable supervised process environment factory is required'); } - environment = options.createEnvironment({ + const supplied = options.createEnvironment({ reservation_id: claim.worker_environment_reservation.reservation_id, binding_sha256: claim.worker_environment_reservation.binding_sha256, }); + let workerHomeSlotId = null; + if (supplied && typeof supplied === 'object' && !Array.isArray(supplied) + && Object.prototype.hasOwnProperty.call(supplied, 'environment')) { + environment = supplied.environment; + workerHomeSlotId = supplied.worker_home_slot_id; + if (!/^[0-9a-f]{64}$/.test(String(workerHomeSlotId || ''))) { + throw new Error('supervised worker-home slot identity is invalid'); + } + } else { + environment = supplied; + } if (!environment) throw new Error('supervised process environment is required for a new launch'); verifyWorkerEnvironment(environment); suppliedLease = workerEnvironmentLease(environment); @@ -353,11 +374,13 @@ function startOrRecoverSupervisedProcess(options) { claim = { ...claim, state: 'preparing', + worker_home_slot_id: workerHomeSlotId, worker_environment_lease: suppliedLease, worker_environment_lease_sha256: sha256(stableStringify(suppliedLease)), }; writeClaimAtomic(claimPath, claim); - } else if (stableStringify(suppliedLease) !== stableStringify(claim.worker_environment_lease)) { + } else if (stableStringify(suppliedLease) !== stableStringify(claim.worker_environment_lease) + || (claim.worker_home_slot_id != null && workerHomeSlotId !== claim.worker_home_slot_id)) { releaseWorkerEnvironmentLease(suppliedLease); suppliedLease = null; throw new Error('supervised worker environment lease differs'); @@ -463,11 +486,13 @@ function startOrRecoverSupervisedProcess(options) { revoke(); } if (revoked.state === 'revoked') releaseWorkerEnvironmentLease(revoked.worker_environment_lease); + const environmentRetained = Boolean(revoked.worker_environment_lease + && pathEntryExists(revoked.worker_environment_lease.home)); return { interrupted: false, state: revoked.state, resource: resourceFromClaim(binding, revoked), - environment_retained: false, + environment_retained: environmentRetained, }; } @@ -499,7 +524,6 @@ function childMain(specPath, expectedSpecSha256, expectedLaunchInvariantSha256) || sha256(`${stableStringify(spec)}\n`) !== initialClaim.launch_spec_sha256 || sha256(stableStringify(validateArgv(spec.argv))) !== initialClaim.argv_sha256 || path.resolve(spec.cwd) !== initialClaim.cwd - || !sameNodeIdentity(fileIdentity(fs.statSync(spec.cwd)), initialClaim.cwd_identity) || spec.inherited_fd_count !== initialClaim.inherited_fd_count || stableStringify(spec.inherited_fd_identities) !== stableStringify(initialClaim.inherited_fd_identities) || sha256(stableStringify(initialClaim.worker_environment_lease)) @@ -516,6 +540,7 @@ function childMain(specPath, expectedSpecSha256, expectedLaunchInvariantSha256) const release = acquireLock(`${claimPath}.lock`); let child; let inherited = []; + let cwdAnchor = null; const closeInherited = () => { for (const fd of inherited) { try { fs.closeSync(fd); } catch (_error) {} @@ -528,6 +553,7 @@ function childMain(specPath, expectedSpecSha256, expectedLaunchInvariantSha256) || claim.startup_token !== spec.startup_token || claim.launch_spec_path !== absoluteSpec) { throw new Error('supervised startup claim is unavailable or revoked'); } + cwdAnchor = openDirectoryAnchor(spec.cwd, claim.cwd_identity); const identity = processStartIdentity(process.pid); if (identity === null) throw new Error('supervised process identity is unavailable'); writeClaimAtomic(claimPath, { @@ -539,13 +565,16 @@ function childMain(specPath, expectedSpecSha256, expectedLaunchInvariantSha256) registered_at_epoch_ms: Date.now(), }); inherited = Array.from({ length: spec.inherited_fd_count }, (_item, index) => index + 3); - child = spawn(validateArgv(spec.argv)[0], spec.argv.slice(1), { - cwd: path.resolve(spec.cwd), + const cwdChildDescriptor = 3 + inherited.length; + const launch = objectBoundExec(cwdAnchor, validateArgv(spec.argv), cwdChildDescriptor); + child = spawn(launch.command, launch.argv, { + cwd: '/', env: process.env, shell: false, detached: false, - stdio: ['ignore', 'ignore', 'ignore', ...inherited], + stdio: ['ignore', 'ignore', 'ignore', ...inherited, cwdAnchor.descriptor], }); + closeDirectoryAnchor(cwdAnchor); closeInherited(); child.once('error', (error) => { transitionClaim(claimPath, spec.startup_token, ['registered', 'running'], { @@ -562,6 +591,7 @@ function childMain(specPath, expectedSpecSha256, expectedLaunchInvariantSha256) started_at_epoch_ms: Date.now(), }); } finally { + closeDirectoryAnchor(cwdAnchor); closeInherited(); release(); } diff --git a/packages/environment-factory/bin/runtime/target-execution-boundary.js b/packages/environment-factory/bin/runtime/target-execution-boundary.js index 2f7d4056..c2c96de6 100644 --- a/packages/environment-factory/bin/runtime/target-execution-boundary.js +++ b/packages/environment-factory/bin/runtime/target-execution-boundary.js @@ -59,19 +59,23 @@ function assertConfigOutsideOperationArtifacts(runtimeConfigRef, artifactRoot) { function validateTargetExecutionBoundary(boundary, repository, context = {}) { if (boundary && typeof boundary === 'object' && !Array.isArray(boundary) - && (boundary.authorization_capability === true || boundary.execution_authorized === true)) { + && (boundary.human_approval_required === true || boundary.authorization_capability === true + || boundary.execution_authorized === true + || boundary.promotion_authorized === true)) { throw new Error('target execution boundary must remain a non-authorizing admission prerequisite'); } if (!boundary || typeof boundary !== 'object' || Array.isArray(boundary) || Object.keys(boundary).sort().join(',') !== [ 'schema', 'mode', 'target_class', 'repository', 'authority', 'policy_revision', - 'authorization_capability', 'execution_authorized', + 'human_approval_required', 'authorization_capability', 'execution_authorized', + 'promotion_authorized', ].sort().join(',')) { throw new Error('HOST_RUNTIME_ISOLATION_REQUIRED: target execution boundary is missing or malformed'); } exactKeys(boundary, [ 'schema', 'mode', 'target_class', 'repository', 'authority', 'policy_revision', - 'authorization_capability', 'execution_authorized', + 'human_approval_required', 'authorization_capability', 'execution_authorized', + 'promotion_authorized', ], 'target execution boundary'); if (boundary.schema !== BOUNDARY_SCHEMA) { @@ -83,7 +87,9 @@ function validateTargetExecutionBoundary(boundary, repository, context = {}) { if (boundary.target_class !== 'host-owned-exact-trusted-fixture') { throw new Error('HOST_RUNTIME_ISOLATION_REQUIRED: target is not a Host-owned exact trusted fixture'); } - if (boundary.authorization_capability !== false || boundary.execution_authorized !== false) { + if (boundary.human_approval_required !== false || boundary.authorization_capability !== false + || boundary.execution_authorized !== false + || boundary.promotion_authorized !== false) { throw new Error('target execution boundary must remain a non-authorizing admission prerequisite'); } if (!validRepository(boundary.repository)) { diff --git a/packages/environment-factory/bin/runtime/worker-home-resource.js b/packages/environment-factory/bin/runtime/worker-home-resource.js new file mode 100644 index 00000000..0b1e0479 --- /dev/null +++ b/packages/environment-factory/bin/runtime/worker-home-resource.js @@ -0,0 +1,418 @@ +'use strict'; + +const path = require('path'); +const { + acquireLock, + artifactPath, + minimalEnvironment, + readJson, + releaseWorkerEnvironmentLease, + releaseWorkerEnvironmentReservation, + reservationMatchesLease, + sha256, + stableStringify, + verifyWorkerEnvironmentLease, + workerEnvironmentReservation, + workerEnvironmentLease, + writeJsonAtomic, + writeJsonImmutable, +} = require('./common'); + +const LEDGER_SCHEMA = 'environment-factory.worker-home-ledger.v1'; +const RETENTION_SCHEMA = 'environment-factory.worker-home-retention.v1'; +const RESOURCE_SCHEMA = 'environment-factory.resource.v1'; +const MAX_ENTRIES = 256; +const MAX_LEDGER_BYTES = 1024 * 1024; + +function durableRoot() { + return path.resolve(process.env.FKST_DURABLE_ROOT || path.join('.testing', 'durable')); +} + +function privatePath(kind, value) { + return path.join(durableRoot(), 'environment-factory', kind, `${sha256(String(value))}.json`); +} + +function readIfExists(filePath) { + try { + return readJson(filePath); + } catch (error) { + if (error.code === 'ENOENT') return null; + throw error; + } +} + +function exactRepository(value) { + if (!value || typeof value.url !== 'string' || value.url === '' + || !/^[0-9a-f]{40}$/.test(String(value.commit_sha || ''))) { + throw new Error('worker-home ledger requires an exact repository identity'); + } + return { url: value.url, commit_sha: value.commit_sha }; +} + +function ledgerIdentity(request) { + if (!request || typeof request.operation_id !== 'string' || request.operation_id === '') { + throw new Error('worker-home ledger requires operation_id'); + } + const repository = exactRepository(request.repository); + const binding = { + schema: 'environment-factory.worker-home-ledger-binding.v1', + operation_id: request.operation_id, + repository, + }; + const ledgerId = sha256(stableStringify(binding)); + const ref = `environment-factory-worker-home-ledger-${ledgerId.slice(0, 32)}`; + return { + binding, + ledgerId, + repository, + cleanupRef: { kind: 'resource-cleanup', ref }, + ledgerPath: privatePath('worker-home-ledgers', ref), + resourcePath: privatePath('resources', ref), + }; +} + +function writeLedger(filePath, value) { + const body = `${stableStringify(value)}\n`; + if (Buffer.byteLength(body) > MAX_LEDGER_BYTES) { + const error = new Error('WORKER_HOME_LEDGER_CAPACITY_EXCEEDED'); + error.code = 'WORKER_HOME_LEDGER_CAPACITY_EXCEEDED'; + throw error; + } + writeJsonAtomic(filePath, value); +} + +function verifyLedger(value, identity) { + if (!value || value.schema !== LEDGER_SCHEMA || value.ledger_id !== identity.ledgerId + || value.operation_id !== identity.binding.operation_id + || stableStringify(value.repository) !== stableStringify(identity.repository) + || value.max_entries !== MAX_ENTRIES || !Number.isInteger(value.revision) + || value.revision < 1 || !Array.isArray(value.entries) || value.entries.length > MAX_ENTRIES) { + throw new Error('worker-home ledger binding differs'); + } + return value; +} + +function initializeWorkerHomeLedger(request) { + const identity = ledgerIdentity(request); + const release = acquireLock(`${identity.ledgerPath}.lock`); + try { + const existing = readIfExists(identity.ledgerPath); + if (existing) { + verifyLedger(existing, identity); + } else { + writeLedger(identity.ledgerPath, { + schema: LEDGER_SCHEMA, + ledger_id: identity.ledgerId, + operation_id: request.operation_id, + repository: identity.repository, + revision: 1, + max_entries: MAX_ENTRIES, + entries: [], + }); + } + const resource = { + schema: RESOURCE_SCHEMA, + kind: 'worker-home-ledger', + operation_id: request.operation_id, + ref: identity.cleanupRef.ref, + ledger_id: identity.ledgerId, + repository: identity.repository, + cleaned: false, + }; + const stored = readIfExists(identity.resourcePath); + if (stored && stableStringify(stored) !== stableStringify(resource)) { + throw new Error('worker-home ledger resource binding differs'); + } + if (!stored) writeJsonAtomic(identity.resourcePath, resource); + return { status: 'passed', ledger_id: identity.ledgerId, cleanup_ref: identity.cleanupRef }; + } finally { + release(); + } +} + +function requireLedgerContext(request) { + const identity = ledgerIdentity(request); + if (request.worker_home_ledger_ref + && stableStringify(request.worker_home_ledger_ref) !== stableStringify(identity.cleanupRef)) { + throw new Error('worker-home ledger cleanup ref differs'); + } + const resource = readIfExists(identity.resourcePath); + if (!resource || resource.schema !== RESOURCE_SCHEMA || resource.kind !== 'worker-home-ledger' + || resource.operation_id !== request.operation_id || resource.ledger_id !== identity.ledgerId + || stableStringify(resource.repository) !== stableStringify(identity.repository)) { + throw new Error('worker-home ledger resource is unavailable'); + } + return identity; +} + +function slotBinding(request, purpose, extra) { + if (typeof request.effect_id !== 'string' || request.effect_id === '' + || typeof purpose !== 'string' || purpose === '' || purpose.length > 180) { + throw new Error('worker-home slot identity is invalid'); + } + return { + schema: 'environment-factory.worker-home-slot-binding.v1', + operation_id: request.operation_id, + effect_id: request.effect_id, + purpose, + repository: exactRepository(request.repository), + environment_sha256: sha256(stableStringify(extra || {})), + }; +} + +function findSlot(ledger, slotId) { + return ledger.entries.find((entry) => entry.slot_id === slotId) || null; +} + +function allocateDurableWorkerEnvironment( + request, purpose, extra = {}, reservationOverride = null, hooks = {}, +) { + const identity = requireLedgerContext(request); + const binding = slotBinding(request, purpose, extra); + const slotId = sha256(stableStringify(binding)); + const reservationId = reservationOverride === null ? slotId.slice(0, 32) : String(reservationOverride); + if (!/^[0-9a-f]{32}$/.test(reservationId)) { + throw new Error('worker-home slot reservation is invalid'); + } + const isolation = { + schema: 'environment-factory.ledger-worker-isolation.v1', + ledger_id: identity.ledgerId, + slot_id: slotId, + operation_id: request.operation_id, + effect_id: request.effect_id, + purpose, + repository: identity.repository, + }; + const reservation = workerEnvironmentReservation(extra, isolation, reservationId); + let release = acquireLock(`${identity.ledgerPath}.lock`); + try { + const ledger = verifyLedger(readJson(identity.ledgerPath), identity); + const existing = findSlot(ledger, slotId); + if (existing && stableStringify(existing.binding) !== stableStringify(binding)) { + throw new Error('worker-home slot binding differs'); + } + if (existing && existing.reservation_id !== reservationId) { + throw new Error('worker-home slot reservation differs'); + } + if (existing && (!existing.worker_environment_reservation + || stableStringify(existing.worker_environment_reservation) !== stableStringify(reservation))) { + throw new Error('worker-home slot reservation binding differs'); + } + if (!existing) { + if (ledger.entries.length >= ledger.max_entries) { + const error = new Error('WORKER_HOME_LEDGER_CAPACITY_EXCEEDED'); + error.code = 'WORKER_HOME_LEDGER_CAPACITY_EXCEEDED'; + throw error; + } + ledger.entries.push({ + slot_id: slotId, + reservation_id: reservationId, + generation: 1, + binding, + state: 'reserved', + release_reason: null, + worker_environment_reservation: reservation, + worker_environment_lease: null, + }); + ledger.revision += 1; + writeLedger(identity.ledgerPath, ledger); + } else if (existing.state === 'released') { + existing.generation += 1; + existing.state = 'reserved'; + existing.release_reason = null; + existing.worker_environment_reservation = reservation; + existing.worker_environment_lease = null; + ledger.revision += 1; + writeLedger(identity.ledgerPath, ledger); + } + } finally { + release(); + } + + const environment = minimalEnvironment(extra, isolation, reservationId, hooks); + if (typeof hooks.afterEnvironmentCreated === 'function') hooks.afterEnvironmentCreated(environment); + const lease = workerEnvironmentLease(environment); + if (!reservationMatchesLease(reservation, lease)) { + throw new Error('worker-home slot lease differs from its durable reservation'); + } + release = acquireLock(`${identity.ledgerPath}.lock`); + try { + const ledger = verifyLedger(readJson(identity.ledgerPath), identity); + const entry = findSlot(ledger, slotId); + if (!entry || stableStringify(entry.binding) !== stableStringify(binding) + || stableStringify(entry.worker_environment_reservation) !== stableStringify(reservation)) { + throw new Error('worker-home slot reservation is unavailable'); + } + if (entry.worker_environment_lease + && stableStringify(entry.worker_environment_lease) !== stableStringify(lease)) { + throw new Error('worker-home slot lease differs'); + } + entry.worker_environment_lease = lease; + entry.state = 'allocated'; + entry.release_reason = null; + ledger.revision += 1; + writeLedger(identity.ledgerPath, ledger); + verifyWorkerEnvironmentLease(lease); + } finally { + release(); + } + return { environment, identity, lease, slot_id: slotId }; +} + +function recordWorkerEnvironmentRelease(allocation, reason = 'effect-complete') { + if (!allocation || !allocation.identity || !allocation.lease || typeof allocation.slot_id !== 'string') { + throw new Error('worker-home allocation is invalid'); + } + let released = false; + let releaseReason = reason; + try { + released = releaseWorkerEnvironmentLease(allocation.lease); + if (!released) releaseReason = 'OBJECT_BOUND_CLEANUP_UNAVAILABLE'; + } catch (error) { + releaseReason = `release-failed:${String(error && error.message || error).slice(0, 120)}`; + } + const release = acquireLock(`${allocation.identity.ledgerPath}.lock`); + try { + const ledger = verifyLedger(readJson(allocation.identity.ledgerPath), allocation.identity); + const entry = findSlot(ledger, allocation.slot_id); + if (!entry || stableStringify(entry.worker_environment_lease) !== stableStringify(allocation.lease)) { + throw new Error('worker-home release binding differs'); + } + entry.state = released ? 'released' : 'retained'; + entry.release_reason = released ? null : releaseReason; + ledger.revision += 1; + writeLedger(allocation.identity.ledgerPath, ledger); + } finally { + release(); + } + return released; +} + +function recordPersistedWorkerEnvironmentRelease(request, slotId, lease, reason) { + return recordWorkerEnvironmentRelease({ + identity: requireLedgerContext(request), + lease, + slot_id: slotId, + }, reason); +} + +function verifyPersistedWorkerEnvironment(request, purpose, extra, slotId, lease) { + const identity = requireLedgerContext(request); + const binding = slotBinding(request, purpose, extra || {}); + const expectedSlotId = sha256(stableStringify(binding)); + const release = acquireLock(`${identity.ledgerPath}.lock`); + try { + const ledger = verifyLedger(readJson(identity.ledgerPath), identity); + const entry = findSlot(ledger, expectedSlotId); + if (slotId !== expectedSlotId || !entry + || stableStringify(entry.binding) !== stableStringify(binding) + || stableStringify(entry.worker_environment_lease) !== stableStringify(lease) + || (entry.state !== 'allocated' && entry.state !== 'retained')) { + throw new Error('worker-home persisted slot binding differs'); + } + verifyWorkerEnvironmentLease(lease); + return { identity, lease, slot_id: slotId }; + } finally { + release(); + } +} + +async function withDurableWorkerEnvironment(request, purpose, extra, callback) { + const allocation = allocateDurableWorkerEnvironment(request, purpose, extra); + try { + return await callback(allocation.environment, allocation); + } finally { + recordWorkerEnvironmentRelease(allocation); + } +} + +function publicEntry(entry) { + const lease = entry.worker_environment_lease; + const reservation = entry.worker_environment_reservation; + const value = { + slot_id: entry.slot_id, + lease_id: lease ? lease.lease_id : entry.reservation_id, + effect_id: entry.binding.effect_id, + purpose: entry.binding.purpose, + generation: entry.generation, + state: entry.state, + reason: entry.release_reason && entry.release_reason.startsWith('release-failed:') + ? 'release-verification-failed' : (entry.release_reason || 'allocation-not-complete'), + }; + if (lease || reservation) { + value.identity_sha256 = (lease || reservation).identity_sha256; + value.marker_sha256 = (lease || reservation).marker_sha256; + } + return value; +} + +function cleanupWorkerHomeLedger(request, resource) { + const identity = requireLedgerContext({ + ...request, + repository: resource.repository, + worker_home_ledger_ref: request.cleanup_ref, + }); + if (resource.ledger_id !== identity.ledgerId) throw new Error('worker-home cleanup ledger differs'); + const release = acquireLock(`${identity.ledgerPath}.lock`); + let ledger; + try { + ledger = verifyLedger(readJson(identity.ledgerPath), identity); + for (const entry of ledger.entries) { + if (entry.state === 'released') continue; + if (entry.worker_environment_lease) { + try { + const released = releaseWorkerEnvironmentLease(entry.worker_environment_lease); + entry.state = released ? 'released' : 'retained'; + entry.release_reason = released ? null : 'OBJECT_BOUND_CLEANUP_UNAVAILABLE'; + } catch (error) { + entry.state = 'retained'; + entry.release_reason = `release-failed:${String(error && error.message || error).slice(0, 120)}`; + } + } else { + try { + const released = releaseWorkerEnvironmentReservation(entry.worker_environment_reservation); + entry.state = released ? 'released' : 'retained'; + entry.release_reason = released ? null : 'OBJECT_BOUND_CLEANUP_UNAVAILABLE'; + } catch (error) { + entry.state = 'retained'; + entry.release_reason = `release-failed:${String(error && error.message || error).slice(0, 120)}`; + } + } + } + ledger.revision += 1; + writeLedger(identity.ledgerPath, ledger); + } finally { + release(); + } + const remaining = ledger.entries.filter((entry) => entry.state !== 'released').map(publicEntry); + if (remaining.length === 0) return { cleaned: true }; + const ref = { + kind: 'artifact', + ref: `${request.artifact_root}/worker-home-retention-${identity.ledgerId.slice(0, 24)}-r${ledger.revision}.json`, + }; + const snapshot = { + schema: RETENTION_SCHEMA, + operation_id: request.operation_id, + ledger_id: identity.ledgerId, + repository: identity.repository, + remaining_count: remaining.length, + entries: remaining, + }; + writeJsonImmutable(artifactPath(ref), snapshot); + return { + cleaned: false, + resource_detail_ref: ref, + resource_detail_sha256: sha256(`${stableStringify(snapshot)}\n`), + remaining_count: remaining.length, + }; +} + +module.exports = { + allocateDurableWorkerEnvironment, + cleanupWorkerHomeLedger, + initializeWorkerHomeLedger, + recordPersistedWorkerEnvironmentRelease, + recordWorkerEnvironmentRelease, + verifyPersistedWorkerEnvironment, + withDurableWorkerEnvironment, +}; diff --git a/packages/environment-factory/bin/runtime/workspace-reservation.js b/packages/environment-factory/bin/runtime/workspace-reservation.js new file mode 100644 index 00000000..9c1de0a9 --- /dev/null +++ b/packages/environment-factory/bin/runtime/workspace-reservation.js @@ -0,0 +1,122 @@ +'use strict'; + +const path = require('path'); +const { + allocateOwnedDirectory, + pathEntryExists, + pathIdentity, + removeOwnedDirectory, + retireOwnedDirectoryMarker, + samePathIdentity, + sha256, + stableStringify, +} = require('./common'); + +const MARKER_NAME = '.fkst-workspace-reservation.json'; + +function reservationMarker(reservation) { + return `${stableStringify({ + schema: 'environment-factory.workspace-reservation-marker.v1', + reservation_id: reservation.reservation_id, + operation_id: reservation.operation_id, + path: reservation.path, + repository: reservation.repository, + ownership_token_sha256: sha256(reservation.ownership_token), + })}\n`; +} + +function verifyReservation(reservation) { + if (!reservation || reservation.reservation_schema !== 'environment-factory.workspace-reservation.v1' + || typeof reservation.reservation_id !== 'string' || reservation.reservation_id === '' + || typeof reservation.operation_id !== 'string' || reservation.operation_id === '' + || typeof reservation.path !== 'string' || !path.isAbsolute(reservation.path) + || typeof reservation.containment_root !== 'string' || !path.isAbsolute(reservation.containment_root) + || path.dirname(reservation.path) !== reservation.containment_root + || typeof reservation.ownership_token !== 'string' || !/^[0-9a-f]{64}$/.test(reservation.ownership_token) + || typeof reservation.cleanup_capture_id !== 'string' + || !/^[0-9a-f]{64}$/.test(reservation.cleanup_capture_id) + || !reservation.repository || typeof reservation.repository.url !== 'string' + || !/^[0-9a-f]{40}$/.test(String(reservation.repository.commit_sha || '')) + || !samePathIdentity(pathIdentity(reservation.containment_root), reservation.containment_root_identity)) { + throw new Error('workspace reservation binding differs'); + } + return reservation; +} + +function allocationId(reservation) { + return sha256(stableStringify({ + schema: 'environment-factory.workspace-directory-allocation.v1', + reservation_id: reservation.reservation_id, + ownership_token_sha256: sha256(reservation.ownership_token), + })); +} + +function prepareReservedWorkspace(reservation, persistedIdentity, options = {}) { + verifyReservation(reservation); + const hooks = options.hooks || {}; + if (persistedIdentity) { + if (pathEntryExists(reservation.path)) { + if (!samePathIdentity(pathIdentity(reservation.path), persistedIdentity)) { + throw new Error('workspace reserved object identity changed'); + } + const recoveryCaptureId = sha256(stableStringify({ + schema: 'environment-factory.workspace-recovery-cleanup.v1', + reservation_id: reservation.reservation_id, + path_identity: persistedIdentity, + cleanup_capture_id: reservation.cleanup_capture_id, + })); + if (!removeOwnedDirectory( + reservation.path, persistedIdentity, reservation.containment_root, + recoveryCaptureId, + )) throw new Error('workspace reserved object cleanup is unavailable'); + } + } else if (options.reservationWasCreated === true && pathEntryExists(reservation.path)) { + throw new Error('workspace path already exists without recoverable reservation ownership'); + } else if (options.reservationWasCreated !== true && !pathEntryExists(reservation.path)) { + throw new Error('workspace allocation identity is unavailable for recovery'); + } + + const existedBefore = pathEntryExists(reservation.path); + const marker = reservationMarker(reservation); + const identifier = allocationId(reservation); + const identity = allocateOwnedDirectory( + reservation.path, + reservation.containment_root, + reservation.containment_root_identity, + identifier, + MARKER_NAME, + marker, + ); + if (!existedBefore && typeof hooks.afterWorkspaceDirectoryCreated === 'function') { + hooks.afterWorkspaceDirectoryCreated({ reservation: { ...reservation } }); + } + if (!samePathIdentity(pathIdentity(reservation.path), identity)) { + throw new Error('workspace identity changed after allocation'); + } + if (typeof options.persistIdentity !== 'function') { + throw new Error('workspace reservation identity persistence is unavailable'); + } + options.persistIdentity(identity); + if (typeof hooks.afterWorkspaceResourceRegistered === 'function') { + hooks.afterWorkspaceResourceRegistered({ reservation: { ...reservation }, path_identity: identity }); + } + if (!samePathIdentity(pathIdentity(reservation.path), identity)) { + throw new Error('workspace identity changed before marker retirement'); + } + retireOwnedDirectoryMarker( + reservation.path, + identity, + reservation.containment_root, + reservation.containment_root_identity, + identifier, + MARKER_NAME, + marker, + ); + return identity; +} + +module.exports = { + prepareReservedWorkspace, + reservationMarker, + verifyReservation, +}; diff --git a/packages/environment-factory/bin/runtime/workspace.js b/packages/environment-factory/bin/runtime/workspace.js index a7332ab9..742f9184 100644 --- a/packages/environment-factory/bin/runtime/workspace.js +++ b/packages/environment-factory/bin/runtime/workspace.js @@ -5,10 +5,19 @@ const fs = require('fs'); const path = require('path'); const { spawnSync } = require('child_process'); const { - minimalEnvironment, - releaseWorkerEnvironment, + pathIdentity, + samePathIdentity, verifyWorkerEnvironment, } = require('./common'); +const { + closeDirectoryAnchor, + objectBoundExec, + openDirectoryAnchor, +} = require('./object-bound-exec'); +const { + allocateDurableWorkerEnvironment, + recordWorkerEnvironmentRelease, +} = require('./worker-home-resource'); function sha256(value) { return crypto.createHash('sha256').update(String(value)).digest('hex'); @@ -45,36 +54,37 @@ function sameRepository(left, right) { return left && right && left.url === right.url && left.commit_sha === right.commit_sha; } -function gitOutput(workspaceRoot, argv, label, request) { - const environment = minimalEnvironment({}, { - schema: 'environment-factory.workspace-integrity-isolation.v1', - operation_id: request.operation_id, - repository: request.repository, - purpose: label, - }); +function gitOutput(workspaceRoot, workspaceIdentity, argv, label, request) { + const allocation = allocateDurableWorkerEnvironment(request, `workspace-${label}`); + let anchor; try { - verifyWorkerEnvironment(environment); - const result = spawnSync('git', argv, { - cwd: workspaceRoot, + verifyWorkerEnvironment(allocation.environment); + anchor = openDirectoryAnchor(workspaceRoot, workspaceIdentity); + const launch = objectBoundExec(anchor, ['git', ...argv], 3); + const result = spawnSync(launch.command, launch.argv, { + cwd: '/', encoding: 'utf8', - env: environment, + env: allocation.environment, shell: false, + stdio: ['ignore', 'pipe', 'pipe', anchor.descriptor], timeout: 5_000, windowsHide: true, }); if (result.error || result.status !== 0) throw new Error(`workspace ${label} is unavailable`); return String(result.stdout || '').trim(); } finally { - releaseWorkerEnvironment(environment); + closeDirectoryAnchor(anchor); + recordWorkerEnvironmentRelease(allocation); } } -function currentCommit(workspaceRoot, request) { - return gitOutput(workspaceRoot, ['rev-parse', 'HEAD'], 'commit', request); +function currentCommit(workspaceRoot, workspaceIdentity, request) { + return gitOutput(workspaceRoot, workspaceIdentity, ['rev-parse', 'HEAD'], 'commit', request); } -function trackedChanges(workspaceRoot, request) { - return gitOutput(workspaceRoot, ['status', '--porcelain', '--untracked-files=no'], 'tracked status', request); +function trackedChanges(workspaceRoot, workspaceIdentity, request) { + return gitOutput(workspaceRoot, workspaceIdentity, + ['status', '--porcelain', '--untracked-files=no'], 'tracked status', request); } function resolveWorkspace(request) { @@ -97,11 +107,22 @@ function resolveWorkspace(request) { throw new Error('working_directory differs from workspace binding'); } const workspaceRoot = fs.realpathSync(resource.path); + if (!samePathIdentity(pathIdentity(resource.path), resource.path_identity)) { + throw new Error('workspace path identity changed'); + } + if (typeof resource.containment_root !== 'string' + || !samePathIdentity(pathIdentity(resource.containment_root), resource.containment_root_identity) + || (workspaceRoot !== fs.realpathSync(resource.containment_root) + && !workspaceRoot.startsWith(`${fs.realpathSync(resource.containment_root)}${path.sep}`))) { + throw new Error('workspace containment binding changed'); + } const isolationRequest = { ...request, repository: resource.repository }; - if (currentCommit(workspaceRoot, isolationRequest) !== resource.repository.commit_sha) { + if (currentCommit(workspaceRoot, resource.path_identity, isolationRequest) + !== resource.repository.commit_sha) { throw new Error('workspace commit binding is invalid'); } - if (request.require_clean === true && trackedChanges(workspaceRoot, isolationRequest) !== '') { + if (request.require_clean === true + && trackedChanges(workspaceRoot, resource.path_identity, isolationRequest) !== '') { throw new Error('workspace tracked files differ from the approved commit'); } const candidate = path.resolve(workspaceRoot, resource.working_directory); @@ -109,7 +130,7 @@ function resolveWorkspace(request) { if (cwd !== workspaceRoot && !cwd.startsWith(`${workspaceRoot}${path.sep}`)) { throw new Error('working_directory escaped workspace through a symbolic link'); } - return { cwd, resource, workspaceRoot }; + return { cwd, cwdIdentity: pathIdentity(cwd), resource, workspaceRoot }; } module.exports = { readResource, resolveWorkspace, resourcePath }; diff --git a/packages/environment-factory/core.lua b/packages/environment-factory/core.lua index 007d8458..0fcd8e9a 100644 --- a/packages/environment-factory/core.lua +++ b/packages/environment-factory/core.lua @@ -145,6 +145,23 @@ local function find_resource(state, id) return nil end +local function worker_home_ledger_ref(state) + local resource = find_resource(state, "worker-homes") + if resource == nil or resource.kind ~= "worker-home-ledger" then error("environment-factory: worker-home-ledger-missing: durable worker isolation ledger is required") end + return copy(resource.cleanup_ref) +end + +local function add_worker_home_ledger(state, request) + request.worker_home_ledger_ref = worker_home_ledger_ref(state) + return request +end + +local function add_worker_home_ledger_if_present(state, request) + local resource = find_resource(state, "worker-homes") + if resource ~= nil then request.worker_home_ledger_ref = copy(resource.cleanup_ref) end + return request +end + local function checkpoint_runtime_cleanup(state, ports, outcome, resource) if type(outcome) ~= "table" or type(outcome.cleanup_ref) ~= "table" then return end local cleanup_ref = { kind = outcome.cleanup_ref.kind, ref = outcome.cleanup_ref.ref } @@ -287,13 +304,49 @@ local function recover_authorized_state(request, ports, existing, remember) remember(state) save_state(ports, state) - local checkout = ports.checkout(add_budget({ + local ledger_resource = find_resource(state, "worker-homes") + if state.completed.worker_home_ledger == true then + if ledger_resource == nil or ledger_resource.kind ~= "worker-home-ledger" then + error("environment-factory: foreign-state: completed worker-home ledger resource is missing") + end + else + if state.completed.checkout == true or state.workspace_ref ~= nil then + error("environment-factory: foreign-state: checkout began without a durable worker-home ledger") + end + local ledger = ports.initialize_worker_home_ledger(add_budget({ + effect_id = effect_id(state, "worker-home-ledger"), + operation_id = state.operation_id, + artifact_root = state.artifact_root, + repository = copy(state.profile_snapshot.repository), + }, effect_budget(state, ports, state.profile_snapshot.timeouts.start_seconds, false))) + runtime_outcomes.validate_effect(ledger, "worker-home-ledger", { + status = true, + ledger_id = true, + cleanup_ref = true, + }) + if ledger.status ~= "passed" or type(ledger.ledger_id) ~= "string" or ledger.ledger_id == "" + or type(ledger.cleanup_ref) ~= "table" then + error("environment-factory: worker-home-ledger-failed: runtime did not initialize the ledger") + end + contract.validate_ref(ledger.cleanup_ref, "worker-home-ledger.cleanup_ref") + append_resource(state, { + id = "worker-homes", + kind = "worker-home-ledger", + cleanup_ref = copy(ledger.cleanup_ref), + timeout_seconds = state.profile_snapshot.timeouts.cleanup_seconds, + }) + state.completed.worker_home_ledger = true + remember(state) + save_state(ports, state) + end + + local checkout = ports.checkout(add_worker_home_ledger(state, add_budget({ effect_id = effect_id(state, "checkout"), operation_id = state.operation_id, repository = copy(state.profile_snapshot.repository), working_directory = state.profile_snapshot.working_directory, artifact_root = state.artifact_root, - }, effect_budget(state, ports, state.profile_snapshot.timeouts.start_seconds, false))) + }, effect_budget(state, ports, state.profile_snapshot.timeouts.start_seconds, false)))) local previous_workspace_ref = state.workspace_ref and copy(state.workspace_ref) or nil if type(checkout) == "table" and type(checkout.workspace_ref) == "table" and type(checkout.cleanup_ref) == "table" then local workspace_ref = { kind = checkout.workspace_ref.kind, ref = checkout.workspace_ref.ref } @@ -343,7 +396,7 @@ local function run_oneshot(state, ports, phase) mode = "oneshot", }, effect_budget(state, ports, state.profile_snapshot.timeouts[phase .. "_seconds"], false)) if phase == "install" then request.requires_frozen_dependencies = true end - local outcome = ports.run_argv(request) + local outcome = ports.run_argv(add_worker_home_ledger(state, request)) runtime_outcomes.validate_effect(outcome, "phase-" .. phase, { status = true, diagnostic_ref = true, @@ -368,7 +421,7 @@ local function start_service(state, ports, service, index, runtime_ports) end return copy(resource.cleanup_ref) end - local outcome = ports.run_argv(add_budget({ + local outcome = ports.run_argv(add_worker_home_ledger(state, add_budget({ effect_id = effect_id(state, "service/" .. index .. "/start"), operation_id = state.operation_id, artifact_root = state.artifact_root, @@ -378,7 +431,7 @@ local function start_service(state, ports, service, index, runtime_ports) mode = "supervised", listener_mode = service.listener_mode, runtime_ports = copy_list(runtime_ports), - }, effect_budget(state, ports, state.profile_snapshot.timeouts.start_seconds, false))) + }, effect_budget(state, ports, state.profile_snapshot.timeouts.start_seconds, false)))) checkpoint_runtime_cleanup(state, ports, outcome, { id = resource_id, kind = "service", @@ -408,7 +461,7 @@ end local function wait_checks(state, ports, checks, suffix, runtime_ports, process_cleanup_ref) local key = "readiness-" .. suffix if state.completed[key] then return end - local outcome = ports.wait_readiness(add_budget({ + local outcome = ports.wait_readiness(add_worker_home_ledger(state, add_budget({ effect_id = effect_id(state, "readiness/" .. suffix), operation_id = state.operation_id, artifact_root = state.artifact_root, @@ -417,8 +470,10 @@ local function wait_checks(state, ports, checks, suffix, runtime_ports, process_ checks = checks, runtime_ports = copy_list(runtime_ports), process_cleanup_ref = copy(process_cleanup_ref), - }, effect_budget(state, ports, state.profile_snapshot.timeouts.readiness_seconds, false))) - runtime_outcomes.validate_effect(outcome, key, { status = true, diagnostic_ref = true }) + }, effect_budget(state, ports, state.profile_snapshot.timeouts.readiness_seconds, false)))) + runtime_outcomes.validate_effect(outcome, key, { + status = true, diagnostic_ref = true, + }) add_diagnostic(state, outcome.diagnostic_ref) if outcome.status ~= "ready" then error("environment-factory: readiness-failed: " .. suffix) end state.completed[key] = true @@ -433,7 +488,7 @@ local function start_application(state, ports, runtime_ports) end return copy(resource.cleanup_ref) end - local outcome = ports.run_argv(add_budget({ + local outcome = ports.run_argv(add_worker_home_ledger(state, add_budget({ effect_id = effect_id(state, "application/start"), operation_id = state.operation_id, artifact_root = state.artifact_root, @@ -443,7 +498,7 @@ local function start_application(state, ports, runtime_ports) mode = "supervised", listener_mode = state.profile_snapshot.application_listener_mode, runtime_ports = copy_list(runtime_ports), - }, effect_budget(state, ports, state.profile_snapshot.timeouts.start_seconds, false))) + }, effect_budget(state, ports, state.profile_snapshot.timeouts.start_seconds, false)))) checkpoint_runtime_cleanup(state, ports, outcome, { id = "application", kind = "application", @@ -475,7 +530,7 @@ local function cleanup_resources(state, ports) for index = #state.resources, 1, -1 do local resource = state.resources[index] if resource.cleaned ~= true then - local request = add_budget({ + local request = add_worker_home_ledger_if_present(state, add_budget({ effect_id = effect_id(state, "cleanup/" .. resource.id), operation_id = state.operation_id, artifact_root = state.artifact_root, @@ -483,18 +538,39 @@ local function cleanup_resources(state, ports) argv = resource.cleanup_argv, workspace_ref = copy(state.workspace_ref), working_directory = state.profile_snapshot.working_directory, - }, effect_budget(state, ports, resource.timeout_seconds, true)) + }, effect_budget(state, ports, resource.timeout_seconds, true))) local ok, outcome = pcall(ports.cleanup, request) if ok then ok = pcall(runtime_outcomes.validate_effect, outcome, "cleanup-" .. resource.id, { status = true, diagnostic_ref = true, + resource_detail_ref = true, + resource_detail_sha256 = true, + remaining_count = true, }) end complete = complete and ok if ok then add_diagnostic(state, outcome.diagnostic_ref) resource.cleanup_diagnostic_ref = copy(outcome.diagnostic_ref) + if outcome.resource_detail_ref ~= nil then + if resource.kind ~= "worker-home-ledger" then + error("environment-factory: unexpected-retention-detail: " .. resource.id) + end + contract.validate_artifact_ref(outcome.resource_detail_ref, + "cleanup." .. resource.id .. ".resource_detail_ref") + if type(outcome.resource_detail_sha256) ~= "string" + or #outcome.resource_detail_sha256 ~= 64 + or outcome.resource_detail_sha256:match("^[0-9a-f]+$") == nil + or type(outcome.remaining_count) ~= "number" + or outcome.remaining_count ~= math.floor(outcome.remaining_count) + or outcome.remaining_count < 1 or outcome.remaining_count > 256 then + error("environment-factory: malformed-retention-detail: " .. resource.id) + end + resource.resource_detail_ref = copy(outcome.resource_detail_ref) + resource.resource_detail_sha256 = outcome.resource_detail_sha256 + resource.remaining_count = outcome.remaining_count + end if outcome.status == "cleaned" then resource.cleaned = true else complete = false end end if not try_save_state(ports, state) then complete = false end @@ -520,11 +596,17 @@ local function cleanup_receipt(state) if status == "cleaned" then table.insert(verified, resource.id) else - table.insert(remaining, { + local retained = { resource_id = resource.id, resource_kind = resource.kind, cleanup_ref = copy(resource.cleanup_ref), - }) + } + if resource.kind == "worker-home-ledger" then + retained.resource_detail_ref = copy(resource.resource_detail_ref) + retained.resource_detail_sha256 = resource.resource_detail_sha256 + retained.remaining_count = resource.remaining_count + end + table.insert(remaining, retained) end end return contract.validate_cleanup_receipt({ diff --git a/packages/environment-factory/ports.lua b/packages/environment-factory/ports.lua index f71f05be..345d3a1b 100644 --- a/packages/environment-factory/ports.lua +++ b/packages/environment-factory/ports.lua @@ -4,6 +4,7 @@ local runtime = require("runtime") local runtime_names = { "load_authorization_bundle", "authorize_claim_ports", + "initialize_worker_home_ledger", "checkout", "remaining_budget", "create_readiness_attempt", @@ -87,6 +88,7 @@ function P.resolve(value) "save_state", "load_authorization_bundle", "authorize_claim_ports", + "initialize_worker_home_ledger", "checkout", "remaining_budget", "create_readiness_attempt", diff --git a/packages/environment-factory/runtime.lua b/packages/environment-factory/runtime.lua index 41995461..bcef8019 100644 --- a/packages/environment-factory/runtime.lua +++ b/packages/environment-factory/runtime.lua @@ -436,6 +436,9 @@ function R.production(options) return pending.outcome end + ports.initialize_worker_home_ledger = function(request) + return invoke("initialize-worker-home-ledger", request, cli_timeout(request.timeout_seconds)) + end ports.checkout = function(request) return invoke("checkout", request, cli_timeout(request.timeout_seconds)) end ports.remaining_budget = function(request) local result = invoke("remaining-budget", request, 15) diff --git a/packages/environment-factory/tests/contract_test.lua b/packages/environment-factory/tests/contract_test.lua index ac6722c9..2afcd463 100644 --- a/packages/environment-factory/tests/contract_test.lua +++ b/packages/environment-factory/tests/contract_test.lua @@ -125,6 +125,30 @@ local function cleanup_receipt() } end +local function worker_home_retention() + return { + schema = contract.schemas.worker_home_retention, + operation_id = "contract-fixture", + ledger_id = string.rep("a", 64), + repository = { + url = "https://example.invalid/testing/fixture.git", + commit_sha = string.rep("b", 40), + }, + remaining_count = 1, + entries = { { + slot_id = string.rep("c", 64), + lease_id = string.rep("d", 32), + effect_id = "worker-effect-1", + purpose = "validation-worker", + generation = 1, + identity_sha256 = string.rep("e", 64), + marker_sha256 = string.rep("f", 64), + state = "retained", + reason = "cleanup must retain an observed replacement", + } }, + } +end + return { test_exported_pointer_copy_is_closed = function() local copied = contract.copy_ref({ kind = "artifact", ref = ".testing/runs/x.json", ignored = true }) @@ -374,4 +398,45 @@ return { t.raises(function() contract.validate_cleanup_receipt(malformed) end) end end, + + test_cleanup_receipt_retention_details_are_closed_and_worker_home_only = function() + local function remaining_receipt(resource_kind) + local value = cleanup_receipt() + value.status = "incomplete" + value.attempted_resources[1].status = "remaining" + value.verified_removals = {} + value.remaining_resources = { { + resource_id = "workspace", + resource_kind = resource_kind, + cleanup_ref = { kind = "resource-cleanup", ref = "workspace" }, + } } + return value + end + + local missing = remaining_receipt("worker-home-ledger") + t.raises(function() contract.validate_cleanup_receipt(missing) end) + + local unexpected = remaining_receipt("workspace") + unexpected.remaining_resources[1].resource_detail_ref = { + kind = "artifact", + ref = ".testing/runs/contract-fixture/worker-home-retention.json", + } + unexpected.remaining_resources[1].resource_detail_sha256 = string.rep("a", 64) + unexpected.remaining_resources[1].remaining_count = 1 + t.raises(function() contract.validate_cleanup_receipt(unexpected) end) + end, + + test_worker_home_retention_contract_rejects_malformed_recovery_state = function() + contract.validate_worker_home_retention(worker_home_retention()) + for _, mutate in ipairs({ + function(v) v.schema = "other" end, + function(v) v.remaining_count = 2 end, + function(v) v.entries[1].lease_id = string.rep("A", 32) end, + function(v) v.entries[1].state = "cleaned" end, + }) do + local malformed = worker_home_retention() + mutate(malformed) + t.raises(function() contract.validate_worker_home_retention(malformed) end) + end + end, } diff --git a/packages/environment-factory/tests/core_test.lua b/packages/environment-factory/tests/core_test.lua index 7c54662d..849ff717 100644 --- a/packages/environment-factory/tests/core_test.lua +++ b/packages/environment-factory/tests/core_test.lua @@ -227,6 +227,16 @@ local function fake_runtime(fx, options, shared) return outcome end) end, + initialize_worker_home_ledger = function(request) + assert_budget(request) + return cached(request.effect_id, function() + record("worker-home-ledger"); target_effects = target_effects + 1 + if opts.fail_worker_home_ledger then return { status = "blocked" } end + return { status = "passed", ledger_id = string.rep("d", 64), cleanup_ref = { + kind = "resource-cleanup", ref = request.operation_id .. "-worker-home-ledger", + } } + end) + end, checkout = function(request) assert_budget(request) return cached(request.effect_id, function() @@ -319,7 +329,15 @@ local function fake_runtime(fx, options, shared) else for ref, _ in pairs(active) do if ref:find(id == "application" and "application-start" or "service-start", 1, true) then active[ref] = nil end end end - return { status = opts.fail_cleanup == id and "blocked" or "cleaned", diagnostic_ref = { kind = "artifact", ref = fx.request.artifact_root .. "/diagnostics/cleanup-" .. id .. ".json" } } + local outcome = { status = opts.fail_cleanup == id and "blocked" or "cleaned", diagnostic_ref = { + kind = "artifact", ref = fx.request.artifact_root .. "/diagnostics/cleanup-" .. id .. ".json", + } } + if opts.retention_detail_for == id then + outcome.resource_detail_ref = { kind = "artifact", ref = fx.request.artifact_root .. "/worker-home-retention.json" } + outcome.resource_detail_sha256 = opts.malformed_retention_detail and "bad" or string.rep("e", 64) + outcome.remaining_count = 1 + end + return outcome end) end, write_receipt = function(request) @@ -429,7 +447,7 @@ return { t.eq(finalized.environment_receipt_ref.ref, fx.request.artifact_root .. "/environment-receipt-finalized.json") t.eq(finalized.environment_receipt_ref.ref == ready.environment_receipt_ref.ref, false) t.eq(observed.receipts[ready.environment_receipt_ref.ref].status, ready_receipt.status) - t.eq(table.concat(observed.cleanup_order, ","), "application,service-1,workspace,ports") + t.eq(table.concat(observed.cleanup_order, ","), "application,service-1,workspace,worker-homes,ports") t.eq(observed.active_count(), 0) end, @@ -533,6 +551,22 @@ return { end end, + test_worker_home_ledger_state_and_initialization_fail_closed = function() + do + local fx = fixture({ operation_id = "worker-home-ledger-init-failed" }); local ports, observed = fake_runtime(fx, { fail_worker_home_ledger = true }) + local blocked = start_environment(fx, ports); t.eq(blocked.status, "blocked"); t.eq(observed.active_count(), 0) + end + do + local fx = fixture({ operation_id = "worker-home-ledger-resource-missing" }); local ports, observed = fake_runtime(fx); core.start(fx.request, ports); local state = observed.state() + for index = #state.resources, 1, -1 do if state.resources[index].id == "worker-homes" then table.remove(state.resources, index) end end + observed.set_state(state, true); t.raises(function() core.start(fx.request, ports) end) + end + do + local fx = fixture({ operation_id = "worker-home-ledger-before-checkout" }); local ports, observed = fake_runtime(fx); core.start(fx.request, ports) + local state = observed.state(); state.completed.worker_home_ledger = false; observed.set_state(state, true) + t.raises(function() core.start(fx.request, ports) end) + end + end, test_diagnostics_are_deduplicated_and_capped_for_max_services = function() local fx = fixture({ service_count = 16 }); local ports = fake_runtime(fx); local ready = start_environment(fx, ports) t.eq(#ready.diagnostic_refs, environment_contract.max_diagnostic_refs); environment_contract.validate_result(ready) @@ -596,7 +630,7 @@ return { test_partial_checkout_failure_persists_handle_unwinds_and_writes_blocked_receipt = function() local fx = fixture(); local ports, observed = fake_runtime(fx, { checkout_partial = true }); local blocked = start_environment(fx, ports) t.eq(blocked.status, "blocked"); t.eq(blocked.failure_class, "checkout-failed"); t.eq(blocked.environment_receipt_ref.ref, fx.request.artifact_root .. "/environment-receipt-blocked.json") - t.eq(table.concat(observed.cleanup_order, ","), "workspace,ports"); t.eq(observed.active_count(), 0) + t.eq(table.concat(observed.cleanup_order, ","), "workspace,worker-homes,ports"); t.eq(observed.active_count(), 0) end, test_serialized_port_lease_prevents_concurrent_exact_port_owners = function() @@ -610,10 +644,10 @@ return { test_cancel_and_interrupt_use_same_reverse_cleanup_for_app_and_partial_service = function() local app = fixture({ operation_id = "cancel-app" }); local app_ports, app_observed = fake_runtime(app); local app_ready = start_environment(app, app_ports) - t.eq(core.interrupt(interrupt_request(app, app_ready, "cancelled"), app_ports).status, "cancelled"); t.eq(table.concat(app_observed.cleanup_order, ","), "application,service-1,workspace,ports") + t.eq(core.interrupt(interrupt_request(app, app_ready, "cancelled"), app_ports).status, "cancelled"); t.eq(table.concat(app_observed.cleanup_order, ","), "application,service-1,workspace,worker-homes,ports") local partial = fixture({ operation_id = "interrupt-service" }); local partial_ports, partial_observed = fake_runtime(partial); local partial_ready = start_environment(partial, partial_ports) local state = partial_observed.state(); table.remove(state.resources, #state.resources); state.status = "provisioning"; state.public_result = nil; state.receipt_refs.ready = nil; partial_observed.set_state(state, true) - t.eq(core.interrupt(interrupt_request(partial, partial_ready, "interrupted"), partial_ports).status, "interrupted"); t.eq(table.concat(partial_observed.cleanup_order, ","), "service-1,workspace,ports") + t.eq(core.interrupt(interrupt_request(partial, partial_ready, "interrupted"), partial_ports).status, "interrupted"); t.eq(table.concat(partial_observed.cleanup_order, ","), "service-1,workspace,worker-homes,ports") end, test_install_requires_explicit_frozen_dependency_enforcement = function() @@ -640,7 +674,7 @@ return { test_resolved_source_mismatch_unwinds_workspace_and_port_claim = function() local fx = fixture(); local ports, observed = fake_runtime(fx, { resolved_commit = string.rep("b", 40) }); local blocked = start_environment(fx, ports) - t.eq(blocked.status, "blocked"); t.eq(blocked.failure_class, "source-mismatch"); t.eq(table.concat(observed.cleanup_order, ","), "workspace,ports") + t.eq(blocked.status, "blocked"); t.eq(blocked.failure_class, "source-mismatch"); t.eq(table.concat(observed.cleanup_order, ","), "workspace,worker-homes,ports") end, test_runtime_contract_failure_matrix_fails_closed = function() @@ -768,8 +802,8 @@ return { local receipt = observed.receipts[blocked.cleanup_receipt_ref.ref] environment_contract.validate_cleanup_receipt(receipt) t.eq(receipt.status, "incomplete") - t.eq(#receipt.attempted_resources, 4) - t.eq(#receipt.verified_removals, 3) + t.eq(#receipt.attempted_resources, 5) + t.eq(#receipt.verified_removals, 4) t.eq(#receipt.remaining_resources, 1) t.eq(receipt.remaining_resources[1].resource_id, "application") end, @@ -785,6 +819,18 @@ return { t.eq(receipt.remaining_resources[1].resource_id, "service-1") end, + test_cleanup_retention_details_are_worker_ledger_only_and_well_formed = function() + do + local fx = fixture({ operation_id = "unexpected-retention-detail" }); local ports = fake_runtime(fx, { retention_detail_for = "application" }); local ready = start_environment(fx, ports) + t.raises(function() core.finalize(finalize_request(fx, ready), ports) end) + end + do + local fx = fixture({ operation_id = "malformed-retention-detail" }); local ports = fake_runtime(fx, { fail_cleanup = "worker-homes", retention_detail_for = "worker-homes", malformed_retention_detail = true }) + local ready = start_environment(fx, ports) + t.raises(function() core.finalize(finalize_request(fx, ready), ports) end) + end + end, + test_terminal_cleanup_receipt_is_complete_replay_safe_and_owned = function() local fx = fixture({ operation_id = "cleanup-receipt-complete" }) local ports, observed = fake_runtime(fx) @@ -796,8 +842,8 @@ return { environment_contract.validate_cleanup_receipt(receipt) t.eq(receipt.operation_id, fx.request.operation_id) t.eq(receipt.status, "complete") - t.eq(#receipt.attempted_resources, 4) - t.eq(#receipt.verified_removals, 4) + t.eq(#receipt.attempted_resources, 5) + t.eq(#receipt.verified_removals, 5) t.eq(#receipt.remaining_resources, 0) local cleanup_count = #observed.cleanup_order local replay = core.finalize(finalize_request(fx, ready), ports) diff --git a/packages/environment-factory/tests/fixtures/runtime/source/credential-isolation.js b/packages/environment-factory/tests/fixtures/runtime/source/credential-isolation.js index cdc226a0..d8dd8710 100644 --- a/packages/environment-factory/tests/fixtures/runtime/source/credential-isolation.js +++ b/packages/environment-factory/tests/fixtures/runtime/source/credential-isolation.js @@ -3,7 +3,12 @@ const path = require('path'); function assertCredentialIsolation() { - for (const key of ['GH_TOKEN', 'GITHUB_TOKEN', 'SSH_AUTH_SOCK', 'GIT_ASKPASS', 'SSH_ASKPASS']) { + for (const key of [ + 'GH_TOKEN', 'GITHUB_TOKEN', 'SSH_AUTH_SOCK', 'GIT_ASKPASS', 'SSH_ASKPASS', + 'FKST_WORKER_RUNTIME_ROOT', + 'FKST_OBJECT_BOUND_ALLOCATION_BROKER', 'FKST_OBJECT_BOUND_ALLOCATION_BROKER_SHA256', + 'FKST_OBJECT_BOUND_CLEANUP_BROKER', 'FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256', + ]) { if (process.env[key]) throw new Error(`worker inherited forbidden authority: ${key}`); } const home = process.env.HOME || ''; diff --git a/packages/environment-factory/tests/fixtures/runtime/source/database_service.py b/packages/environment-factory/tests/fixtures/runtime/source/database_service.py index 3f6806ba..1fe65fd1 100644 --- a/packages/environment-factory/tests/fixtures/runtime/source/database_service.py +++ b/packages/environment-factory/tests/fixtures/runtime/source/database_service.py @@ -15,7 +15,12 @@ def assert_credential_isolation(): - for key in ("GH_TOKEN", "GITHUB_TOKEN", "SSH_AUTH_SOCK", "GIT_ASKPASS", "SSH_ASKPASS"): + for key in ( + "GH_TOKEN", "GITHUB_TOKEN", "SSH_AUTH_SOCK", "GIT_ASKPASS", "SSH_ASKPASS", + "FKST_WORKER_RUNTIME_ROOT", + "FKST_OBJECT_BOUND_ALLOCATION_BROKER", "FKST_OBJECT_BOUND_ALLOCATION_BROKER_SHA256", + "FKST_OBJECT_BOUND_CLEANUP_BROKER", "FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256", + ): if os.environ.get(key): raise RuntimeError(f"worker inherited forbidden authority: {key}") home = Path(os.environ.get("HOME", "")) diff --git a/packages/environment-factory/tests/hermetic_e2e_test.lua b/packages/environment-factory/tests/hermetic_e2e_test.lua index 12cd8ed9..13a8803e 100644 --- a/packages/environment-factory/tests/hermetic_e2e_test.lua +++ b/packages/environment-factory/tests/hermetic_e2e_test.lua @@ -9,6 +9,7 @@ local t = fkst.test local fixture_root = "packages/environment-factory/tests/fixtures/runtime/source" local command_sequence = 0 +local trusted_runtime_environment = {} local function copy(value) if type(value) ~= "table" then return value end @@ -41,8 +42,16 @@ local function direct_exec_argv(request) local argv = type(request) == "table" and request.argv or nil if type(argv) ~= "table" or #argv == 0 then error("hermetic exec requires argv") end local rendered = {} + for _, key in ipairs({ + "FKST_OBJECT_BOUND_ALLOCATION_BROKER", + "FKST_OBJECT_BOUND_ALLOCATION_BROKER_SHA256", + }) do + local value = trusted_runtime_environment[key] + if value ~= nil then table.insert(rendered, shell_quote(key .. "=" .. value)) end + end for _, item in ipairs(argv) do table.insert(rendered, shell_quote(item)) end - local command = table.concat(rendered, " ") + local command = (next(trusted_runtime_environment) ~= nil and "env " or "") + .. table.concat(rendered, " ") if request.cwd ~= nil then command = "cd " .. shell_quote(request.cwd) .. " && " .. command end command_sequence = command_sequence + 1 @@ -112,6 +121,18 @@ local function run_argv(argv, cwd, timeout) end project_root = run_argv({ "node", "-e", "process.stdout.write(process.cwd())" }):gsub("%s+$", "") +local allocation_broker = project_root + .. "/packages/environment-factory/bin/object-bound-cleanup-broker.py" +local allocation_broker_sha256 = run_argv({ + "node", "-e", + "const fs=require('fs'),c=require('crypto');" + .. "process.stdout.write(c.createHash('sha256').update(fs.readFileSync(process.argv[1])).digest('hex'));", + allocation_broker, +}) +trusted_runtime_environment = { + FKST_OBJECT_BOUND_ALLOCATION_BROKER = allocation_broker, + FKST_OBJECT_BOUND_ALLOCATION_BROKER_SHA256 = allocation_broker_sha256, +} local function make_context(suffix) local operation_id = "environment-hermetic-" .. suffix @@ -180,6 +201,15 @@ local function assert_path_absent(path) }, nil, 5) end +local function assert_path_present(path) + run_argv({ + "node", + "-e", + "if(!require('fs').existsSync(process.argv[1])) process.exit(43)", + path, + }, nil, 5) +end + local function http_json(port, path) local script = table.concat({ "const http=require('http');", @@ -295,8 +325,10 @@ local function request_fixture(ctx, ports, commit_sha) repository = repository, authority = { kind = "host-policy", ref = "fixtures/environment-factory-hermetic" }, policy_revision = "environment-factory-hermetic-v1", + human_approval_required = false, authorization_capability = false, execution_authorized = false, + promotion_authorized = false, }, command_environment = { FKST_FIXTURE_EVIDENCE_DIR = ctx.evidence_root, @@ -548,20 +580,57 @@ local function assert_cleanup_evidence(ctx, optional) end end -local function finalize_ready(ctx, request, ready, ports, optional_evidence) +local function finalize_cleanup_blocked(ctx, request, ready, ports, optional_evidence) local final_result = core.finalize(termination_request(request)) - if final_result.status ~= "finalized" then - local details = {} - for _, ref in ipairs(final_result.diagnostic_refs or {}) do - local read_ok, body = pcall(file.read, ref.ref) - if read_ok then table.insert(details, tostring(body)) end + t.eq(final_result.status, "blocked") + t.eq(final_result.failure_class, "cleanup-incomplete") + t.eq(final_result.cleanup_status, "incomplete") + t.is_true(#file.read(final_result.environment_receipt_ref.ref) > 0) + t.is_true(#file.read(final_result.cleanup_receipt_ref.ref) > 0) + local unavailable = 0 + for _, ref in ipairs(final_result.diagnostic_refs or {}) do + local read_ok, body = pcall(file.read, ref.ref) + if read_ok and tostring(body):find("OBJECT_BOUND_CLEANUP_UNAVAILABLE", 1, true) ~= nil then + unavailable = unavailable + 1 end - error("finalization blocked diagnostics=" .. table.concat(details, " | ")) end - t.eq(final_result.cleanup_status, "complete") - t.is_true(#file.read(final_result.environment_receipt_ref.ref) > 0) + t.is_true(unavailable >= 2) assert_cleanup_evidence(ctx, optional_evidence) for _, port in pairs(ports) do assert_listener_released(port) end + local state = runtime.production().load_state(request.operation_state_ref) + t.eq(state.authenticated, true) + t.eq(state.state.status, "blocked") + t.eq(state.state.cleanup_status, "incomplete") + local cleanup_receipt = json.decode(file.read(final_result.cleanup_receipt_ref.ref)) + local remaining = {} + for _, resource in ipairs(cleanup_receipt.remaining_resources) do + remaining[resource.resource_id] = resource + end + local worker_homes = remaining["worker-homes"] + t.eq(worker_homes.resource_kind, "worker-home-ledger") + t.is_true(type(worker_homes.resource_detail_ref.ref) == "string") + t.is_true(type(worker_homes.resource_detail_sha256) == "string") + t.is_true(worker_homes.remaining_count >= 2) + local retention = json.decode(file.read(worker_homes.resource_detail_ref.ref)) + contract.validate_worker_home_retention(retention) + t.eq(retention.operation_id, request.operation_id) + t.eq(retention.remaining_count, worker_homes.remaining_count) + t.eq(runtime.call_cli("sha256", { + artifact_root = ctx.artifact_root, + value = file.read(worker_homes.resource_detail_ref.ref), + }, 15).digest, worker_homes.resource_detail_sha256) + assert_path_present(workspace_path(ctx)) + local retained_homes = 0 + for _, resource in ipairs(state.state.resources) do + if resource.kind == "application" or resource.kind == "service" then + local record = resource_record(ctx, resource.cleanup_ref) + if type(record.worker_environment_lease) == "table" then + assert_path_present(record.worker_environment_lease.home) + retained_homes = retained_homes + 1 + end + end + end + t.is_true(retained_homes >= 2) return final_result end @@ -717,10 +786,10 @@ return { t.eq(missing_outcome.status, "blocked") t.eq(missing_outcome.frozen_dependencies_enforced, false) - local final_result = finalize_ready(ctx, request, ready, ports) + local final_result = finalize_cleanup_blocked(ctx, request, ready, ports) mark_finalized() t.eq(file.read(ready.environment_receipt_ref.ref), ready_receipt_body) - assert_path_absent(checkout_path) + assert_path_present(checkout_path) t.eq(run_argv({ "git", "status", "--porcelain" }, ctx.source_root), "") local _, operation_digest = workspace_path(ctx) @@ -735,8 +804,8 @@ return { local state_after = runtime.production().load_state(request.operation_state_ref) t.eq(state_after.authenticated, true) - t.eq(state_after.state.status, "finalized") - t.eq(state_after.state.cleanup_status, "complete") + t.eq(state_after.state.status, "blocked") + t.eq(state_after.state.cleanup_status, "incomplete") local overwrite_ok = pcall(runtime.production().write_receipt, { effect_id = request.dedup_key .. "/environment-factory/receipt/overwrite-attempt", @@ -754,7 +823,7 @@ return { file.write(request.operation_state_ref.ref, json_codec.encode(forged_envelope) .. "\n") local forged_state = runtime.production().load_state(request.operation_state_ref) t.eq(forged_state.authenticated, false) - t.eq(final_result.status, "finalized") + t.eq(final_result.status, "blocked") end) end, @@ -833,7 +902,7 @@ return { t.eq(process_blocked.status, "blocked") t.eq(diagnostic(process_blocked).reason, "process-budget-exceeded") - finalize_ready(ctx, request, ready, ports) + finalize_cleanup_blocked(ctx, request, ready, ports) mark_finalized() end) end, @@ -883,7 +952,7 @@ return { end if not ok then error(failure, 0) end - finalize_ready(ctx, request, ready, ports, { ["application-stopped.json"] = true }) + finalize_cleanup_blocked(ctx, request, ready, ports, { ["application-stopped.json"] = true }) mark_finalized() end) end, diff --git a/packages/environment-factory/tests/node_runtime_test.js b/packages/environment-factory/tests/node_runtime_test.js index bd2daac2..77802eff 100644 --- a/packages/environment-factory/tests/node_runtime_test.js +++ b/packages/environment-factory/tests/node_runtime_test.js @@ -5,31 +5,212 @@ const fs = require('fs'); const http = require('http'); const os = require('os'); const path = require('path'); -const { spawn } = require('child_process'); +const { spawn, spawnSync } = require('child_process'); const { acquireLock, authorizationArtifact, minimalEnvironment, + ownedDirectoryReleaseProven, + pathIdentity, + readBoundedRegularFile, + removeOwnedDirectory, releaseWorkerEnvironment, releaseWorkerEnvironmentLease, stableStringify, verifyWorkerEnvironment, + workerEnvironmentReleaseProven, workerEnvironmentLease, } = require('../bin/runtime/common'); const { validateTargetExecutionBoundary } = require('../bin/runtime/target-execution-boundary'); const { startOrRecoverSupervisedProcess } = require('../bin/runtime/supervised-process'); +const { + allocateDurableWorkerEnvironment, + initializeWorkerHomeLedger, + recordWorkerEnvironmentRelease, + verifyPersistedWorkerEnvironment, +} = require('../bin/runtime/worker-home-resource'); const { runMeasuredCommand } = require('../bin/runtime/measured-command'); const { listenersOwnedByProcessGroup, processGroupState, terminateProcessGroup, } = require('../bin/runtime/platform'); -const { dispatch, initialReadinessState, sha256 } = require('../bin/environment-factory-runtime'); +const { + checkout: checkoutWithHooks, dispatch, initialReadinessState, resourceIsReleased, sha256, +} = require('../bin/environment-factory-runtime'); function delay(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } +const cleanupRaceHarness = String.raw` +import importlib.util +import json +import os +import stat +import sys + +sys.dont_write_bytecode = True +spec = importlib.util.spec_from_file_location("object_bound_cleanup_broker", sys.argv[1]) +broker = importlib.util.module_from_spec(spec) +spec.loader.exec_module(broker) +fixture = json.loads(sys.argv[2]) +request = fixture["request"] +scenario = fixture["scenario"] +external = fixture["external"] +original_rename_noreplace = broker.rename_noreplace +injected = False + +def write_relative(directory_fd, name, body): + descriptor = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600, dir_fd=directory_fd) + try: + os.write(descriptor, body.encode("utf-8")) + finally: + os.close(descriptor) + +def injected_rename(source_name, source_fd, destination_name, destination_fd): + global injected + target_name = os.path.basename(request["target"]) + should_inject = ( + (scenario == "file-replacement" and source_name == "victim.txt") + or (scenario == "child-move" and source_name == "child") + or (scenario in ("target-move", "root-move") and source_name == target_name) + ) + if should_inject and not injected: + injected = True + if scenario == "file-replacement": + os.rename(source_name, external, src_dir_fd=source_fd) + write_relative(source_fd, source_name, "external-replacement\n") + elif scenario in ("child-move", "target-move"): + os.rename(source_name, external, src_dir_fd=source_fd) + else: + os.rename(request["containment_root"], external) + return original_rename_noreplace(source_name, source_fd, destination_name, destination_fd) + +broker.rename_noreplace = injected_rename +outcome = "cleaned" +reason = None +retry_outcome = "not-run" +try: + broker.cleanup(request) +except (broker.CleanupBlocked, FileNotFoundError, NotADirectoryError, PermissionError, OSError) as error: + outcome = "blocked" + reason = str(error) + try: + broker.cleanup(request) + retry_outcome = "cleaned" + except (broker.CleanupBlocked, FileNotFoundError, NotADirectoryError, PermissionError, OSError): + retry_outcome = "blocked" + +bodies = [] +for current_root, directories, files in os.walk(fixture["audit_root"], followlinks=False): + directories.sort() + files.sort() + for name in files: + candidate = os.path.join(current_root, name) + linked = os.lstat(candidate) + if stat.S_ISREG(linked.st_mode) and linked.st_size <= 1024: + with open(candidate, "r", encoding="utf-8") as handle: + bodies.append(handle.read()) +print(json.dumps({ + "outcome": outcome, + "retry_outcome": retry_outcome, + "reason": reason, + "injected": injected, + "bodies": bodies, +})) +`; + +function runCleanupRace(cleanupBroker, fixture) { + const environment = Object.create(null); + for (const key of ['LANG', 'LC_ALL']) { + if (typeof process.env[key] === 'string') environment[key] = process.env[key]; + } + const result = spawnSync('/usr/bin/python3', [ + '-I', '-c', cleanupRaceHarness, cleanupBroker, JSON.stringify(fixture), + ], { + env: environment, + shell: false, + encoding: 'utf8', + timeout: 10_000, + maxBuffer: 64 * 1024, + }); + assert.strictEqual(result.status, 0, result.stderr || result.stdout); + return JSON.parse(result.stdout); +} + +function runCleanupBroker(cleanupBroker, request) { + const environment = Object.create(null); + for (const key of ['LANG', 'LC_ALL']) { + if (typeof process.env[key] === 'string') environment[key] = process.env[key]; + } + const result = spawnSync('/usr/bin/python3', ['-I', cleanupBroker], { + input: `${stableStringify(request)}\n`, + env: environment, + shell: false, + encoding: 'utf8', + timeout: 10_000, + maxBuffer: 64 * 1024, + }); + return { + status: result.status, + value: JSON.parse(result.stdout), + stderr: result.stderr, + }; +} + +function captureLeaseBeforeCallerStateUpdate(cleanupBroker, lease) { + const request = { + schema: 'environment-factory.object-bound-cleanup-request.v1', + operation: 'capture-delete', + capture_id: lease.cleanup_capture_id, + target: lease.home_identity.realpath, + target_identity: lease.home_identity, + containment_root: lease.homes_root_identity.realpath, + containment_root_identity: lease.homes_root_identity, + }; + const stateRoot = path.join(process.env.FKST_DURABLE_ROOT, 'cleanup-captures'); + fs.mkdirSync(stateRoot, { recursive: true, mode: 0o700 }); + const statePath = path.join(stateRoot, `${lease.cleanup_capture_id}.json`); + const pendingState = { + schema: 'environment-factory.object-bound-cleanup-capture-state.v1', + capture_id: lease.cleanup_capture_id, + target: lease.home_identity.realpath, + target_identity: lease.home_identity, + containment_root: lease.homes_root_identity.realpath, + containment_root_identity: lease.homes_root_identity, + state: 'pending', + }; + if (fs.existsSync(statePath)) { + assert.deepStrictEqual(JSON.parse(fs.readFileSync(statePath, 'utf8')), pendingState); + } else { + fs.writeFileSync(statePath, `${stableStringify(pendingState)}\n`, { flag: 'wx' }); + } + const captured = runCleanupBroker(cleanupBroker, request); + assert.strictEqual(captured.status, 0, captured.stderr); + assert.strictEqual(captured.value.status, 'captured-cleaned'); + assert.strictEqual(fs.existsSync(lease.home), false); +} + +function cleanupRaceFixture(root, target, scenario, external) { + const rootIdentity = pathIdentity(root); + const targetIdentity = pathIdentity(target); + return { + scenario, + external, + audit_root: path.dirname(root), + request: { + schema: 'environment-factory.object-bound-cleanup-request.v1', + operation: 'capture-delete', + capture_id: 'a'.repeat(64), + target: targetIdentity.realpath, + target_identity: targetIdentity, + containment_root: rootIdentity.realpath, + containment_root_identity: rootIdentity, + }, + }; +} + async function removeTreeEventually(target, timeoutMs = 2_000) { const deadline = Date.now() + timeoutMs; while (true) { @@ -132,19 +313,28 @@ async function main() { const hostRoot = `.testing/host/environment-factory/environment-node-runtime-${process.pid}`; const previousDurable = process.env.FKST_DURABLE_ROOT; const previousRuntime = process.env.FKST_RUNTIME_ROOT; + const previousWorkerRuntime = process.env.FKST_WORKER_RUNTIME_ROOT; + const previousCleanupBroker = process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER; + const previousCleanupBrokerSha256 = process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256; process.env.FKST_DURABLE_ROOT = path.join(temp, 'durable'); process.env.FKST_RUNTIME_ROOT = path.join(temp, 'runtime'); + process.env.FKST_WORKER_RUNTIME_ROOT = path.join(temp, 'worker-runtime'); fs.rmSync(artifactRoot, { recursive: true, force: true }); fs.rmSync(hostRoot, { recursive: true, force: true }); let crashWindowResource = null; let firstStartupEnvironment = null; let firstLease = null; try { + const cleanupBroker = path.resolve(__dirname, '..', 'bin', 'object-bound-cleanup-broker.py'); + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER = cleanupBroker; + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = sha256(fs.readFileSync(cleanupBroker)); const ambientHome = path.join(temp, 'ambient-home'); fs.mkdirSync(ambientHome); const isolated = minimalEnvironment({ FKST_SAFE_MARKER: 'present' }, 'node-runtime-isolation'); assert.notStrictEqual(isolated.HOME, ambientHome); assert.strictEqual(path.basename(path.dirname(isolated.HOME)), 'worker-homes'); + assert.strictEqual(path.dirname(path.dirname(isolated.HOME)), + fs.realpathSync(process.env.FKST_WORKER_RUNTIME_ROOT)); assert.strictEqual(isolated.FKST_SAFE_MARKER, 'present'); assert.strictEqual(isolated.GIT_CONFIG_NOSYSTEM, '1'); assert.strictEqual(isolated.GIT_CONFIG_GLOBAL, process.platform === 'win32' ? 'NUL' : '/dev/null'); @@ -168,9 +358,350 @@ async function main() { assert.strictEqual(releaseWorkerEnvironment(secondIsolated), true); assert.strictEqual(fs.existsSync(isolatedHome), false); assert.strictEqual(fs.existsSync(secondIsolatedHome), false); + const brokerEnvironment = minimalEnvironment({}, 'node-runtime-broker-cleanup'); + const externalDirectory = path.join(temp, 'external-cleanup-sentinel'); + fs.mkdirSync(externalDirectory); + const brokerExternalSentinel = path.join(externalDirectory, 'sentinel.txt'); + fs.writeFileSync(brokerExternalSentinel, 'preserve\n'); + fs.writeFileSync(path.join(brokerEnvironment.HOME, 'worker-output.txt'), 'generated\n'); + fs.symlinkSync(externalDirectory, path.join(brokerEnvironment.HOME, 'external-link')); + assert.strictEqual(releaseWorkerEnvironment(brokerEnvironment), true); + assert.strictEqual(fs.existsSync(brokerEnvironment.HOME), false); + assert.strictEqual(fs.readFileSync(brokerExternalSentinel, 'utf8'), 'preserve\n'); + + const finalizeRecoveryRoot = path.join(temp, 'broker-finalize-recovery', 'containment'); + const finalizeRecoveryTarget = path.join(finalizeRecoveryRoot, 'target'); + fs.mkdirSync(finalizeRecoveryTarget, { recursive: true }); + fs.writeFileSync(path.join(finalizeRecoveryTarget, 'owned.txt'), 'owned\n'); + const finalizeRecoveryRootIdentity = pathIdentity(finalizeRecoveryRoot); + const finalizeRecoveryTargetIdentity = pathIdentity(finalizeRecoveryTarget); + const finalizeRecoveryCaptureId = sha256(`finalize-recovery\0${process.pid}`); + const finalizeRecoveryRequest = { + schema: 'environment-factory.object-bound-cleanup-request.v1', + capture_id: finalizeRecoveryCaptureId, + target: finalizeRecoveryTargetIdentity.realpath, + target_identity: finalizeRecoveryTargetIdentity, + containment_root: finalizeRecoveryRootIdentity.realpath, + containment_root_identity: finalizeRecoveryRootIdentity, + }; + const captureResult = runCleanupBroker(cleanupBroker, { + ...finalizeRecoveryRequest, operation: 'capture-delete', + }); + assert.strictEqual(captureResult.status, 0, captureResult.stderr); + assert.strictEqual(captureResult.value.status, 'captured-cleaned'); + assert.strictEqual(fs.existsSync(finalizeRecoveryTarget), false); + assert.strictEqual(ownedDirectoryReleaseProven( + finalizeRecoveryTarget, + finalizeRecoveryTargetIdentity, + finalizeRecoveryRoot, + finalizeRecoveryCaptureId, + ), false); + const finalizeResult = runCleanupBroker(cleanupBroker, { + ...finalizeRecoveryRequest, operation: 'finalize', + }); + assert.strictEqual(finalizeResult.status, 0, finalizeResult.stderr); + assert.strictEqual(finalizeResult.value.status, 'finalized'); + const proofPath = path.join( + path.dirname(finalizeRecoveryRootIdentity.realpath), + `.fkst-object-cleanup-${finalizeRecoveryCaptureId}`, + ); + assert.strictEqual(fs.existsSync(path.join(proofPath, 'finalized')), true); + const captureStateRoot = path.join(process.env.FKST_DURABLE_ROOT, 'cleanup-captures'); + fs.mkdirSync(captureStateRoot, { recursive: true, mode: 0o700 }); + fs.writeFileSync( + path.join(captureStateRoot, `${finalizeRecoveryCaptureId}.json`), + `${stableStringify({ + schema: 'environment-factory.object-bound-cleanup-capture-state.v1', + capture_id: finalizeRecoveryCaptureId, + target: finalizeRecoveryTargetIdentity.realpath, + target_identity: finalizeRecoveryTargetIdentity, + containment_root: finalizeRecoveryRootIdentity.realpath, + containment_root_identity: finalizeRecoveryRootIdentity, + state: 'captured-cleaned', + })}\n`, + { flag: 'wx' }, + ); + assert.strictEqual(ownedDirectoryReleaseProven( + finalizeRecoveryTarget, + finalizeRecoveryTargetIdentity, + finalizeRecoveryRoot, + finalizeRecoveryCaptureId, + ), false); + assert.strictEqual(removeOwnedDirectory( + finalizeRecoveryTarget, + finalizeRecoveryTargetIdentity, + finalizeRecoveryRoot, + finalizeRecoveryCaptureId, + ), true); + assert.strictEqual(fs.existsSync(proofPath), false); + assert.strictEqual( + JSON.parse(fs.readFileSync( + path.join(captureStateRoot, `${finalizeRecoveryCaptureId}.json`), 'utf8', + )).state, + 'released', + ); + assert.strictEqual(ownedDirectoryReleaseProven( + finalizeRecoveryTarget, + finalizeRecoveryTargetIdentity, + finalizeRecoveryRoot, + finalizeRecoveryCaptureId, + ), true); + + const missingProofRoot = path.join(temp, 'broker-missing-proof', 'containment'); + const missingProofTarget = path.join(missingProofRoot, 'target'); + fs.mkdirSync(missingProofTarget, { recursive: true }); + const missingProofRequest = { + schema: 'environment-factory.object-bound-cleanup-request.v1', + operation: 'finalize', + capture_id: sha256(`missing-proof\0${process.pid}`), + target: pathIdentity(missingProofTarget).realpath, + target_identity: pathIdentity(missingProofTarget), + containment_root: pathIdentity(missingProofRoot).realpath, + containment_root_identity: pathIdentity(missingProofRoot), + }; + const missingProofResult = runCleanupBroker(cleanupBroker, missingProofRequest); + assert.strictEqual(missingProofResult.status, 2); + assert.strictEqual(missingProofResult.value.status, 'blocked'); + + const fileRaceRoot = path.join(temp, 'broker-file-replacement', 'containment'); + const fileRaceTarget = path.join(fileRaceRoot, 'target'); + const fileRaceExternal = path.join(path.dirname(fileRaceRoot), 'moved-owned-file.txt'); + fs.mkdirSync(fileRaceTarget, { recursive: true }); + fs.writeFileSync(path.join(fileRaceTarget, 'victim.txt'), 'owned-file\n'); + const fileRace = runCleanupRace(cleanupBroker, cleanupRaceFixture( + fileRaceRoot, fileRaceTarget, 'file-replacement', fileRaceExternal, + )); + assert.strictEqual(fileRace.outcome, 'blocked'); + assert.strictEqual(fileRace.retry_outcome, 'blocked'); + assert.strictEqual(fileRace.injected, true, JSON.stringify(fileRace)); + assert.strictEqual(fs.readFileSync(fileRaceExternal, 'utf8'), 'owned-file\n'); + assert.ok(fileRace.bodies.includes('external-replacement\n')); + + const childRaceRoot = path.join(temp, 'broker-child-move', 'containment'); + const childRaceTarget = path.join(childRaceRoot, 'target'); + const childRaceExternal = path.join(path.dirname(childRaceRoot), 'moved-child'); + fs.mkdirSync(path.join(childRaceTarget, 'child'), { recursive: true }); + fs.writeFileSync(path.join(childRaceTarget, 'child', 'sentinel.txt'), 'child-preserved\n'); + const childRace = runCleanupRace(cleanupBroker, cleanupRaceFixture( + childRaceRoot, childRaceTarget, 'child-move', childRaceExternal, + )); + assert.strictEqual(childRace.outcome, 'blocked'); + assert.strictEqual(childRace.retry_outcome, 'blocked'); + assert.strictEqual(childRace.injected, true, JSON.stringify(childRace)); + assert.strictEqual( + fs.readFileSync(path.join(childRaceExternal, 'sentinel.txt'), 'utf8'), + 'child-preserved\n', + ); + + const targetRaceRoot = path.join(temp, 'broker-target-move', 'containment'); + const targetRaceTarget = path.join(targetRaceRoot, 'target'); + const targetRaceExternal = path.join(path.dirname(targetRaceRoot), 'moved-target'); + fs.mkdirSync(targetRaceTarget, { recursive: true }); + fs.writeFileSync(path.join(targetRaceTarget, 'sentinel.txt'), 'target-preserved\n'); + const targetRace = runCleanupRace(cleanupBroker, cleanupRaceFixture( + targetRaceRoot, targetRaceTarget, 'target-move', targetRaceExternal, + )); + assert.strictEqual(targetRace.outcome, 'blocked'); + assert.strictEqual(targetRace.retry_outcome, 'blocked'); + assert.strictEqual(targetRace.injected, true, JSON.stringify(targetRace)); + assert.strictEqual( + fs.readFileSync(path.join(targetRaceExternal, 'sentinel.txt'), 'utf8'), + 'target-preserved\n', + ); + + const rootRaceRoot = path.join(temp, 'broker-root-move', 'containment'); + const rootRaceTarget = path.join(rootRaceRoot, 'target'); + const rootRaceExternal = path.join(path.dirname(rootRaceRoot), 'moved-containment'); + fs.mkdirSync(rootRaceTarget, { recursive: true }); + fs.writeFileSync(path.join(rootRaceTarget, 'sentinel.txt'), 'root-preserved\n'); + const rootRace = runCleanupRace(cleanupBroker, cleanupRaceFixture( + rootRaceRoot, rootRaceTarget, 'root-move', rootRaceExternal, + )); + assert.strictEqual(rootRace.outcome, 'blocked'); + assert.strictEqual(rootRace.retry_outcome, 'blocked'); + assert.strictEqual(rootRace.injected, true, JSON.stringify(rootRace)); + assert.strictEqual( + fs.readFileSync(path.join(rootRaceExternal, 'target', 'sentinel.txt'), 'utf8'), + 'root-preserved\n', + ); + + const mismatchedBrokerEnvironment = minimalEnvironment({}, 'node-runtime-broker-digest-mismatch'); + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = '0'.repeat(64); + assert.throws(() => releaseWorkerEnvironment(mismatchedBrokerEnvironment), /broker digest differs/); + assert.strictEqual(fs.existsSync(mismatchedBrokerEnvironment.HOME), true); + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = sha256(fs.readFileSync(cleanupBroker)); + const workerRequest = { + operation_id: `node-operation-${process.pid}`, + repository: { url: 'https://github.com/example/repo.git', commit_sha: 'a'.repeat(40) }, + artifact_root: artifactRoot, + }; + const ledger = initializeWorkerHomeLedger(workerRequest); + const operationWorker = allocateDurableWorkerEnvironment({ + ...workerRequest, + effect_id: `node-operation-${process.pid}/checkout`, + worker_home_ledger_ref: ledger.cleanup_ref, + }, 'checkout'); + const operationWorkerReplay = allocateDurableWorkerEnvironment({ + ...workerRequest, + effect_id: `node-operation-${process.pid}/checkout`, + worker_home_ledger_ref: ledger.cleanup_ref, + }, 'checkout'); + const readinessWorker = allocateDurableWorkerEnvironment({ + ...workerRequest, + effect_id: `node-operation-${process.pid}/readiness`, + worker_home_ledger_ref: ledger.cleanup_ref, + }, 'readiness'); + let interruptedWorkerHome = null; + assert.throws(() => allocateDurableWorkerEnvironment({ + ...workerRequest, + effect_id: `node-operation-${process.pid}/interrupted-allocation`, + worker_home_ledger_ref: ledger.cleanup_ref, + }, 'interrupted-allocation', {}, null, { + afterEnvironmentCreated(environment) { + interruptedWorkerHome = environment.HOME; + throw new Error('simulated crash after worker HOME creation'); + }, + }), /simulated crash/); + assert.strictEqual(fs.existsSync(interruptedWorkerHome), true); + assert.strictEqual(operationWorker.environment.HOME, operationWorkerReplay.environment.HOME); + assert.strictEqual(operationWorker.slot_id, operationWorkerReplay.slot_id); + assert.notStrictEqual(operationWorker.environment.HOME, readinessWorker.environment.HOME); + delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER; + delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256; + assert.strictEqual(recordWorkerEnvironmentRelease(operationWorker), false); + assert.strictEqual(recordWorkerEnvironmentRelease(readinessWorker), false); + const workerCleanup = await dispatch('cleanup', { + effect_id: `node-operation-${process.pid}/cleanup/worker-homes`, + operation_id: `node-operation-${process.pid}`, + artifact_root: artifactRoot, + cleanup_ref: ledger.cleanup_ref, + worker_home_ledger_ref: ledger.cleanup_ref, + timeout_seconds: 1, + }); + assert.strictEqual(workerCleanup.status, 'blocked'); + assert.match(workerCleanup.resource_detail_sha256, /^[0-9a-f]{64}$/); + assert.strictEqual(workerCleanup.remaining_count, 3); + assert.strictEqual(fs.existsSync(operationWorker.environment.HOME), true); + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER = cleanupBroker; + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = sha256(fs.readFileSync(cleanupBroker)); + captureLeaseBeforeCallerStateUpdate(cleanupBroker, operationWorker.lease); + assert.strictEqual(fs.existsSync(operationWorker.lease.home), false); + assert.strictEqual(workerEnvironmentReleaseProven(operationWorker.lease), false); + captureLeaseBeforeCallerStateUpdate(cleanupBroker, readinessWorker.lease); + const recoveredWorkerCleanup = await dispatch('cleanup', { + effect_id: `node-operation-${process.pid}/cleanup/worker-homes`, + operation_id: `node-operation-${process.pid}`, + artifact_root: artifactRoot, + cleanup_ref: ledger.cleanup_ref, + worker_home_ledger_ref: ledger.cleanup_ref, + timeout_seconds: 1, + }); + assert.strictEqual(recoveredWorkerCleanup.status, 'cleaned'); + assert.strictEqual(fs.existsSync(interruptedWorkerHome), false); + + const interruptedAllocationRequest = { + operation_id: `node-interrupted-allocation-${process.pid}`, + repository: workerRequest.repository, + artifact_root: artifactRoot, + }; + const interruptedAllocationLedger = initializeWorkerHomeLedger(interruptedAllocationRequest); + let interruptedAllocatedHome = null; + assert.throws(() => allocateDurableWorkerEnvironment({ + ...interruptedAllocationRequest, + effect_id: `${interruptedAllocationRequest.operation_id}/checkout`, + worker_home_ledger_ref: interruptedAllocationLedger.cleanup_ref, + }, 'checkout', {}, null, { + afterHomeDirectoryCreated({ home }) { + interruptedAllocatedHome = home; + throw new Error('simulated crash after atomic worker HOME publication'); + }, + }), /simulated crash after atomic worker HOME publication/); + assert.strictEqual(fs.existsSync(interruptedAllocatedHome), true); + assert.strictEqual( + fs.existsSync(path.join(interruptedAllocatedHome, '.fkst-worker-home.json')), true, + ); + const interruptedAllocationCleanup = await dispatch('cleanup', { + effect_id: `${interruptedAllocationRequest.operation_id}/cleanup/worker-homes`, + operation_id: interruptedAllocationRequest.operation_id, + artifact_root: artifactRoot, + cleanup_ref: interruptedAllocationLedger.cleanup_ref, + worker_home_ledger_ref: interruptedAllocationLedger.cleanup_ref, + timeout_seconds: 1, + }); + assert.strictEqual(interruptedAllocationCleanup.status, 'cleaned'); + assert.strictEqual(fs.existsSync(interruptedAllocatedHome), false); + + const replacementRequest = { + operation_id: `node-worker-home-replacement-${process.pid}`, + repository: workerRequest.repository, + artifact_root: artifactRoot, + }; + const replacementLedger = initializeWorkerHomeLedger(replacementRequest); + const externalWorkerHome = path.join(temp, 'external-worker-home'); + fs.mkdirSync(path.join(externalWorkerHome, '.config', 'gh'), { recursive: true, mode: 0o700 }); + const externalCredential = path.join(externalWorkerHome, '.config', 'gh', 'hosts.yml'); + fs.writeFileSync(externalCredential, 'external-credential-sentinel\n'); + let displacedWorkerHome = null; + assert.throws(() => allocateDurableWorkerEnvironment({ + ...replacementRequest, + effect_id: `${replacementRequest.operation_id}/checkout`, + worker_home_ledger_ref: replacementLedger.cleanup_ref, + }, 'checkout', {}, null, { + afterHomeDirectoryCreated({ home }) { + displacedWorkerHome = `${home}.displaced`; + fs.renameSync(home, displacedWorkerHome); + fs.symlinkSync(externalWorkerHome, home); + }, + }), /worker environment home identity changed after allocation/); + assert.strictEqual(fs.readFileSync(externalCredential, 'utf8'), 'external-credential-sentinel\n'); + const replacementHome = displacedWorkerHome.slice(0, -'.displaced'.length); + assert.strictEqual(fs.lstatSync(replacementHome).isSymbolicLink(), true); + fs.unlinkSync(replacementHome); + const replacementCleanup = await dispatch('cleanup', { + effect_id: `${replacementRequest.operation_id}/cleanup/worker-homes`, + operation_id: replacementRequest.operation_id, + artifact_root: artifactRoot, + cleanup_ref: replacementLedger.cleanup_ref, + worker_home_ledger_ref: replacementLedger.cleanup_ref, + timeout_seconds: 1, + }); + assert.strictEqual(replacementCleanup.status, 'blocked'); + assert.strictEqual(replacementCleanup.remaining_count, 1); + assert.strictEqual(fs.existsSync(displacedWorkerHome), true); + assert.strictEqual(fs.readFileSync(externalCredential, 'utf8'), 'external-credential-sentinel\n'); + + const retainedProcessResource = { + kind: 'process', pid: 2147483647, pgid: 2147483647, + process_start_identity: 'not-running', worker_environment_lease: operationWorker.lease, + cleaned: false, + }; + assert.strictEqual(resourceIsReleased(retainedProcessResource), false); + assert.strictEqual(resourceIsReleased(retainedProcessResource, true), true); + assert.strictEqual(resourceIsReleased({ ...retainedProcessResource, cleaned: true }), true); + const linkedHomeEnvironment = minimalEnvironment({}, 'linked-home-isolation'); + const linkedHomeLease = workerEnvironmentLease(linkedHomeEnvironment); + delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER; + delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256; + const originalLinkedHome = `${linkedHomeLease.home}.original`; + const missingLinkedHomeTarget = `${linkedHomeLease.home}.missing`; + fs.renameSync(linkedHomeLease.home, originalLinkedHome); + fs.symlinkSync(missingLinkedHomeTarget, linkedHomeLease.home); + assert.throws( + () => releaseWorkerEnvironmentLease(linkedHomeLease), + /worker environment lease identity changed/, + ); + assert.strictEqual(fs.lstatSync(linkedHomeLease.home).isSymbolicLink(), true); + fs.unlinkSync(linkedHomeLease.home); + fs.renameSync(originalLinkedHome, linkedHomeLease.home); + assert.strictEqual(releaseWorkerEnvironment(linkedHomeEnvironment), false); + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER = cleanupBroker; + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = sha256(fs.readFileSync(cleanupBroker)); for (const key of [ 'HOME', 'USERPROFILE', 'XDG_CONFIG_HOME', 'GH_CONFIG_DIR', 'GH_TOKEN', 'GITHUB_TOKEN', 'GIT_CONFIG_GLOBAL', 'GIT_ASKPASS', 'SSH_AUTH_SOCK', 'SSH_ASKPASS', 'CREDENTIAL_HELPER', + 'FKST_WORKER_RUNTIME_ROOT', + 'FKST_OBJECT_BOUND_ALLOCATION_BROKER', 'FKST_OBJECT_BOUND_ALLOCATION_BROKER_SHA256', + 'FKST_OBJECT_BOUND_CLEANUP_BROKER', 'FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256', ]) { assert.throws(() => minimalEnvironment({ [key]: 'forbidden' }, 'node-runtime-isolation'), /forbidden worker authority key/); @@ -179,9 +710,9 @@ async function main() { const symlinkTarget = path.join(temp, 'symlink-runtime-target'); fs.mkdirSync(symlinkTarget); fs.symlinkSync(symlinkTarget, symlinkRuntime); - process.env.FKST_RUNTIME_ROOT = symlinkRuntime; + process.env.FKST_WORKER_RUNTIME_ROOT = symlinkRuntime; assert.throws(() => minimalEnvironment({}, 'symlink-runtime'), /not a real directory/); - process.env.FKST_RUNTIME_ROOT = path.join(temp, 'runtime'); + process.env.FKST_WORKER_RUNTIME_ROOT = path.join(temp, 'worker-runtime'); const trustedRepository = { url: 'https://example.invalid/testing/trusted-fixture.git', @@ -194,8 +725,10 @@ async function main() { repository: trustedRepository, authority: { kind: 'host-policy', ref: 'fixtures/runtime-target-boundary' }, policy_revision: 'runtime-test-boundary-v1', + human_approval_required: false, authorization_capability: false, execution_authorized: false, + promotion_authorized: false, }; assert.deepStrictEqual(validateTargetExecutionBoundary(boundary, trustedRepository, { runtimeConfigRef: { kind: 'artifact', ref: `${hostRoot}/runtime-config.json` }, @@ -211,14 +744,147 @@ async function main() { assert.throws(() => validateTargetExecutionBoundary({ ...boundary, repository: { url: 'git@example.invalid:testing/trusted-fixture.git', commit_sha: '1'.repeat(40), } }, trustedRepository), /HOST_RUNTIME_ISOLATION_REQUIRED/); + assert.throws(() => validateTargetExecutionBoundary({ + ...boundary, human_approval_required: true, + }, trustedRepository), /non-authorizing admission prerequisite/); assert.throws(() => validateTargetExecutionBoundary({ ...boundary, authorization_capability: true, }, trustedRepository), /non-authorizing admission prerequisite/); + assert.throws(() => validateTargetExecutionBoundary({ + ...boundary, execution_authorized: true, + }, trustedRepository), /non-authorizing admission prerequisite/); + assert.throws(() => validateTargetExecutionBoundary({ + ...boundary, promotion_authorized: true, + }, trustedRepository), /non-authorizing admission prerequisite/); assert.throws(() => validateTargetExecutionBoundary(boundary, trustedRepository, { runtimeConfigRef: { kind: 'artifact', ref: `${artifactRoot}/runtime-config.json` }, artifactRoot, }), /Host control namespace/); + const checkoutSource = path.join(temp, 'workspace-recovery-source'); + fs.mkdirSync(checkoutSource); + const git = (argv, cwd = checkoutSource) => { + const result = spawnSync('git', argv, { cwd, encoding: 'utf8', shell: false }); + assert.strictEqual(result.status, 0, result.stderr || result.stdout); + return String(result.stdout || '').trim(); + }; + git(['init', '--quiet']); + git(['config', 'user.email', 'workspace-recovery@example.invalid']); + git(['config', 'user.name', 'Workspace Recovery']); + fs.writeFileSync(path.join(checkoutSource, 'fixture.txt'), 'immutable fixture\n'); + git(['add', 'fixture.txt']); + git(['commit', '--quiet', '-m', 'fixture']); + const checkoutRepository = { + url: 'https://example.invalid/testing/workspace-recovery.git', + commit_sha: git(['rev-parse', 'HEAD']), + }; + const checkoutConfigRef = { + kind: 'artifact', ref: `${hostRoot}/workspace-recovery-runtime-config.json`, + }; + fs.mkdirSync(path.dirname(checkoutConfigRef.ref), { recursive: true }); + fs.writeFileSync(checkoutConfigRef.ref, `${stableStringify({ + schema: 'environment-factory.runtime-config.v1', + state_auth_key: 'workspace-recovery-state-key-which-is-long-enough', + state_mac_generation: 'workspace-recovery-v1', + repository_mirrors: { [checkoutRepository.url]: checkoutSource }, + command_environment: {}, + target_execution_boundary: { + ...boundary, + repository: checkoutRepository, + authority: { kind: 'host-policy', ref: 'fixtures/workspace-recovery' }, + }, + })}\n`); + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER = cleanupBroker; + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = sha256(fs.readFileSync(cleanupBroker)); + const checkoutInterruptions = [ + ['afterWorkspaceDirectoryCreated', 'directory-created'], + ['afterWorkspaceResourceRegistered', 'allocation-registered'], + ['afterSuccessfulCheckout', 'checkout-succeeded'], + ]; + for (const [hookName, label] of checkoutInterruptions) { + const operationId = `workspace-recovery-${label}-${process.pid}`; + const ledgerResult = await dispatch('initialize-worker-home-ledger', { + effect_id: `${operationId}/worker-home-ledger`, operation_id: operationId, + repository: checkoutRepository, artifact_root: artifactRoot, + runtime_config_ref: checkoutConfigRef, timeout_seconds: 20, + }); + const payload = { + effect_id: `${operationId}/checkout`, operation_id: operationId, + repository: checkoutRepository, worker_home_ledger_ref: ledgerResult.cleanup_ref, + working_directory: '.', artifact_root: artifactRoot, + runtime_config_ref: checkoutConfigRef, timeout_seconds: 20, output_bytes: 65536, + resource_budgets: { + cpu_millis: 60000, memory_mb: 256, disk_mb: 128, + processes: 8, network_requests: 0, output_bytes: 65536, + }, + }; + let interrupted = 0; + await assert.rejects(() => checkoutWithHooks(payload, { + [hookName]() { + interrupted += 1; + throw new Error(`simulated checkout interruption: ${label}`); + }, + }), new RegExp(`simulated checkout interruption: ${label}`)); + assert.strictEqual(interrupted, 1); + const recovered = await checkoutWithHooks(payload); + assert.strictEqual(recovered.status, 'passed'); + assert.strictEqual(recovered.resolved_commit, checkoutRepository.commit_sha); + const replayed = await checkoutWithHooks(payload); + assert.deepStrictEqual(replayed, recovered); + } + const replacementOperationId = `workspace-replacement-${process.pid}`; + const replacementWorkspaceLedger = await dispatch('initialize-worker-home-ledger', { + effect_id: `${replacementOperationId}/worker-home-ledger`, + operation_id: replacementOperationId, repository: checkoutRepository, + artifact_root: artifactRoot, runtime_config_ref: checkoutConfigRef, timeout_seconds: 20, + }); + const externalWorkspace = path.join(temp, 'external-workspace-replacement'); + fs.mkdirSync(externalWorkspace, { mode: 0o700 }); + const externalWorkspaceSentinel = path.join(externalWorkspace, 'sentinel.txt'); + fs.writeFileSync(externalWorkspaceSentinel, 'external-workspace-sentinel\n'); + let replacementWorkspacePath = null; + let displacedWorkspacePath = null; + await assert.rejects(() => checkoutWithHooks({ + effect_id: `${replacementOperationId}/checkout`, operation_id: replacementOperationId, + repository: checkoutRepository, + worker_home_ledger_ref: replacementWorkspaceLedger.cleanup_ref, + working_directory: '.', artifact_root: artifactRoot, runtime_config_ref: checkoutConfigRef, + timeout_seconds: 20, output_bytes: 65536, + resource_budgets: { + cpu_millis: 60000, memory_mb: 256, disk_mb: 128, processes: 8, + network_requests: 0, output_bytes: 65536, + }, + }, { + afterWorkspaceDirectoryCreated({ reservation }) { + replacementWorkspacePath = reservation.path; + displacedWorkspacePath = `${reservation.path}.displaced`; + fs.renameSync(reservation.path, displacedWorkspacePath); + fs.symlinkSync(externalWorkspace, reservation.path); + }, + }), /workspace identity changed after allocation/); + assert.strictEqual( + fs.readFileSync(externalWorkspaceSentinel, 'utf8'), 'external-workspace-sentinel\n', + ); + assert.strictEqual(fs.lstatSync(replacementWorkspacePath).isSymbolicLink(), true); + fs.unlinkSync(replacementWorkspacePath); + await assert.rejects(() => checkoutWithHooks({ + effect_id: `${replacementOperationId}/checkout`, operation_id: replacementOperationId, + repository: checkoutRepository, + worker_home_ledger_ref: replacementWorkspaceLedger.cleanup_ref, + working_directory: '.', artifact_root: artifactRoot, runtime_config_ref: checkoutConfigRef, + timeout_seconds: 20, output_bytes: 65536, + resource_budgets: { + cpu_millis: 60000, memory_mb: 256, disk_mb: 128, processes: 8, + network_requests: 0, output_bytes: 65536, + }, + }), /workspace allocation identity is unavailable for recovery/); + assert.strictEqual(fs.existsSync(displacedWorkspacePath), true); + assert.strictEqual( + fs.readFileSync(externalWorkspaceSentinel, 'utf8'), 'external-workspace-sentinel\n', + ); + delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER; + delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256; + const lockPath = path.join(temp, 'stale.lock'); fs.mkdirSync(lockPath); fs.writeFileSync(path.join(lockPath, 'owner.json'), `${JSON.stringify({ @@ -227,11 +893,15 @@ async function main() { process_start_identity: 'dead process', token: 'stale-owner-token', })}\n`); - const release = acquireLock(lockPath, 250); - const recovered = JSON.parse(fs.readFileSync(lockPath, 'utf8')); - assert.strictEqual(recovered.pid, process.pid); - release(); - assert.strictEqual(fs.existsSync(lockPath), false); + assert.throws( + () => acquireLock(lockPath, 250), + /legacy lock directory cleanup requires an object-bound broker/, + ); + assert.strictEqual(fs.lstatSync(lockPath).isDirectory(), true); + assert.strictEqual( + JSON.parse(fs.readFileSync(path.join(lockPath, 'owner.json'), 'utf8')).token, + 'stale-owner-token', + ); const staleAtomicLockPath = path.join(temp, 'stale-atomic.lock'); const staleAtomicOwner = { @@ -248,6 +918,102 @@ async function main() { releaseStaleAtomic(); assert.strictEqual(fs.existsSync(staleAtomicLockPath), false); + const releaseRaceLockPath = path.join(temp, 'release-race.lock'); + const releaseRaceDisplacedPath = path.join(temp, 'release-race.displaced'); + const releaseRace = acquireLock(releaseRaceLockPath, 250); + const releaseRaceOwner = JSON.parse(fs.readFileSync(releaseRaceLockPath, 'utf8')); + const releaseRaceSuccessor = { + ...releaseRaceOwner, + token: '2'.repeat(32), + }; + const originalRenameSync = fs.renameSync; + let releaseRaceInjected = false; + fs.renameSync = (sourcePath, destinationPath) => { + if (!releaseRaceInjected && sourcePath === releaseRaceLockPath) { + releaseRaceInjected = true; + originalRenameSync(sourcePath, releaseRaceDisplacedPath); + fs.writeFileSync(sourcePath, `${stableStringify(releaseRaceSuccessor)}\n`, { flag: 'wx' }); + } + return originalRenameSync(sourcePath, destinationPath); + }; + try { + releaseRace(); + } finally { + fs.renameSync = originalRenameSync; + } + assert.strictEqual(releaseRaceInjected, true); + assert.strictEqual(fs.readFileSync(releaseRaceLockPath, 'utf8'), + `${stableStringify(releaseRaceSuccessor)}\n`); + assert.strictEqual(fs.existsSync(releaseRaceDisplacedPath), true); + fs.unlinkSync(releaseRaceLockPath); + + const takeoverRaceLockPath = path.join(temp, 'takeover-race.lock'); + const takeoverRaceDisplacedPath = path.join(temp, 'takeover-race.displaced'); + const takeoverRaceStaleOwner = { + schema: 'environment-factory.lock-owner.v1', + pid: 2147483647, + process_start_identity: 'dead process', + token: '3'.repeat(32), + }; + const takeoverRaceSuccessor = { + ...releaseRaceOwner, + token: '4'.repeat(32), + }; + fs.writeFileSync(takeoverRaceLockPath, `${stableStringify(takeoverRaceStaleOwner)}\n`); + let takeoverRaceInjected = false; + fs.renameSync = (sourcePath, destinationPath) => { + if (!takeoverRaceInjected && sourcePath === takeoverRaceLockPath) { + takeoverRaceInjected = true; + originalRenameSync(sourcePath, takeoverRaceDisplacedPath); + fs.writeFileSync(sourcePath, `${stableStringify(takeoverRaceSuccessor)}\n`, { flag: 'wx' }); + } + return originalRenameSync(sourcePath, destinationPath); + }; + try { + assert.throws(() => acquireLock(takeoverRaceLockPath, 250), /lock timeout/); + } finally { + fs.renameSync = originalRenameSync; + } + assert.strictEqual(takeoverRaceInjected, true); + assert.strictEqual(fs.readFileSync(takeoverRaceLockPath, 'utf8'), + `${stableStringify(takeoverRaceSuccessor)}\n`); + assert.strictEqual(fs.existsSync(takeoverRaceDisplacedPath), true); + fs.unlinkSync(takeoverRaceLockPath); + + const pendingRaceLockPath = path.join(temp, 'pending-race.lock'); + const pendingRaceStaleOwner = { + schema: 'environment-factory.lock-owner.v1', + pid: 2147483647, + process_start_identity: 'dead process', + token: '5'.repeat(32), + }; + const pendingRacePath = `${pendingRaceLockPath}.owner.${pendingRaceStaleOwner.pid}.${pendingRaceStaleOwner.token}`; + const pendingRaceDisplacedPath = path.join(temp, 'pending-race.displaced'); + const pendingRaceSuccessor = `${stableStringify(pendingRaceStaleOwner)}\n`; + fs.writeFileSync(pendingRacePath, `${stableStringify(pendingRaceStaleOwner)}\n`); + fs.linkSync(pendingRacePath, pendingRaceLockPath); + let pendingRaceInjected = false; + fs.renameSync = (sourcePath, destinationPath) => { + if (!pendingRaceInjected && sourcePath === pendingRacePath) { + pendingRaceInjected = true; + originalRenameSync(sourcePath, pendingRaceDisplacedPath); + fs.writeFileSync(sourcePath, pendingRaceSuccessor, { flag: 'wx' }); + } + return originalRenameSync(sourcePath, destinationPath); + }; + let releasePendingRace; + try { + releasePendingRace = acquireLock(pendingRaceLockPath, 250); + } finally { + fs.renameSync = originalRenameSync; + } + assert.strictEqual(pendingRaceInjected, true); + assert.strictEqual(fs.readFileSync(pendingRacePath, 'utf8'), pendingRaceSuccessor); + assert.strictEqual(fs.existsSync(pendingRaceDisplacedPath), true); + releasePendingRace(); + assert.strictEqual(fs.existsSync(pendingRaceLockPath), false); + fs.unlinkSync(pendingRacePath); + const ownerlessLockPath = path.join(temp, 'ownerless.lock'); fs.mkdirSync(ownerlessLockPath); assert.throws(() => acquireLock(ownerlessLockPath, 250), /cannot be recovered safely/); @@ -261,105 +1027,17 @@ async function main() { assert.strictEqual(fs.lstatSync(malformedOwnerLockPath).isDirectory(), true); fs.rmSync(malformedOwnerLockPath, { recursive: true }); - const concurrentLockPath = path.join(temp, 'concurrent-stale.lock'); - const concurrentActivePath = path.join(temp, 'concurrent-stale.active'); - const concurrentEntriesPath = path.join(temp, 'concurrent-stale.entries'); - const concurrentViolationPath = path.join(temp, 'concurrent-stale.violation'); - fs.mkdirSync(concurrentLockPath); - fs.writeFileSync(path.join(concurrentLockPath, 'owner.json'), `${JSON.stringify({ - schema: 'environment-factory.lock-owner.v1', - pid: 2147483647, - process_start_identity: 'dead process', - token: 'concurrent-stale-owner-token', - })}\n`); - const lockModulePath = path.resolve(__dirname, '../bin/runtime/common.js'); - const contenderSource = [ - "'use strict';", - "const fs = require('fs');", - 'const { acquireLock } = require(process.argv[1]);', - 'const lockPath = process.argv[2];', - 'const activePath = process.argv[3];', - 'const entriesPath = process.argv[4];', - 'const violationPath = process.argv[5];', - 'const release = acquireLock(lockPath, 5000);', - 'let ownsActive = false;', - 'try {', - " fs.writeFileSync(activePath, String(process.pid), { flag: 'wx' });", - ' ownsActive = true;', - " fs.appendFileSync(entriesPath, String(process.pid) + '\\n');", - ' Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100);', - '} catch (error) {', - " fs.appendFileSync(violationPath, String(process.pid) + ':' + error.code + '\\n');", - ' process.exitCode = 1;', - '} finally {', - ' if (ownsActive) fs.unlinkSync(activePath);', - ' release();', - '}', - ].join('\n'); - const contenders = Array.from({ length: 4 }, () => new Promise((resolve, reject) => { - const child = spawn(process.execPath, [ - '-e', contenderSource, lockModulePath, concurrentLockPath, concurrentActivePath, - concurrentEntriesPath, concurrentViolationPath, - ], { stdio: ['ignore', 'ignore', 'pipe'] }); - let stderr = ''; - child.stderr.on('data', (chunk) => { stderr += chunk.toString(); }); - child.once('error', reject); - child.once('close', (code) => { - if (code === 0) resolve(); - else reject(new Error(`concurrent stale-lock contender failed: ${stderr}`)); - }); - })); - await Promise.all(contenders); - assert.strictEqual(fs.existsSync(concurrentViolationPath), false); - assert.strictEqual(fs.readFileSync(concurrentEntriesPath, 'utf8').trim().split('\n').length, 4); - assert.strictEqual(fs.existsSync(concurrentLockPath), false); - assert.strictEqual(fs.existsSync(`${concurrentLockPath}.takeover.active`), false); - - const crashedTakeoverLockPath = path.join(temp, 'crashed-takeover.lock'); - const crashedTakeoverReadyPath = path.join(temp, 'crashed-takeover.ready'); - const crashedTakeoverEnteredPath = path.join(temp, 'crashed-takeover.entered'); - fs.mkdirSync(crashedTakeoverLockPath); - fs.writeFileSync(path.join(crashedTakeoverLockPath, 'owner.json'), `${JSON.stringify({ - schema: 'environment-factory.lock-owner.v1', - pid: 2147483647, - process_start_identity: 'dead process', - token: 'crashed-takeover-stale-owner-token', - })}\n`); - const crashedTakeoverSource = [ - "'use strict';", - "const fs = require('fs');", - 'const { acquireLock } = require(process.argv[1]);', - 'const release = acquireLock(process.argv[2], 5000, {', - ' afterTakeoverAcquired() {', - " fs.writeFileSync(process.argv[3], 'ready', { flag: 'wx' });", - ' Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1000);', - ' },', - '});', - "fs.writeFileSync(process.argv[4], 'entered', { flag: 'wx' });", - 'release();', - ].join('\n'); - const crashedTakeover = spawn(process.execPath, [ - '-e', crashedTakeoverSource, lockModulePath, crashedTakeoverLockPath, - crashedTakeoverReadyPath, crashedTakeoverEnteredPath, - ], { stdio: ['ignore', 'ignore', 'pipe'] }); - let crashedTakeoverStderr = ''; - crashedTakeover.stderr.on('data', (chunk) => { crashedTakeoverStderr += chunk.toString(); }); - const readyDeadline = Date.now() + 5_000; - while (!fs.existsSync(crashedTakeoverReadyPath) && Date.now() < readyDeadline) await delay(10); - assert.strictEqual(fs.existsSync(crashedTakeoverReadyPath), true); - const takeoverMarkerPath = `${crashedTakeoverLockPath}.takeover.active`; - const takeoverOwner = JSON.parse(fs.readFileSync(takeoverMarkerPath, 'utf8')); - process.kill(takeoverOwner.pid, 'SIGKILL'); - const crashedTakeoverExit = await new Promise((resolve, reject) => { - crashedTakeover.once('error', reject); - crashedTakeover.once('close', (code) => resolve(code)); - }); - assert.notStrictEqual(crashedTakeoverExit, 0, crashedTakeoverStderr); - assert.strictEqual(fs.existsSync(crashedTakeoverEnteredPath), false); - const recoveredAfterGuardCrash = acquireLock(crashedTakeoverLockPath, 5_000); - recoveredAfterGuardCrash(); - assert.strictEqual(fs.existsSync(crashedTakeoverLockPath), false); - assert.strictEqual(fs.existsSync(takeoverMarkerPath), false); + const oversizedMetadataPath = path.join(temp, 'oversized-lock-metadata.json'); + fs.writeFileSync(oversizedMetadataPath, 'x'.repeat(8193)); + assert.throws( + () => readBoundedRegularFile(oversizedMetadataPath, 8192), + /bounded regular file is invalid/, + ); + const linkedMetadataPath = path.join(temp, 'linked-lock-metadata.json'); + fs.symlinkSync(oversizedMetadataPath, linkedMetadataPath); + assert.throws(() => readBoundedRegularFile(linkedMetadataPath, 8192)); + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER = cleanupBroker; + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = sha256(fs.readFileSync(cleanupBroker)); const startupCounter = path.join(temp, 'supervised-startup-count.txt'); const startupClaim = path.join(temp, 'supervised-startup', 'claim.json'); @@ -380,17 +1058,30 @@ async function main() { `fs.appendFileSync(${JSON.stringify(startupCounter)}, 'started\\n');`, 'setInterval(() => {}, 1000);', ].join('')]; + const supervisedWorkerRequest = { + operation_id: `supervised-node-operation-${process.pid}`, + effect_id: 'crash-window-effect', + repository: trustedRepository, + }; + const supervisedLedger = initializeWorkerHomeLedger(supervisedWorkerRequest); + supervisedWorkerRequest.worker_home_ledger_ref = supervisedLedger.cleanup_ref; + const supervisedPurpose = 'supervised:crash-window-effect'; let launchedSupervisorPid = null; + let firstStartupAllocation = null; const interrupted = startOrRecoverSupervisedProcess({ claimPath: startupClaim, argv: startupArgv, cwd: temp, createEnvironment(reservation) { - firstStartupEnvironment = minimalEnvironment( - {}, 'supervised-crash-window-first', reservation.reservation_id, + firstStartupAllocation = allocateDurableWorkerEnvironment( + supervisedWorkerRequest, supervisedPurpose, {}, reservation.reservation_id, ); + firstStartupEnvironment = firstStartupAllocation.environment; firstLease = workerEnvironmentLease(firstStartupEnvironment); - return firstStartupEnvironment; + return { + environment: firstStartupEnvironment, + worker_home_slot_id: firstStartupAllocation.slot_id, + }; }, binding: startupBinding, afterLaunch(pid) { @@ -419,6 +1110,21 @@ async function main() { assert.strictEqual(recoveredStartup.state, 'running'); assert.strictEqual(recoveredStartup.resource.pid, launchedSupervisorPid); assert.strictEqual(recoveredStartup.resource.worker_environment_lease.home, firstLease.home); + assert.strictEqual(recoveredStartup.resource.worker_home_slot_id, firstStartupAllocation.slot_id); + assert.deepStrictEqual( + verifyPersistedWorkerEnvironment( + supervisedWorkerRequest, + supervisedPurpose, + {}, + recoveredStartup.resource.worker_home_slot_id, + recoveredStartup.resource.worker_environment_lease, + ), + { + identity: firstStartupAllocation.identity, + lease: firstStartupAllocation.lease, + slot_id: firstStartupAllocation.slot_id, + }, + ); assert.strictEqual(recoveredStartup.environment_retained, false); assert.strictEqual(recoveryEnvironmentCreated, false); crashWindowResource = recoveredStartup.resource; @@ -444,6 +1150,7 @@ async function main() { }); assert.strictEqual(replayedStartup.resource.pid, launchedSupervisorPid); assert.strictEqual(replayedStartup.resource.worker_environment_lease.home, firstLease.home); + assert.strictEqual(replayedStartup.resource.worker_home_slot_id, firstStartupAllocation.slot_id); assert.strictEqual(replayEnvironmentCreated, false); await delay(50); assert.strictEqual(fs.readFileSync(startupCounter, 'utf8'), 'started\n'); @@ -737,6 +1444,56 @@ async function main() { assert.strictEqual(result.maxProcesses >= 1, true); } + const anchoredCwd = path.join(temp, 'anchored-command-cwd'); + const movedAnchoredCwd = path.join(temp, 'anchored-command-cwd-moved'); + fs.mkdirSync(anchoredCwd); + fs.writeFileSync(path.join(anchoredCwd, 'identity.txt'), 'validated-object\n'); + const anchoredCwdIdentity = pathIdentity(anchoredCwd); + const anchoredResult = await runMeasuredCommand(['/bin/cat', 'identity.txt'], { + cwd: anchoredCwd, + cwdIdentity: anchoredCwdIdentity, + timeoutMs: 2_000, + outputBytes: 1024, + afterCwdAnchored() { + fs.renameSync(anchoredCwd, movedAnchoredCwd); + fs.mkdirSync(anchoredCwd); + fs.writeFileSync(path.join(anchoredCwd, 'identity.txt'), 'replacement-object\n'); + }, + }); + assert.strictEqual(anchoredResult.exitCode, 0, anchoredResult.stderr); + assert.strictEqual(anchoredResult.stdout, 'validated-object\n'); + + const supervisedCwd = path.join(temp, 'supervised-cwd-race'); + const movedSupervisedCwd = path.join(temp, 'supervised-cwd-race-moved'); + const supervisedCwdOutput = path.join(temp, 'supervised-cwd-race-output.txt'); + fs.mkdirSync(supervisedCwd); + const supervisedCwdIdentity = pathIdentity(supervisedCwd); + let supervisedCwdLease = null; + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER = cleanupBroker; + process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = sha256(fs.readFileSync(cleanupBroker)); + const supervisedCwdRace = startOrRecoverSupervisedProcess({ + claimPath: path.join(temp, 'supervised-cwd-race-claim.json'), + argv: [process.execPath, '-e', `require('fs').writeFileSync(${JSON.stringify(supervisedCwdOutput)}, 'ran')`], + cwd: supervisedCwd, + cwdIdentity: supervisedCwdIdentity, + createEnvironment(reservation) { + const environment = minimalEnvironment({}, 'supervised-cwd-race', reservation.reservation_id); + supervisedCwdLease = workerEnvironmentLease(environment); + return environment; + }, + binding: { ...startupBinding, effect_id: 'supervised-cwd-race' }, + registrationTimeoutMs: 250, + beforeSupervisorLaunch() { + fs.renameSync(supervisedCwd, movedSupervisedCwd); + fs.mkdirSync(supervisedCwd); + }, + }); + assert.strictEqual(supervisedCwdRace.state, 'revoked'); + assert.strictEqual(fs.existsSync(supervisedCwdOutput), false); + assert.strictEqual(fs.existsSync(supervisedCwdLease.home), false); + delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER; + delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256; + const bounded = await runMeasuredCommand([ process.execPath, '-e', @@ -838,6 +1595,77 @@ async function main() { }), /resource ownership binding is invalid/); assert.strictEqual(fs.existsSync(ownedWorkspace), true); + const substitutedWorkspace = path.join(temp, 'substituted-workspace'); + const originalWorkspace = `${substitutedWorkspace}.original`; + fs.mkdirSync(substitutedWorkspace); + const substitutedResourceRef = `substituted-workspace-${process.pid}`; + const substitutedResourcePath = path.join( + process.env.FKST_DURABLE_ROOT, + 'environment-factory', + 'resources', + `${sha256(substitutedResourceRef)}.json`, + ); + const substitutedOperationId = `substituted-${process.pid}`; + fs.writeFileSync(substitutedResourcePath, `${JSON.stringify({ + schema: 'environment-factory.resource.v1', + kind: 'workspace', + operation_id: substitutedOperationId, + ref: substitutedResourceRef, + path: substitutedWorkspace, + path_identity: pathIdentity(substitutedWorkspace), + containment_root: temp, + containment_root_identity: pathIdentity(temp), + cleanup_capture_id: 'b'.repeat(64), + cleaned: false, + })}\n`); + fs.renameSync(substitutedWorkspace, originalWorkspace); + fs.mkdirSync(substitutedWorkspace); + const externalSentinel = path.join(substitutedWorkspace, 'external-sentinel.txt'); + fs.writeFileSync(externalSentinel, 'externally-owned\n'); + await assert.rejects(() => dispatch('cleanup', { + effect_id: `${substitutedOperationId}/cleanup/workspace`, + operation_id: substitutedOperationId, + artifact_root: artifactRoot, + cleanup_ref: { kind: 'resource-cleanup', ref: substitutedResourceRef }, + timeout_seconds: 1, + }), /owned directory identity changed/); + assert.strictEqual(fs.readFileSync(externalSentinel, 'utf8'), 'externally-owned\n'); + assert.strictEqual(fs.existsSync(originalWorkspace), true); + + const linkedWorkspace = path.join(temp, 'linked-workspace'); + const linkedWorkspaceTarget = path.join(temp, 'missing-external-workspace'); + fs.mkdirSync(linkedWorkspace); + const linkedResourceRef = `linked-workspace-${process.pid}`; + const linkedResourcePath = path.join( + process.env.FKST_DURABLE_ROOT, + 'environment-factory', + 'resources', + `${sha256(linkedResourceRef)}.json`, + ); + const linkedOperationId = `linked-${process.pid}`; + fs.writeFileSync(linkedResourcePath, `${JSON.stringify({ + schema: 'environment-factory.resource.v1', + kind: 'workspace', + operation_id: linkedOperationId, + ref: linkedResourceRef, + path: linkedWorkspace, + path_identity: pathIdentity(linkedWorkspace), + containment_root: temp, + containment_root_identity: pathIdentity(temp), + cleanup_capture_id: 'c'.repeat(64), + cleaned: false, + })}\n`); + fs.rmdirSync(linkedWorkspace); + fs.symlinkSync(linkedWorkspaceTarget, linkedWorkspace); + await assert.rejects(() => dispatch('cleanup', { + effect_id: `${linkedOperationId}/cleanup/workspace`, + operation_id: linkedOperationId, + artifact_root: artifactRoot, + cleanup_ref: { kind: 'resource-cleanup', ref: linkedResourceRef }, + timeout_seconds: 1, + })); + assert.strictEqual(fs.lstatSync(linkedWorkspace).isSymbolicLink(), true); + const runtimeConfigRef = { kind: 'artifact', ref: `${hostRoot}/runtime-config.json` }; const stateRef = { kind: 'artifact', ref: `${artifactRoot}/operation-state.json` }; fs.mkdirSync(hostRoot, { recursive: true }); @@ -882,6 +1710,12 @@ async function main() { else process.env.FKST_DURABLE_ROOT = previousDurable; if (previousRuntime === undefined) delete process.env.FKST_RUNTIME_ROOT; else process.env.FKST_RUNTIME_ROOT = previousRuntime; + if (previousWorkerRuntime === undefined) delete process.env.FKST_WORKER_RUNTIME_ROOT; + else process.env.FKST_WORKER_RUNTIME_ROOT = previousWorkerRuntime; + if (previousCleanupBroker === undefined) delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER; + else process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER = previousCleanupBroker; + if (previousCleanupBrokerSha256 === undefined) delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256; + else process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = previousCleanupBrokerSha256; await removeTreeEventually(temp); await removeTreeEventually(artifactRoot); await removeTreeEventually(hostRoot); diff --git a/packages/environment-factory/tests/object_bound_cleanup_backend_test.lua b/packages/environment-factory/tests/object_bound_cleanup_backend_test.lua new file mode 100644 index 00000000..b52b1db4 --- /dev/null +++ b/packages/environment-factory/tests/object_bound_cleanup_backend_test.lua @@ -0,0 +1,18 @@ +local t = fkst.test + +return { + test_object_bound_cleanup_backend_selection = function() + local candidates = { + "packages/environment-factory/tests/object_bound_cleanup_backend_test.py", + "tests/object_bound_cleanup_backend_test.py", + } + local script + for _, candidate in ipairs(candidates) do + local handle = io.open(candidate, "rb") + if handle then handle:close(); script = candidate; break end + end + t.is_true(script ~= nil) + local ok, why, code = os.execute("/usr/bin/python3 -I " .. script) + t.is_true(ok == true or code == 0, tostring(why) .. ":" .. tostring(code)) + end, +} diff --git a/packages/environment-factory/tests/object_bound_cleanup_backend_test.py b/packages/environment-factory/tests/object_bound_cleanup_backend_test.py new file mode 100644 index 00000000..5733bd93 --- /dev/null +++ b/packages/environment-factory/tests/object_bound_cleanup_backend_test.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Backend-selection tests for atomic object-bound cleanup capture.""" + +import importlib.util +import pathlib +import sys +import unittest +from unittest import mock + + +BROKER_PATH = pathlib.Path(__file__).resolve().parents[1] / "bin" / "object-bound-cleanup-broker.py" +SPEC = importlib.util.spec_from_file_location("object_bound_cleanup_broker", BROKER_PATH) +BROKER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(BROKER) + + +class Primitive: + def __init__(self): + self.calls = [] + self.argtypes = None + self.restype = None + + def __call__(self, *args): + self.calls.append(args) + return 0 + + +class Library: + pass + + +class BackendSelectionTest(unittest.TestCase): + def call(self, platform, library): + with mock.patch.object(BROKER.sys, "platform", platform), mock.patch.object( + BROKER.ctypes, "CDLL", return_value=library + ): + BROKER.rename_noreplace("source", 11, "destination", 22) + + def test_linux_uses_renameat2_with_noreplace(self): + library = Library() + library.renameat2 = Primitive() + library.renameatx_np = Primitive() + self.call("linux", library) + self.assertEqual(len(library.renameat2.calls), 1) + self.assertEqual(library.renameat2.calls[0][0], 11) + self.assertEqual(library.renameat2.calls[0][2], 22) + self.assertEqual(library.renameat2.calls[0][4], BROKER.RENAME_NOREPLACE) + self.assertEqual(library.renameatx_np.calls, []) + + def test_darwin_uses_renameatx_np_with_exclusive_flag(self): + library = Library() + library.renameat2 = Primitive() + library.renameatx_np = Primitive() + self.call("darwin", library) + self.assertEqual(len(library.renameatx_np.calls), 1) + self.assertEqual(library.renameatx_np.calls[0][0], 11) + self.assertEqual(library.renameatx_np.calls[0][2], 22) + self.assertEqual(library.renameatx_np.calls[0][4], BROKER.RENAME_EXCL) + self.assertEqual(library.renameat2.calls, []) + + def test_missing_platform_primitive_fails_closed(self): + with self.assertRaisesRegex(BROKER.CleanupBlocked, "atomic-capture-unsupported"): + self.call("linux", Library()) + with self.assertRaisesRegex(BROKER.CleanupBlocked, "atomic-capture-unsupported"): + self.call("darwin", Library()) + + def test_unsupported_platform_has_no_ordinary_rename_fallback(self): + library = Library() + library.rename = Primitive() + with mock.patch.object(BROKER.os, "rename", side_effect=AssertionError("unsafe fallback")): + with self.assertRaisesRegex(BROKER.CleanupBlocked, "atomic-capture-unsupported"): + self.call("win32", library) + self.assertEqual(library.rename.calls, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/environment-factory/tests/ports_test.lua b/packages/environment-factory/tests/ports_test.lua index 8878bfd8..d8b7367c 100644 --- a/packages/environment-factory/tests/ports_test.lua +++ b/packages/environment-factory/tests/ports_test.lua @@ -88,7 +88,8 @@ return { test_host_runtime_is_preferred_and_resolve_rejects_missing_methods = function() local host = {} for _, name in ipairs({ - "load_state", "save_state", "load_authorization_bundle", "authorize_claim_ports", "checkout", + "load_state", "save_state", "load_authorization_bundle", "authorize_claim_ports", + "initialize_worker_home_ledger", "checkout", "remaining_budget", "create_readiness_attempt", "run_argv", "wait_readiness", "cleanup", "write_receipt", }) do host[name] = function() return name end end with_globals({ environment_factory_runtime = host }, function() diff --git a/packages/testing-runner/structured_execution.lua b/packages/testing-runner/structured_execution.lua index 8f788b17..a3e94b9c 100644 --- a/packages/testing-runner/structured_execution.lua +++ b/packages/testing-runner/structured_execution.lua @@ -256,6 +256,15 @@ local function valid_effect_response(case, response) and type(response.headers) == "table" end +local function effect_authorization_time(ports, context) + return ports.now({ + artifact_root = context.request.artifact_root, + operation_id = context.environment.operation_id, + trace_id = context.request.trace_id, + dedup_key = context.request.dedup_key, + }) +end + local function execute_case(case, grant, ports, context) if case.skip_reason ~= nil then return { @@ -311,7 +320,7 @@ local function execute_case(case, grant, ports, context) trace_id = context.request.trace_id, dedup_key = context.request.dedup_key, }) local receipt_ok = pcall(execution_contract.validate_effect_authorization_receipt, - receipt, envelope, context.now) + receipt, envelope, effect_authorization_time(ports, context)) local authorization_path = context.request.artifact_root .. "/authorization/" .. case.case_id .. ".json" if not receipt_ok or ports.write_artifact(authorization_path, receipt) ~= true then @@ -366,12 +375,10 @@ local function execute_case(case, grant, ports, context) trace_id = context.request.trace_id, dedup_key = context.request.dedup_key, }) local receipt_ok = pcall(execution_contract.validate_effect_authorization_receipt, - receipt, envelope, context.now) + receipt, envelope, effect_authorization_time(ports, context)) local authorization_path = context.request.artifact_root .. "/authorization/" .. case.case_id .. ".json" - if not receipt_ok or ports.write_artifact(authorization_path, receipt) ~= true then - error("testing-runner: structured-execution: malformed HTTP authorization receipt") - end + if not receipt_ok or ports.write_artifact(authorization_path, receipt) ~= true then error("testing-runner: structured-execution: malformed HTTP authorization receipt") end if receipt.decision ~= "allow" then return { case_id = case.case_id, kind = case.kind, status = "error", @@ -667,8 +674,9 @@ function M.run(request, ports) trace_id = request.trace_id, dedup_key = request.dedup_key, }) - local grant_ok = pcall(execution_contract.validate_grant, grant.value, now) - if not grant_ok or grant.value.plan_sha256 ~= request.test_plan_sha256 + local authorization_window_ok = pcall(execution_contract.validate_grant_authorization_window, + preauthorization.value, grant.value, now) + if not authorization_window_ok or grant.value.plan_sha256 ~= request.test_plan_sha256 or grant.value.parent_authorization_sha256 ~= preauthorization.digest or grant.value.environment_receipt_sha256 ~= request.environment_receipt_sha256 or not same_repository(grant.value.repository, request.repository) @@ -760,7 +768,6 @@ function M.run(request, ports) plan = plan, grant = grant, claim = claim, - now = now, } for index, case in ipairs(plan.value.cases) do local started_at = current_time(ports, request, environment.value) diff --git a/packages/testing-runner/tests/authorization_lineage_projection_test.lua b/packages/testing-runner/tests/authorization_lineage_projection_test.lua new file mode 100644 index 00000000..6535327a --- /dev/null +++ b/packages/testing-runner/tests/authorization_lineage_projection_test.lua @@ -0,0 +1,270 @@ +local canonical_json = require("contract.canonical_json") +local lineage = require("contract.execution_authorization_lineage") +local projection = require("testing_runtime.authorization_lineage_projection") +local sha256 = require("contract.sha256") +local t = fkst.test + +local run_id = "authorization-lineage-projection-run" +local root = ".testing/runs/" .. run_id +local repository = { + url = "https://example.invalid/testing/fixture.git", + commit_sha = string.rep("1", 40), +} + +local function digest(char) return string.rep(char, 64) end + +local function copy(value) + if type(value) ~= "table" then return value end + local result = {} + for key, item in pairs(value) do result[copy(key)] = copy(item) end + return result +end + +local function memory_store(options) + options = options or {} + local store = { artifacts = {} } + + function store:load(path) + return self.artifacts[path] and copy(self.artifacts[path]) or nil + end + + function store:write_raw(path, body) + if options.reject_writes then return false end + if self.artifacts[path] ~= nil then return self.artifacts[path].raw == body end + self.artifacts[path] = { + raw = body, + digest = sha256.hex(body), + value = json.decode(body), + } + return true + end + + return store +end + +local function new_projector(store, overrides) + local options = { + store = store or memory_store(), + sha256 = sha256.hex, + fingerprint_secret = string.rep("private-fixture-secret-", 2), + repository = copy(repository), + run_id = run_id, + trace_id = "trace-authorization-lineage-projection", + dedup_key = run_id, + artifact_root = root, + } + for key, value in pairs(overrides or {}) do options[key] = value end + return projection.new(options), options +end + +local function source(fields) + local value = { + repository = copy(repository), + run_id = run_id, + trace_id = "trace-authorization-lineage-projection", + dedup_key = run_id, + } + for key, item in pairs(fields) do value[key] = copy(item) end + return value +end + +local function receipt_fields(value) + local result = copy(value) + result.repository = nil + result.run_id = nil + result.trace_id = nil + result.dedup_key = nil + return result +end + +local function write_lineage(projector) + local expected = {} + local artifacts = {} + expected.profile_claim = source({ + profile_source_ref = { kind = "host-profile-policy", ref = "policies/profile-v1" }, + profile_artifact_ref = root .. "/authorization/project-profile.json", + profile_artifact_sha256 = digest("1"), + profile_sha256 = digest("2"), + profile_revision = "profile-v1", + approval_artifact_ref = root .. "/authorization/profile-approval.json", + approval_artifact_sha256 = digest("3"), + approval_id = "profile-approval-1", + approval_sha256 = digest("4"), + approval_authority = { kind = "host-policy", ref = "policies/profile-approval-v1" }, + policy_revision = "profile-policy-v1", + evidence_ref = { kind = "signed-attestation", ref = "attestations/profile-approval-1" }, + validation_receipt_ref = root .. "/authorization/profile-validation.json", + validation_receipt_sha256 = digest("5"), + claim_fingerprint_sha256 = digest("6"), + claimed_at = "2026-09-10T00:01:00Z", + }) + artifacts.profile_claim = projector:write_receipt( + "profile_claim", "profile-claim-1", "2026-09-10T00:01:00Z", + receipt_fields(expected.profile_claim), expected.profile_claim) + + expected.preauthorization_claim = source({ + profile_claim_receipt_ref = artifacts.profile_claim.ref, + profile_claim_receipt_sha256 = artifacts.profile_claim.sha256, + preauthorization_ref = root .. "/execution/preauthorization.json", + preauthorization_sha256 = digest("7"), + authorization_id = "preauthorization-1", + profile_sha256 = digest("2"), + case_catalog_ref = root .. "/execution/case-catalog.json", + case_catalog_sha256 = digest("8"), + plan_ref = root .. "/execution/structured-plan.json", + plan_sha256 = digest("9"), + environment_receipt_ref = root .. "/environment/ready.json", + environment_receipt_sha256 = digest("a"), + authority = { kind = "host-policy", ref = "policies/execution-v1" }, + policy_revision = "execution-v1", + evidence_ref = { kind = "signed-attestation", ref = "attestations/preauthorization-1" }, + claim_fingerprint_sha256 = digest("b"), + claimed_at = "2026-09-10T00:02:00Z", + }) + artifacts.preauthorization_claim = projector:write_receipt( + "preauthorization_claim", "preauthorization-claim-1", "2026-09-10T00:02:00Z", + receipt_fields(expected.preauthorization_claim), expected.preauthorization_claim) + + expected.grant_verification = source({ + preauthorization_claim_receipt_ref = artifacts.preauthorization_claim.ref, + preauthorization_claim_receipt_sha256 = artifacts.preauthorization_claim.sha256, + grant_ref = root .. "/execution/execution-grant.json", + grant_sha256 = digest("c"), + grant_id = "execution-grant-1", + parent_authorization_ref = root .. "/execution/preauthorization.json", + parent_authorization_sha256 = digest("7"), + plan_ref = root .. "/execution/structured-plan.json", + plan_sha256 = digest("9"), + environment_receipt_ref = root .. "/environment/ready.json", + environment_receipt_sha256 = digest("a"), + authority = { kind = "host-policy", ref = "policies/execution-v1" }, + policy_revision = "execution-v1", + evidence_ref = { kind = "signed-attestation", ref = "attestations/execution-grant-1" }, + verifier_ref = { kind = "host-verifier", ref = "verifiers/execution-v1" }, + verification_id = "grant-verification-1", + verified_at = "2026-09-10T00:03:00Z", + }) + artifacts.grant_verification = projector:write_receipt( + "grant_verification", "grant-verification-1", "2026-09-10T00:03:00Z", + receipt_fields(expected.grant_verification), expected.grant_verification) + + expected.execution_claim = source({ + grant_verification_receipt_ref = artifacts.grant_verification.ref, + grant_verification_receipt_sha256 = artifacts.grant_verification.sha256, + preauthorization_claim_receipt_ref = artifacts.preauthorization_claim.ref, + preauthorization_claim_receipt_sha256 = artifacts.preauthorization_claim.sha256, + grant_ref = root .. "/execution/execution-grant.json", + grant_sha256 = digest("c"), + grant_id = "execution-grant-1", + plan_ref = root .. "/execution/structured-plan.json", + plan_sha256 = digest("9"), + environment_receipt_ref = root .. "/environment/ready.json", + environment_receipt_sha256 = digest("a"), + artifact_root = root .. "/execution", + operation_id = run_id, + claim_fingerprint_sha256 = digest("d"), + claimed_at = "2026-09-10T00:04:00Z", + }) + artifacts.execution_claim = projector:write_receipt( + "execution_claim", "execution-claim-1", "2026-09-10T00:04:00Z", + receipt_fields(expected.execution_claim), expected.execution_claim) + + expected.execution_completion = source({ + execution_claim_receipt_ref = artifacts.execution_claim.ref, + execution_claim_receipt_sha256 = artifacts.execution_claim.sha256, + result_ref = root .. "/execution/execution.json", + result_sha256 = digest("e"), + case_result_set_ref = root .. "/execution/case-result-set.json", + case_result_set_artifact_sha256 = digest("f"), + evidence_manifest_ref = root .. "/execution/evidence-manifest.json", + evidence_manifest_artifact_sha256 = digest("0"), + completed_at = "2026-09-10T00:05:00Z", + }) + artifacts.execution_completion = projector:write_receipt( + "execution_completion", "execution-completion-1", "2026-09-10T00:05:00Z", + receipt_fields(expected.execution_completion), expected.execution_completion) + + return artifacts, expected +end + +return { + test_persists_complete_canonical_lineage_and_replays_immutably = function() + local store = memory_store() + local projector = new_projector(store) + local artifacts, expected = write_lineage(projector) + local index = projector:write_index("2026-09-10T00:06:00Z", artifacts, expected) + t.eq(index.ref, root .. "/authorization-lineage/index.json") + t.eq(index.value.status, "complete") + t.eq(index.value.lineage_complete, true) + t.eq(index.value.human_approval_required, false) + t.eq(index.value.authorization_capability, false) + t.eq(index.value.execution_authorized, false) + t.eq(index.value.promotion_authorized, false) + t.eq(index.value.receipts.execution_completion.sha256, artifacts.execution_completion.sha256) + + local replayed, replay_expected = write_lineage(projector) + local replayed_index = projector:write_index("2026-09-10T00:06:00Z", replayed, replay_expected) + t.eq(replayed_index.sha256, index.sha256) + t.eq(projector:load_receipt("profile_claim", expected.profile_claim).sha256, + artifacts.profile_claim.sha256) + end, + + test_fingerprint_is_private_deterministic_and_domain_separated = function() + local projector = new_projector(memory_store()) + local first = projector:fingerprint("execution-claim", "private-claim-1") + t.eq(#first, 64) + t.eq(first, projector:fingerprint("execution-claim", "private-claim-1")) + t.is_true(first ~= projector:fingerprint("execution-claim", "private-claim-2")) + t.is_true(first ~= projector:fingerprint("grant-claim", "private-claim-1")) + end, + + test_constructor_and_identity_inputs_fail_closed = function() + t.raises(function() projection.new(nil) end) + t.raises(function() new_projector(memory_store(), { fingerprint_secret = "short" }) end) + t.raises(function() new_projector(memory_store(), { run_id = "bad run" }) end) + t.raises(function() new_projector(memory_store(), { artifact_root = root .. "/foreign" }) end) + local projector = new_projector(memory_store()) + t.raises(function() projector:path("unknown") end) + t.raises(function() projector:fingerprint("bad domain", "claim") end) + end, + + test_receipt_writes_reject_missing_sources_shadowing_and_storage_failure = function() + local projector = new_projector(memory_store()) + t.raises(function() projector:write_receipt("unknown", "receipt", "2026-09-10T00:00:00Z", {}, {}) end) + t.raises(function() + projector:write_receipt("profile_claim", "receipt", "2026-09-10T00:00:00Z", + { run_id = "shadow" }, {}) + end) + local rejecting = new_projector(memory_store({ reject_writes = true })) + local expected = source({}) + t.raises(function() + rejecting:write_receipt("profile_claim", "receipt", "2026-09-10T00:00:00Z", {}, expected) + end) + end, + + test_loading_and_indexing_require_canonical_complete_predecessors = function() + local store = memory_store() + local projector = new_projector(store) + t.raises(function() projector:load_receipt("profile_claim", {}) end) + t.raises(function() projector:load_receipt("profile_claim", nil) end) + t.raises(function() projector:write_index("2026-09-10T00:00:00Z", nil, {}) end) + + local artifacts, expected = write_lineage(projector) + local profile_path = projector:path("profile_claim") + store.artifacts[profile_path].raw = canonical_json.encode({ foreign = true }) + t.raises(function() projector:load_receipt("profile_claim", expected.profile_claim) end) + + local foreign_value = copy(artifacts.profile_claim.value) + foreign_value.profile_revision = "foreign-profile" + local foreign_raw = canonical_json.encode(foreign_value) + store.artifacts[profile_path] = { + raw = foreign_raw, + digest = sha256.hex(foreign_raw), + value = foreign_value, + } + t.raises(function() + projector:write_receipt("profile_claim", "profile-claim-1", "2026-09-10T00:01:00Z", + receipt_fields(expected.profile_claim), expected.profile_claim) + end) + end, +} diff --git a/packages/testing-runner/tests/execution_authorization_lineage_contract_test.lua b/packages/testing-runner/tests/execution_authorization_lineage_contract_test.lua index 66b7404f..9ef8ed04 100644 --- a/packages/testing-runner/tests/execution_authorization_lineage_contract_test.lua +++ b/packages/testing-runner/tests/execution_authorization_lineage_contract_test.lua @@ -15,6 +15,12 @@ end local function ref(run_id, suffix) return ".testing/runs/" .. run_id .. "/" .. suffix end +local function expect_failure(fragment, fn) + local ok, err = pcall(fn) + t.eq(ok, false) + t.is_true(tostring(err):find(fragment, 1, true) ~= nil) +end + local run_id = "authorization-lineage-run" local repository = { url = "https://example.invalid/testing/fixture.git", @@ -26,7 +32,9 @@ local function envelope(value) repository = copy(repository), run_id = run_id, trace_id = "trace-authorization-lineage", dedup_key = run_id, recorded_at = "2026-09-10T00:10:00Z", source_max_uses = 1, - evidence_role = "audit-only", authorization_capability = false, reusable = false, + evidence_role = "audit-only", human_approval_required = false, + authorization_capability = false, + execution_authorized = false, promotion_authorized = false, reusable = false, } for key, item in pairs(value) do result[key] = copy(item) end return result @@ -36,7 +44,8 @@ local function expected(value) local result = copy(value) for _, key in ipairs({ "schema", "status", "receipt_id", "recorded_at", "source_max_uses", - "evidence_role", "authorization_capability", "reusable", + "evidence_role", "human_approval_required", "authorization_capability", "execution_authorized", + "promotion_authorized", "reusable", }) do result[key] = nil end return result end @@ -170,8 +179,9 @@ local function lineage_fixture() schema = lineage.schemas.lineage_index, status = "complete", repository = copy(repository), run_id = run_id, trace_id = "trace-authorization-lineage", dedup_key = run_id, recorded_at = "2026-09-10T00:10:00Z", receipts = bindings, lineage_complete = true, - source_max_uses = 1, evidence_role = "audit-only", - authorization_capability = false, reusable = false, + source_max_uses = 1, evidence_role = "audit-only", human_approval_required = false, + authorization_capability = false, execution_authorized = false, + promotion_authorized = false, reusable = false, } return index, artifacts, expected_values end @@ -213,12 +223,37 @@ return { t.eq(lineage.validate_profile_claim_receipt(value, expected(value)), value) end, + test_execution_claim_cannot_be_recorded_before_the_claim_event = function() + local value = execution_claim() + value.recorded_at = "2026-09-10T00:03:00Z" + t.raises(function() lineage.validate_execution_claim_receipt(value, expected(value)) end) + end, + + test_profile_claim_cannot_be_recorded_before_the_claim_event = function() + local value = profile_claim() + value.recorded_at = "2026-09-10T00:00:59Z" + expect_failure("malformed-time: profile-approval-claim-receipt cannot be recorded before its claim", function() + lineage.validate_profile_claim_receipt(value, expected(value)) + end) + end, + + test_execution_completion_cannot_be_recorded_before_completion = function() + local value = execution_completion() + value.recorded_at = "2026-09-10T00:04:59Z" + expect_failure("malformed-time: execution-completion-receipt cannot be recorded before completion", function() + lineage.validate_execution_completion_receipt(value, expected(value)) + end) + end, + test_exported_claim_receipts_reject_raw_claim_handles_and_capability_flags = function() local value = execution_claim() value.claim_id = "runtime-fence-handle" t.raises(function() lineage.validate_execution_claim_receipt(value, expected(value)) end) for _, mutate in ipairs({ + function(item) item.human_approval_required = true end, function(item) item.authorization_capability = true end, + function(item) item.execution_authorized = true end, + function(item) item.promotion_authorized = true end, function(item) item.reusable = true end, function(item) item.source_max_uses = 2 end, function(item) item.evidence_role = "authorization" end, @@ -365,4 +400,107 @@ return { reseal_lineage(index, artifacts, expected_values) t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) end, + + test_receipts_reject_malformed_identity_digest_repository_pointer_and_time = function() + local cases = { + { + factory = profile_claim, + validate = lineage.validate_profile_claim_receipt, + mutate = function(value) value.receipt_id = "" end, + }, + { + factory = profile_claim, + validate = lineage.validate_profile_claim_receipt, + mutate = function(value) value.profile_sha256 = "bad" end, + }, + { + factory = profile_claim, + validate = lineage.validate_profile_claim_receipt, + mutate = function(value) value.repository.url = "http://example.invalid/repository.git" end, + }, + { + factory = profile_claim, + validate = lineage.validate_profile_claim_receipt, + mutate = function(value) value.repository.commit_sha = "mutable" end, + }, + { + factory = profile_claim, + validate = lineage.validate_profile_claim_receipt, + mutate = function(value) value.profile_artifact_ref = "outside.json" end, + }, + { + factory = profile_claim, + validate = lineage.validate_profile_claim_receipt, + mutate = function(value) value.schema = "unknown-profile-claim" end, + }, + { + factory = preauthorization_claim, + validate = lineage.validate_preauthorization_claim_receipt, + mutate = function(value) value.claimed_at = "2026-09-10T00:10:01Z" end, + }, + { + factory = grant_verification, + validate = lineage.validate_grant_verification_receipt, + mutate = function(value) value.verified_at = "2026-09-10T00:10:01Z" end, + }, + { + factory = execution_claim, + validate = lineage.validate_execution_claim_receipt, + mutate = function(value) value.artifact_root = ref("another-run", "execution") end, + }, + } + for _, item in ipairs(cases) do + local value = item.factory() + item.mutate(value) + t.raises(function() item.validate(value, expected(value)) end) + end + end, + + test_all_receipt_families_reject_execution_authority_capability = function() + for _, item in ipairs({ + { factory = profile_claim, validate = lineage.validate_profile_claim_receipt }, + { factory = preauthorization_claim, validate = lineage.validate_preauthorization_claim_receipt }, + { factory = grant_verification, validate = lineage.validate_grant_verification_receipt }, + { factory = execution_claim, validate = lineage.validate_execution_claim_receipt }, + { factory = execution_completion, validate = lineage.validate_execution_completion_receipt }, + }) do + local value = item.factory() + value.execution_authorized = true + t.raises(function() item.validate(value, expected(value)) end) + end + end, + + test_index_rejects_capability_missing_sources_noncanonical_and_foreign_receipts = function() + local index, artifacts, expected_values = lineage_fixture() + index.authorization_capability = true + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + + index, artifacts, expected_values = lineage_fixture() + index.human_approval_required = true + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + + index, artifacts, expected_values = lineage_fixture() + index.execution_authorized = true + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + + index, artifacts, expected_values = lineage_fixture() + index.promotion_authorized = true + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + + index, artifacts, expected_values = lineage_fixture() + t.raises(function() lineage.validate_lineage_index(index, nil, expected_values) end) + + index, artifacts, expected_values = lineage_fixture() + index.receipts.profile_claim.ref = ref(run_id, "authorization-lineage/not-canonical.json") + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + + index, artifacts, expected_values = lineage_fixture() + artifacts.profile_claim = nil + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + + index, artifacts, expected_values = lineage_fixture() + artifacts.profile_claim.value.repository.commit_sha = string.rep("2", 40) + reseal_lineage(index, artifacts, expected_values) + t.raises(function() lineage.validate_lineage_index(index, artifacts, expected_values) end) + end, } diff --git a/packages/testing-runner/tests/generic_host_workflow_qa_test.lua b/packages/testing-runner/tests/generic_host_workflow_qa_test.lua new file mode 100644 index 00000000..3ca5581e --- /dev/null +++ b/packages/testing-runner/tests/generic_host_workflow_qa_test.lua @@ -0,0 +1,17 @@ +local fixture_factory = require("testing_runtime.tests.runtime_client_fixture") +local generic_host = require("testing_runtime.generic_host_workflow_qa") +local t = fkst.test + +return { + test_production_reconciles_preauthorization_claim_through_host_runtime = function() + local fixture = fixture_factory.new({ + ["host-reconcile-preauthorization-claim"] = function(payload) + return { reconciled = payload.dedup_key == "dedup-reconcile" } + end, + }) + local ports = generic_host.production(fixture.options) + t.eq(ports.reconcile_preauthorization_claim({ dedup_key = "dedup-reconcile" }), true) + t.eq(ports.reconcile_preauthorization_claim({ dedup_key = "foreign" }), false) + t.eq(fixture.effect_calls()[1].name, "host-reconcile-preauthorization-claim") + end, +} diff --git a/packages/testing-runner/tests/structured_authorization_contract_test.lua b/packages/testing-runner/tests/structured_authorization_contract_test.lua index d3dd6a08..bb9ed8b1 100644 --- a/packages/testing-runner/tests/structured_authorization_contract_test.lua +++ b/packages/testing-runner/tests/structured_authorization_contract_test.lua @@ -123,6 +123,9 @@ return { local foreign = fixtures.copy(receipt) foreign.fence_id = "claim-foreign" t.raises(function() contract.validate_effect_authorization_receipt(foreign, envelope) end) + foreign = fixtures.copy(receipt) + foreign.envelope_sha256 = string.rep("f", 64) + t.raises(function() contract.validate_effect_authorization_receipt(foreign, envelope) end) end, test_http_action_envelope_and_receipt_are_grant_bound = function() @@ -163,6 +166,18 @@ return { local cli_schema = fixtures.copy(envelope) cli_schema.schema = contract.schemas.cli_action_envelope t.raises(function() contract.validate_action_envelope(cli_schema) end) + + local unsupported = fixtures.copy(envelope) + unsupported.capability = "direct-http" + t.raises(function() contract.validate_http_action_envelope(unsupported) end) + + local unbounded = fixtures.copy(envelope) + unbounded.resource_bounds.output_bytes = 1023 + t.raises(function() contract.validate_http_action_envelope(unbounded) end) + + local unknown = fixtures.copy(envelope) + unknown.schema = "testing-unknown-action-envelope.v1" + t.raises(function() contract.validate_action_envelope(unknown) end) end, @@ -179,6 +194,23 @@ return { t.eq(grant.max_uses, 1) end, + test_grant_validity_must_be_contained_by_parent_preauthorization = function() + local request = fixtures.request() + local plan = fixtures.plan(request) + for _, mutate in ipairs({ + function(value) value.issued_at = "2026-07-19T23:59:59Z" end, + function(value) value.expires_at = "2026-07-20T01:00:01Z" end, + }) do + local grant_values = values() + mutate(grant_values) + t.raises(function() + contract.derive_grant(preauthorization(request), fixtures.digest_authorization, + plan, request.test_plan_sha256, request.environment_receipt_sha256, + grant_request(request), grant_values) + end) + end + end, + test_rejects_plan_capability_escalation = function() local request = fixtures.request() local plan = fixtures.plan(request) diff --git a/packages/testing-runner/tests/structured_execution_helpers.lua b/packages/testing-runner/tests/structured_execution_helpers.lua index 2c479051..e9e28806 100644 --- a/packages/testing-runner/tests/structured_execution_helpers.lua +++ b/packages/testing-runner/tests/structured_execution_helpers.lua @@ -1,4 +1,6 @@ local M = {} +local canonical_json = require("contract.canonical_json") +local sha256 = require("contract.sha256") local json = require("testing_runtime.json") local sha256_bytes = require("tests.fixtures.sha256_helpers") @@ -84,7 +86,7 @@ function M.authorization_receipt(envelope, decision, reason) return { schema = "testing-effect-authorization-receipt.v1", decision = decision or "allow", reason_code = reason or "authorized", receipt_id = "receipt-110", - envelope_sha256 = string.rep("5", 64), + envelope_sha256 = sha256.hex(canonical_json.encode(envelope)), evaluated_input_digests = { profile = M.digest_profile, validation_receipt = M.digest_validation, preauthorization = M.digest_authorization, environment_receipt = M.digest_environment, plan = envelope.plan_sha256, grant = M.digest_grant }, diff --git a/packages/testing-runner/tests/structured_execution_test.lua b/packages/testing-runner/tests/structured_execution_test.lua index 08d17c7f..94fbf525 100644 --- a/packages/testing-runner/tests/structured_execution_test.lua +++ b/packages/testing-runner/tests/structured_execution_test.lua @@ -158,6 +158,58 @@ return { t.eq(writes[receipt_path].decision, "deny") end, + test_http_pep_denial_records_receipt_and_performs_zero_http_effects = function() + local request = fixtures.request() + local plan = fixtures.plan(request, { { + case_id = "health-api", kind = "http", + request = { method = "GET", url = "http://127.0.0.1:4173/health", headers = {} }, + timeout_seconds = 10, + assertions = { { type = "status-code", expected = 200 } }, + } }) + local grant = fixtures.grant(request, { + cli = {}, + http = { { + origin = "http://127.0.0.1:4173", methods = { "GET" }, path_prefixes = { "/health" }, + } }, + }) + local ports, effects, writes = runtime(fixtures.artifacts(request, plan, grant), { + authorize_http_effect = function(input) + return fixtures.authorization_receipt(input.action_envelope, "deny", "scope-denied") + end, + }) + local result = structured_execution.run(request, ports) + t.eq(result.status, "blocked") + t.eq(result.error_count, 1) + t.eq(#effects, 0) + local receipt_path = request.artifact_root .. "/authorization/health-api.json" + t.eq(writes[receipt_path].decision, "deny") + t.eq(writes[receipt_path].reason_code, "scope-denied") + end, + + test_malformed_http_pep_receipt_blocks_before_http_effect = function() + local request = fixtures.request() + local plan = fixtures.plan(request, { { + case_id = "health-api", kind = "http", + request = { method = "GET", url = "http://127.0.0.1:4173/health", headers = {} }, + timeout_seconds = 10, + assertions = { { type = "status-code", expected = 200 } }, + } }) + local grant = fixtures.grant(request, { + cli = {}, + http = { { + origin = "http://127.0.0.1:4173", methods = { "GET" }, path_prefixes = { "/health" }, + } }, + }) + local ports, effects = runtime(fixtures.artifacts(request, plan, grant), { + authorize_http_effect = function() return { decision = "allow" } end, + }) + local result = structured_execution.run(request, ports) + t.eq(result.status, "blocked") + t.eq(result.classification, "harness-tooling-issue") + t.is_true(result.message:find("malformed HTTP authorization receipt", 1, true) ~= nil) + t.eq(#effects, 0) + end, + test_unpersisted_local_pep_receipt_blocks_before_cli_effect = function() local request = fixtures.request() local ports, effects = runtime(fixtures.artifacts(request), { @@ -224,8 +276,19 @@ return { local ports, effects, writes = runtime(artifacts, { now = { "2026-07-20T00:30:00Z", "2026-07-20T00:30:01Z", "2026-07-20T00:30:02Z", - "2026-07-20T00:30:03Z", "2026-07-20T00:30:05Z", + "2026-07-20T00:30:02Z", "2026-07-20T00:30:03Z", "2026-07-20T00:30:04Z", + "2026-07-20T00:30:05Z", }, + authorize_cli_effect = function(input) + local receipt = fixtures.authorization_receipt(input.action_envelope) + receipt.issued_at = "2026-07-20T00:30:02Z" + return receipt + end, + authorize_http_effect = function(input) + local receipt = fixtures.authorization_receipt(input.action_envelope) + receipt.issued_at = "2026-07-20T00:30:04Z" + return receipt + end, verify_grant = function(input) t.eq(input.grant_sha256, fixtures.digest_grant) t.eq(input.grant.authority.ref, "testing-authority") @@ -563,6 +626,28 @@ return { t.eq(#legacy.cases[1].assertions, 0) end, + test_assertion_facts_fail_closed_if_the_bound_plan_changes_during_projection = function() + local request = fixtures.request() + local plan = fixtures.plan(request) + local artifacts = fixtures.artifacts(request, plan) + local ports, effects = runtime(artifacts, { + write_artifact = function(path, value, writes, stored) + fixtures.persist_write(stored, writes, path, value) + if path == request.artifact_root .. "/evidence/cli-version.json" then + plan.cases[1].assertions[1].type = "stdout-contains" + end + return true + end, + }) + + local result = structured_execution.run(request, ports) + + t.eq(result.status, "blocked") + t.eq(result.classification, "harness-tooling-issue") + t.is_true(result.message:find("assertion execution facts differ from plan", 1, true) ~= nil) + t.eq(#effects, 1) + end, + test_malformed_table_effect_responses_cannot_pass_or_report_product_defects = function() local request = fixtures.request() local ports, _, writes = runtime(fixtures.artifacts(request), { exec_result = {} }) @@ -617,6 +702,23 @@ return { t.eq(#effects, 0) end, + test_expired_parent_preauthorization_fails_before_replay_claim_or_effect = function() + local request = fixtures.request() + local artifacts = fixtures.artifacts(request) + artifacts[request.preauthorization_ref].value.expires_at = "2026-07-20T00:10:00Z" + local claims = 0 + local ports, effects = runtime(artifacts, { + replay_guard = function() + claims = claims + 1 + return { status = "claimed", claim_id = "late-parent" } + end, + }) + local result = structured_execution.run(request, ports) + t.eq(result.status, "blocked") + t.eq(claims, 0) + t.eq(#effects, 0) + end, + test_malformed_plan_fails_before_replay_claim_or_effect = function() local request = fixtures.request() local plan = fixtures.plan(request) diff --git a/packages/testing-runner/tests/workflow_qa_host_adapter_test.lua b/packages/testing-runner/tests/workflow_qa_host_adapter_test.lua index 1b21bfd7..ceec1370 100644 --- a/packages/testing-runner/tests/workflow_qa_host_adapter_test.lua +++ b/packages/testing-runner/tests/workflow_qa_host_adapter_test.lua @@ -365,6 +365,22 @@ return { t.raises(function() adapter.handle_execution_grant(request, {}) end) end, + test_derived_grants_reject_foreign_environment_and_http_origins = function() + local request, materials = fixture() + table.insert(materials.preauthorization.capabilities.http, { + origin = "http://127.0.0.1:43110", methods = { "GET" }, path_prefixes = { "/" }, + }) + t.raises(function() adapter.derive_execution_grant(request, materials, values()) end) + + request, materials = fixture() + materials.environment_receipt_sha256 = digest("7") + t.raises(function() adapter.derive_execution_grant(request, materials, values()) end) + + request, materials = browser_fixture() + materials.environment_receipt_sha256 = digest("7") + t.raises(function() adapter.derive_execution_grant(request, materials, browser_values()) end) + end, + test_replayed_grant_rejects_foreign_http_origin_and_malformed_artifact = function() local request, materials = fixture() local ports, artifacts = runtime(request, materials) diff --git a/packages/workflow-qa/cleanup_state.lua b/packages/workflow-qa/cleanup_state.lua new file mode 100644 index 00000000..303d312c --- /dev/null +++ b/packages/workflow-qa/cleanup_state.lua @@ -0,0 +1,79 @@ +local C = {} + +local function verify_retention_snapshots(state, receipt, ports, deps) + local snapshots = {} + local ledger_ids = {} + for _, resource in ipairs(receipt.remaining_resources) do + if resource.resource_kind == "worker-home-ledger" then + local detail_ref = resource.resource_detail_ref.ref + local actual_sha256 = deps.digest(ports, detail_ref) + if actual_sha256 ~= resource.resource_detail_sha256 then + error("workflow-qa: cleanup-unverified: worker-home retention digest differs") + end + local snapshot = deps.load_bound(ports, detail_ref, actual_sha256, + "worker-home-retention") + deps.environment_contract.validate_worker_home_retention(snapshot) + if snapshot.operation_id ~= state.request.run_id + or not deps.environment_contract.same_repository(snapshot.repository, + state.request.repository) + or snapshot.remaining_count ~= resource.remaining_count then + error("workflow-qa: cleanup-unverified: worker-home retention binding differs") + end + if ledger_ids[snapshot.ledger_id] then + error("workflow-qa: cleanup-unverified: duplicate worker-home retention ledger") + end + ledger_ids[snapshot.ledger_id] = true + state.digests[detail_ref] = actual_sha256 + table.insert(snapshots, { + resource_id = resource.resource_id, + ledger_id = snapshot.ledger_id, + resource_detail_ref = deps.copy(resource.resource_detail_ref), + resource_detail_sha256 = actual_sha256, + remaining_count = snapshot.remaining_count, + }) + end + end + return snapshots +end + +function C.accept(state, payload, ports, deps) + if payload.cleanup_receipt_ref == nil then + error("workflow-qa: cleanup-unverified: cleanup receipt is missing") + end + local ref = payload.cleanup_receipt_ref.ref + local sha256 = deps.digest(ports, ref) + local receipt = deps.load_bound(ports, ref, sha256, "cleanup-receipt") + deps.environment_contract.validate_cleanup_receipt(receipt) + if receipt.status ~= payload.cleanup_status + or receipt.operation_id ~= state.request.run_id + or receipt.artifact_root ~= state.request.environment_start.artifact_root + or receipt.trace_id ~= state.request.trace_id + or receipt.dedup_key ~= state.request.dedup_key then + error("workflow-qa: cleanup-unverified: cleanup receipt binding differs") + end + + state.cleanup_result = deps.copy(payload) + state.digests[ref] = sha256 + if payload.cleanup_status == "incomplete" then + local retention_snapshots = verify_retention_snapshots(state, receipt, ports, deps) + state.terminal_status = "blocked" + state.cleanup_blocked = { + status = "blocked", + reason = "cleanup-incomplete", + cleanup_receipt_ref = ref, + cleanup_receipt_sha256 = sha256, + remaining_resources = deps.copy(receipt.remaining_resources), + worker_home_retention = retention_snapshots, + } + state.phase = "cleanup-blocked" + state.pending_actions = {} + deps.save(ports, state) + return {} + end + if payload.cleanup_status ~= "complete" then + error("workflow-qa: cleanup-unverified: cleanup status is not terminal") + end + return deps.prepare_finalization(state, ports) +end + +return C diff --git a/packages/workflow-qa/core.lua b/packages/workflow-qa/core.lua index 47d027e5..e6d3ba17 100644 --- a/packages/workflow-qa/core.lua +++ b/packages/workflow-qa/core.lua @@ -3,6 +3,7 @@ local checkpoints = require("checkpoints") local browser_contract = require("contract.browser_control") local browser_readiness_contract = require("contract.browser_readiness") local environment_contract = require("contract.environment_factory") +local cleanup_state = require("cleanup_state") local design_contract = require("contract.testing_design") local project_profile = require("contract.project_profile") local execution_contract = require("contract.structured_execution") @@ -288,7 +289,7 @@ function M.start(request, supplied_ports) }, ports) end -local cleanup_action, begin_cleanup, prepare_finalization +local cleanup_action, begin_cleanup, prepare_finalization, accept_cleanup_result function M.handle_environment_result(payload, request, supplied_ports) local ports = ports_module.resolve(supplied_ports) @@ -333,14 +334,9 @@ function M.handle_environment_result(payload, request, supplied_ports) }, ports) end - if payload.cleanup_status ~= "complete" or payload.cleanup_receipt_ref == nil then - error("workflow-qa: cleanup-unverified: blocked environment cleanup is incomplete") - end - state.cleanup_result = copy(payload) - state.digests[payload.cleanup_receipt_ref.ref] = digest(ports, payload.cleanup_receipt_ref.ref) state.counts = { planned = 0, executed = 0, passed = 0, failed = 0, skipped = 0, error = 0, blocked = 1 } state.terminal_status = state.interruption_requested or payload.status - return prepare_finalization(state, ports) + return accept_cleanup_result(state, payload, ports) end function M.handle_analysis_result(payload, request, supplied_ports) @@ -840,6 +836,14 @@ prepare_finalization = function(state, ports) }, ports) end +accept_cleanup_result = function(state, payload, ports) + return cleanup_state.accept(state, payload, ports, { + copy = copy, digest = digest, load_bound = load_bound, save = save, + environment_contract = environment_contract, + prepare_finalization = prepare_finalization, + }) +end + function M.handle_cleanup_result(payload, request, supplied_ports) local ports = ports_module.resolve(supplied_ports) request = resolve_request(payload, request, ports) @@ -853,12 +857,7 @@ function M.handle_cleanup_result(payload, request, supplied_ports) error("workflow-qa: foreign-cleanup-result: operation or source identity differs") end if state.phase ~= "cleanup-pending" then return copy(state.pending_actions or {}) end - if payload.cleanup_status ~= "complete" or payload.cleanup_receipt_ref == nil then - error("workflow-qa: cleanup-unverified: terminal cleanup receipt is incomplete") - end - state.cleanup_result = copy(payload) - state.digests[payload.cleanup_receipt_ref.ref] = digest(ports, payload.cleanup_receipt_ref.ref) - return prepare_finalization(state, ports) + return accept_cleanup_result(state, payload, ports) end function M.handle_environment_event(payload, supplied_ports) @@ -876,7 +875,8 @@ function M.handle_interrupt(payload, supplied_ports) local request = resolve_request(payload, nil, ports) local state = load_for(request, ports) if state == nil then error("workflow-qa: interruption-unavailable: durable run is missing") end - if state.phase == "cleanup-pending" or state.phase == "publication-pending" or state.phase == "terminal" then + if state.phase == "cleanup-pending" or state.phase == "cleanup-blocked" + or state.phase == "publication-pending" or state.phase == "terminal" then return copy(state.pending_actions or {}) end state.interruption_requested = payload.interruption diff --git a/packages/workflow-qa/tests/cleanup_state_test.lua b/packages/workflow-qa/tests/cleanup_state_test.lua new file mode 100644 index 00000000..ee8dd30c --- /dev/null +++ b/packages/workflow-qa/tests/cleanup_state_test.lua @@ -0,0 +1,53 @@ +local cleanup_state = require("cleanup_state") +local t = fkst.test + +local function state() + return { + request = { + run_id = "cleanup-state-run", + repository = { + slug = "owner/repo", + url = "https://github.com/owner/repo.git", + commit_sha = string.rep("1", 40), + }, + environment_start = { artifact_root = ".testing/runs/cleanup-state-run/environment" }, + trace_id = "trace-cleanup-state", + dedup_key = "dedup-cleanup-state", + }, + digests = {}, + } +end + +local deps = { + copy = function(value) return value end, + digest = function() return string.rep("a", 64) end, + load_bound = function() + return { + status = "pending", + operation_id = "cleanup-state-run", + artifact_root = ".testing/runs/cleanup-state-run/environment", + trace_id = "trace-cleanup-state", + dedup_key = "dedup-cleanup-state", + } + end, + environment_contract = { validate_cleanup_receipt = function() return true end }, + prepare_finalization = function() error("must not finalize") end, + save = function() return true end, +} + +return { + test_missing_cleanup_receipt_fails_closed = function() + t.raises(function() + cleanup_state.accept(state(), { cleanup_receipt_ref = nil }, {}, deps) + end) + end, + + test_nonterminal_cleanup_status_fails_closed = function() + t.raises(function() + cleanup_state.accept(state(), { + cleanup_receipt_ref = { kind = "artifact", ref = ".testing/runs/cleanup-state-run/cleanup.json" }, + cleanup_status = "pending", + }, {}, deps) + end) + end, +} diff --git a/packages/workflow-qa/tests/core_test.lua b/packages/workflow-qa/tests/core_test.lua index ece11623..99e19452 100644 --- a/packages/workflow-qa/tests/core_test.lua +++ b/packages/workflow-qa/tests/core_test.lua @@ -17,6 +17,7 @@ local helpers = require("tests.workflow_core_test_helpers").build({ local analysis_result = helpers.analysis_result local artifact_summary = helpers.artifact_summary local checkpoint_receipt = helpers.checkpoint_receipt +local cleanup_incomplete = helpers.cleanup_incomplete local copy = helpers.copy local digest = helpers.digest local execution_result = helpers.execution_result @@ -897,6 +898,7 @@ local workflow_core_coverage_cases = require("tests.workflow_core_coverage_helpe contract = contract, core = core, digest = digest, drive_to_grant = drive_to_grant, execution_result = execution_result, expect_failure = expect_failure, finalized = finalized, fixture = fixture, grant_result = grant_result, module_terminal = module_terminal, plan_result = plan_result, pointer = pointer, + cleanup_incomplete = cleanup_incomplete, ready_result = ready_result, release_checkpoint = release_checkpoint, runtime = runtime, t = t, }) @@ -905,4 +907,14 @@ tests.test_blocked_environment_summary_cleanup_and_publication_fail_closed = tests.test_remaining_workflow_identity_grant_and_publication_boundaries = workflow_core_coverage_cases.remaining_workflow_identity_grant_and_publication_boundaries +local workflow_cleanup_retention_cases = require("tests.workflow_cleanup_retention_test_helpers").build({ + cleanup_incomplete = cleanup_incomplete, core = core, digest = digest, + expect_failure = expect_failure, fixture = fixture, ready_result = ready_result, + release_checkpoint = release_checkpoint, runtime = runtime, t = t, +}) +tests.test_incomplete_cleanup_becomes_durable_non_publishable_blocked_state = + workflow_cleanup_retention_cases.incomplete_cleanup_becomes_durable_non_publishable_blocked_state +tests.test_incomplete_cleanup_receipt_binding_and_status_fail_closed = + workflow_cleanup_retention_cases.incomplete_cleanup_receipt_binding_and_status_fail_closed + return tests diff --git a/packages/workflow-qa/tests/workflow_cleanup_retention_test_helpers.lua b/packages/workflow-qa/tests/workflow_cleanup_retention_test_helpers.lua new file mode 100644 index 00000000..76e17419 --- /dev/null +++ b/packages/workflow-qa/tests/workflow_cleanup_retention_test_helpers.lua @@ -0,0 +1,86 @@ +local M = {} + +function M.build(deps) + local cleanup_incomplete = deps.cleanup_incomplete + local core = deps.core + local digest = deps.digest + local expect_failure = deps.expect_failure + local fixture = deps.fixture + local ready_result = deps.ready_result + local release_checkpoint = deps.release_checkpoint + local runtime = deps.runtime + local t = deps.t + local cases = {} + + local function setup() + local request = fixture() + local ports, state, put, artifacts = runtime(request) + core.start(request, ports) + release_checkpoint(request, ports, state, "environment-factory.environment_start") + core.handle_environment_result(ready_result(request, put), request, ports) + state().phase = "cleanup-pending" + return request, ports, state, put, artifacts + end + + cases.incomplete_cleanup_becomes_durable_non_publishable_blocked_state = function() + local request, ports, state, put = setup() + state().pending_actions = { { queue = "environment-factory.environment_finalize", payload = {} } } + local blocked = cleanup_incomplete(request, put) + local actions = core.handle_cleanup_result(blocked, request, ports) + t.eq(#actions, 0) + t.eq(state().phase, "cleanup-blocked") + t.eq(state().terminal_status, "blocked") + t.eq(#state().pending_actions, 0) + t.eq(state().cleanup_blocked.reason, "cleanup-incomplete") + t.eq(state().cleanup_blocked.cleanup_receipt_sha256, digest("f")) + t.eq(#state().cleanup_blocked.remaining_resources, 1) + t.eq(#state().cleanup_blocked.worker_home_retention, 1) + t.eq(state().cleanup_blocked.worker_home_retention[1].ledger_id, digest("8")) + t.eq(state().cleanup_blocked.worker_home_retention[1].resource_detail_sha256, digest("1")) + t.eq(state().digests[request.environment_start.artifact_root + .. "/worker-home-retention.json"], digest("1")) + t.eq(state().finalization_request, nil) + t.eq(state().artifacts.terminal_summary_ref, nil) + t.eq(#core.redrive({ run_id = request.run_id }, ports), 0) + t.eq(#core.handle_cleanup_result(blocked, request, ports), 0) + end + + cases.incomplete_cleanup_receipt_binding_and_status_fail_closed = function() + local mutations = { + function(blocked) blocked.cleanup_receipt_ref = nil end, + function(blocked, artifacts) artifacts[blocked.cleanup_receipt_ref.ref].value.status = "complete" end, + function(blocked, artifacts) artifacts[blocked.cleanup_receipt_ref.ref].value.operation_id = "foreign" end, + function(blocked, artifacts) + local receipt = artifacts[blocked.cleanup_receipt_ref.ref].value + artifacts[receipt.remaining_resources[1].resource_detail_ref.ref].digest = digest("2") + end, + function(blocked, artifacts) + local receipt = artifacts[blocked.cleanup_receipt_ref.ref].value + artifacts[receipt.remaining_resources[1].resource_detail_ref.ref].value.operation_id = "foreign" + end, + function(blocked, artifacts) + local receipt = artifacts[blocked.cleanup_receipt_ref.ref].value + artifacts[receipt.remaining_resources[1].resource_detail_ref.ref].value.repository.commit_sha = + string.rep("b", 40) + end, + function(blocked, artifacts) + artifacts[blocked.cleanup_receipt_ref.ref].value.remaining_resources[1].remaining_count = 2 + end, + } + local expected = { + "contract.environment-factory", "cleanup-status-mismatch", "cleanup receipt binding differs", + "worker-home retention digest differs", "worker-home retention binding differs", + "worker-home retention binding differs", "worker-home retention binding differs", + } + for index, mutate in ipairs(mutations) do + local request, ports, _, put, artifacts = setup() + local blocked = cleanup_incomplete(request, put) + mutate(blocked, artifacts) + expect_failure(expected[index], function() core.handle_cleanup_result(blocked, request, ports) end) + end + end + + return cases +end + +return M diff --git a/packages/workflow-qa/tests/workflow_core_coverage_helpers.lua b/packages/workflow-qa/tests/workflow_core_coverage_helpers.lua index ef157985..57f9d1c0 100644 --- a/packages/workflow-qa/tests/workflow_core_coverage_helpers.lua +++ b/packages/workflow-qa/tests/workflow_core_coverage_helpers.lua @@ -3,6 +3,7 @@ local M = {} function M.build(deps) local artifact_summary = deps.artifact_summary local checkpoint_receipt = deps.checkpoint_receipt + local cleanup_incomplete = deps.cleanup_incomplete local checkpoints = deps.checkpoints local contract = deps.contract local core = deps.core @@ -96,10 +97,9 @@ function M.build(deps) local cleanup = finalized(request, put) cleanup.operation_id = "foreign" expect_failure("foreign-cleanup-result", function() core.handle_cleanup_result(cleanup, request, ports) end) - cleanup = finalized(request, put) - cleanup.cleanup_status = "incomplete" - cleanup.cleanup_receipt_ref = pointer(request.environment_start.artifact_root .. "/cleanup-receipt-incomplete.json") - expect_failure("cleanup-unverified", function() core.handle_cleanup_result(cleanup, request, ports) end) + cleanup = cleanup_incomplete(request, put) + t.eq(#core.handle_cleanup_result(cleanup, request, ports), 0) + t.eq(state().phase, "cleanup-blocked") end end @@ -168,17 +168,10 @@ function M.build(deps) local ports, state, put = runtime(request) core.start(request, ports) release_checkpoint(request, ports, state, "environment-factory.environment_start") - local blocked = finalized(request, put) - blocked.status = "blocked" - blocked.failure_class = "provisioning-failed" - blocked.environment_receipt_ref = pointer( - request.environment_start.artifact_root .. "/environment-receipt-blocked.json") - blocked.cleanup_status = "incomplete" - blocked.cleanup_receipt_ref = pointer( - request.environment_start.artifact_root .. "/cleanup-receipt-incomplete.json") - expect_failure("cleanup-unverified", function() - core.handle_environment_result(blocked, request, ports) - end) + local blocked = cleanup_incomplete(request, put) + t.eq(#core.handle_environment_result(blocked, request, ports), 0) + t.eq(state().phase, "cleanup-blocked") + t.eq(state().terminal_status, "blocked") end do local request = fixture() diff --git a/packages/workflow-qa/tests/workflow_core_test_helpers.lua b/packages/workflow-qa/tests/workflow_core_test_helpers.lua index 516997af..b185f717 100644 --- a/packages/workflow-qa/tests/workflow_core_test_helpers.lua +++ b/packages/workflow-qa/tests/workflow_core_test_helpers.lua @@ -451,7 +451,19 @@ function M.build(deps) local function finalized(request, put) local cleanup_ref = request.environment_start.artifact_root .. "/cleanup-receipt-complete.json" - put(cleanup_ref, { schema = "environment-factory.cleanup-receipt.v1" }, digest("e")) + put(cleanup_ref, { + schema = "environment-factory.cleanup-receipt.v1", + operation_id = request.run_id, + status = "complete", + attempted_resources = { { + resource_id = "workspace", resource_kind = "workspace", status = "cleaned", + } }, + verified_removals = { "workspace" }, + remaining_resources = {}, + artifact_root = request.environment_start.artifact_root, + trace_id = request.trace_id, + dedup_key = request.dedup_key, + }, digest("e")) return { schema = "environment-factory.result.v1", operation_id = request.run_id, status = "finalized", environment_receipt_ref = pointer(request.environment_start.artifact_root .. "/environment-receipt-finalized.json"), @@ -463,6 +475,60 @@ function M.build(deps) } end + local function cleanup_incomplete(request, put) + local result = finalized(request, put) + local cleanup_ref = request.environment_start.artifact_root .. "/cleanup-receipt-incomplete.json" + local retention_ref = request.environment_start.artifact_root .. "/worker-home-retention.json" + put(retention_ref, { + schema = "environment-factory.worker-home-retention.v1", + operation_id = request.run_id, + ledger_id = digest("8"), + repository = { + url = request.repository.url, + commit_sha = request.repository.commit_sha, + }, + remaining_count = 1, + entries = { { + slot_id = digest("7"), + lease_id = string.rep("6", 32), + effect_id = "effect-checkout", + purpose = "checkout", + generation = 1, + identity_sha256 = digest("5"), + marker_sha256 = digest("4"), + state = "retained", + reason = "OBJECT_BOUND_CLEANUP_UNAVAILABLE", + } }, + }, digest("1")) + put(cleanup_ref, { + schema = "environment-factory.cleanup-receipt.v2", + operation_id = request.run_id, + status = "incomplete", + attempted_resources = { { + resource_id = "worker-homes", resource_kind = "worker-home-ledger", status = "remaining", + } }, + verified_removals = {}, + remaining_resources = { { + resource_id = "worker-homes", + resource_kind = "worker-home-ledger", + cleanup_ref = { kind = "resource-cleanup", ref = "worker-homes-cleanup" }, + resource_detail_ref = pointer(retention_ref), + resource_detail_sha256 = digest("1"), + remaining_count = 1, + } }, + artifact_root = request.environment_start.artifact_root, + trace_id = request.trace_id, + dedup_key = request.dedup_key, + }, digest("f")) + result.status = "blocked" + result.failure_class = "cleanup-incomplete" + result.cleanup_status = "incomplete" + result.cleanup_receipt_ref = pointer(cleanup_ref) + result.environment_receipt_ref = pointer( + request.environment_start.artifact_root .. "/environment-receipt-blocked.json") + return result + end + local function checkpoint_receipt(request, pending) return { schema = "test-publication.qa-publication-receipt.v2", @@ -487,6 +553,7 @@ function M.build(deps) analysis_result = analysis_result, artifact_summary = artifact_summary, checkpoint_receipt = checkpoint_receipt, + cleanup_incomplete = cleanup_incomplete, copy = copy, digest = digest, execution_result = execution_result, From 3a779536eea1b82747b8353ec6d1a0d76d153947 Mon Sep 17 00:00:00 2001 From: Shaw Zheng Date: Sat, 12 Sep 2026 17:32:40 +0800 Subject: [PATCH 09/11] fix(testing): bind durable workspace generations --- .../generic-host/bin/generic-host-runtime.js | 30 ++++++- .../generic-host/bin/worker-home-ledger.js | 55 ++++++++++--- .../tests/worker_home_ledger_test.js | 54 +++++++++++++ .../tests/workspace_checkout_recovery_test.js | 80 +++++++++++++++++++ .../bin/runtime/worker-home-resource.js | 56 ++++++++++--- .../bin/runtime/workspace-reservation.js | 24 ++++-- .../tests/node_runtime_test.js | 68 ++++++++++++++++ 7 files changed, 339 insertions(+), 28 deletions(-) diff --git a/examples/generic-host/bin/generic-host-runtime.js b/examples/generic-host/bin/generic-host-runtime.js index 5f1d0324..db383408 100755 --- a/examples/generic-host/bin/generic-host-runtime.js +++ b/examples/generic-host/bin/generic-host-runtime.js @@ -1127,12 +1127,34 @@ function recoverRegisteredCheckout(projectRoot, config, payload, workspaceRef, c if (!cleanupStored.written && !cleanupStored.replayed) { fail('registered checkout cleanup resource binding differs'); } + const resolvedCommit = verifyCheckoutGitState(config, payload, existing, 'checkout-recovery'); return { - status: 'passed', resolved_commit: config.commit_sha, + status: 'passed', resolved_commit: resolvedCommit, workspace_ref: workspaceRef, cleanup_ref: cleanupRef, }; } +function verifyCheckoutGitState(config, payload, workspace, purpose) { + const root = runRoot(config.run_id); + const workerRequest = ledgerWorkerRequest(config, payload); + return workerHomeLedger.withEnvironment( + root, workerRequest, purpose, config.command_environment || {}, (environment) => { + const resolved = directExec(['git', 'rev-parse', 'HEAD'], workspace.path, + payload.timeout_seconds, undefined, environment, workspace.path_identity); + const observed = String(resolved.stdout || '').trim(); + if (resolved.exit_code !== 0 || observed !== config.commit_sha) { + fail('workspace resolved commit differs from its durable binding'); + } + const status = directExec(['git', 'status', '--porcelain', '--untracked-files=no'], workspace.path, + payload.timeout_seconds, undefined, environment, workspace.path_identity); + if (status.exit_code !== 0 || String(status.stdout || '').trim() !== '') { + fail('workspace tracked working tree differs from its durable binding'); + } + return observed; + }, + ); +} + function startApplication(projectRoot, payload) { const runId = runIdFor(payload); const root = runRoot(runId); @@ -1147,6 +1169,7 @@ function startApplication(projectRoot, payload) { const workspace = workspaceResource(root, payload.workspace_ref); const workspaceState = verifyWorkspace(config, workspace); if (!workspaceState.owned) fail(`workspace ownership failed: ${workspaceState.reason}`); + verifyCheckoutGitState(config, payload, workspace, `application-start:${payload.effect_id}`); const purpose = `supervised:${payload.effect_id}`; const commandEnvironment = config.command_environment || {}; const workerRequest = ledgerWorkerRequest(config, payload); @@ -2275,6 +2298,11 @@ function dispatch(name, payload, projectRoot, hooks = {}) { reservation.record, reservation.record.path_identity, { reservationWasCreated: reservation.created, hooks: { + afterWorkspaceRecoveryReleased(details) { + if (typeof hooks.afterWorkspaceRecoveryReleased === 'function') { + hooks.afterWorkspaceRecoveryReleased(details); + } + }, afterWorkspaceDirectoryCreated(details) { if (typeof hooks.afterWorkspaceDirectoryCreated === 'function') { hooks.afterWorkspaceDirectoryCreated(details); diff --git a/examples/generic-host/bin/worker-home-ledger.js b/examples/generic-host/bin/worker-home-ledger.js index 4ff38e6b..0cfd92fb 100644 --- a/examples/generic-host/bin/worker-home-ledger.js +++ b/examples/generic-host/bin/worker-home-ledger.js @@ -140,12 +140,22 @@ function create(deps) { return ledger.entries.find((entry) => entry.slot_id === slotId) || null; } + function generationReservationId(baseReservationId, generation) { + if (generation === 1) return baseReservationId; + return sha256(stable({ + schema: 'environment-factory.worker-home-generation-reservation.v1', + base_reservation_id: baseReservationId, + generation, + })).slice(0, 32); + } + function allocate(root, request, purpose, extra = {}, reservationOverride = null, hooks = {}) { const expected = requireContext(root, request); const binding = slotBinding(request, purpose, extra); const slotId = sha256(stable(binding)); - const reservationId = reservationOverride === null ? slotId.slice(0, 32) : String(reservationOverride); - if (!/^[0-9a-f]{32}$/.test(reservationId)) fail('worker-home slot reservation is invalid'); + const baseReservationId = reservationOverride === null + ? slotId.slice(0, 32) : String(reservationOverride); + if (!/^[0-9a-f]{32}$/.test(baseReservationId)) fail('worker-home slot reservation is invalid'); const isolation = { schema: 'environment-factory.ledger-worker-isolation.v1', ledger_id: expected.ledgerId, @@ -155,20 +165,27 @@ function create(deps) { purpose, repository: expected.repository, }; - const reservation = workerEnvironmentReservation(extra, isolation, reservationId); + let reservation; + let reservationId; + let generation; updateLedger(root, expected, (ledger) => { let entry = findSlot(ledger, slotId); - if (entry && (stable(entry.binding) !== stable(binding) - || entry.reservation_id !== reservationId - || stable(entry.worker_environment_reservation) !== stable(reservation))) { + if (entry && stable(entry.binding) !== stable(binding)) { fail('worker-home slot binding differs'); } + if (entry && (!Number.isInteger(entry.generation) || entry.generation < 1 + || !['reserved', 'allocated', 'retained', 'released'].includes(entry.state))) { + fail('worker-home slot lifecycle differs'); + } if (!entry) { if (ledger.entries.length >= ledger.max_entries) fail('WORKER_HOME_LEDGER_CAPACITY_EXCEEDED'); + generation = 1; + reservationId = generationReservationId(baseReservationId, generation); + reservation = workerEnvironmentReservation(extra, isolation, reservationId); entry = { slot_id: slotId, reservation_id: reservationId, - generation: 1, + generation, binding, state: 'reserved', release_reason: null, @@ -177,11 +194,31 @@ function create(deps) { }; ledger.entries.push(entry); } else if (entry.state === 'released') { - entry.generation += 1; + if (reservationOverride !== null) { + fail('released worker-home slot cannot reuse a supervised reservation'); + } + generation = entry.generation + 1; + reservationId = generationReservationId(baseReservationId, generation); + reservation = workerEnvironmentReservation(extra, isolation, reservationId); + entry.generation = generation; + entry.reservation_id = reservationId; entry.state = 'reserved'; entry.release_reason = null; entry.worker_environment_reservation = reservation; entry.worker_environment_lease = null; + } else { + generation = entry.generation; + reservationId = generationReservationId(baseReservationId, generation); + reservation = workerEnvironmentReservation(extra, isolation, reservationId); + if (entry.reservation_id !== reservationId + || stable(entry.worker_environment_reservation) !== stable(reservation)) { + fail('worker-home slot reservation binding differs'); + } + if (entry.worker_environment_lease) { + verifyWorkerEnvironmentLease(entry.worker_environment_lease); + } else { + fail('worker-home allocation has no durable lease or release proof'); + } } }); @@ -194,7 +231,7 @@ function create(deps) { updateLedger(root, expected, (ledger) => { const entry = findSlot(ledger, slotId); if (!entry || stable(entry.binding) !== stable(binding) - || entry.reservation_id !== reservationId + || entry.generation !== generation || entry.reservation_id !== reservationId || stable(entry.worker_environment_reservation) !== stable(reservation)) { fail('worker-home slot reservation is unavailable'); } diff --git a/examples/generic-host/tests/worker_home_ledger_test.js b/examples/generic-host/tests/worker_home_ledger_test.js index 878bc680..baa3aeab 100644 --- a/examples/generic-host/tests/worker_home_ledger_test.js +++ b/examples/generic-host/tests/worker_home_ledger_test.js @@ -97,6 +97,9 @@ assert.throws(() => ledger.allocate(durable, { }, }), /simulated crash/); assert.equal(fs.existsSync(interruptedWorkerHome), true); +assert.throws(() => ledger.allocate(durable, { + ...base, effect_id: 'interrupted-allocation', worker_home_ledger_ref: initialized.cleanup_ref, +}, 'interrupted-allocation', {}), /worker-home allocation has no durable lease or release proof/); const externalWorkerHome = path.join(root, 'external-worker-home'); fs.mkdirSync(path.join(externalWorkerHome, '.config', 'gh'), { recursive: true, mode: 0o700 }); const externalCredential = path.join(externalWorkerHome, '.config', 'gh', 'hosts.yml'); @@ -129,6 +132,30 @@ assert.equal(current.entries.length, 4); assert.equal(current.entries.filter((entry) => entry.state === 'allocated').length, 2); assert.equal(current.entries.filter((entry) => entry.state === 'reserved').length, 2); +const generationRequest = { + ...base, effect_id: 'generation-retry', worker_home_ledger_ref: initialized.cleanup_ref, +}; +const generationOne = ledger.allocate(durable, generationRequest, 'generation-retry', {}); +assert.equal(ledger.recordRelease(durable, generationOne), true); +const generationTwo = ledger.allocate(durable, generationRequest, 'generation-retry', {}); +assert.notEqual(generationTwo.environment.HOME, generationOne.environment.HOME); +assert.notEqual(generationTwo.lease.cleanup_capture_id, generationOne.lease.cleanup_capture_id); +assert.equal(ledger.recordRelease(durable, generationTwo), true); +const supervisedRequest = { + ...base, effect_id: 'supervised-reservation', worker_home_ledger_ref: initialized.cleanup_ref, +}; +const supervisedReservationId = 'b'.repeat(32); +const supervised = ledger.allocate( + durable, supervisedRequest, 'supervised-reservation', {}, supervisedReservationId, +); +assert.equal(ledger.recordRelease(durable, supervised), true); +assert.throws( + () => ledger.allocate( + durable, supervisedRequest, 'supervised-reservation', {}, supervisedReservationId, + ), + /released worker-home slot cannot reuse a supervised reservation/, +); + const resource = read(resourceKey(initialized.cleanup_ref)); assert.equal(ledger.releaseProven( durable, checkoutRequest, checkout.slot_id, checkout.lease, @@ -167,6 +194,33 @@ assert.equal(fs.readFileSync(externalCredential, 'utf8'), 'external-credential-s assert.equal(ledger.releaseProven( durable, checkoutRequest, checkout.slot_id, checkout.lease, ), true); + +let missingReservedHome = null; +const missingReservationRequest = { + ...base, effect_id: 'missing-reservation', worker_home_ledger_ref: initialized.cleanup_ref, +}; +assert.throws(() => ledger.allocate( + durable, missingReservationRequest, 'missing-reservation', {}, null, { + afterEnvironmentCreated(environment) { + missingReservedHome = environment.HOME; + throw new Error('simulated crash before worker HOME lease persistence'); + }, + }, +), /simulated crash before worker HOME lease persistence/); +fs.rmSync(missingReservedHome, { recursive: true }); +const retainedCleanup = ledger.cleanup(durable, root, { + ...base, + artifact_root: '.testing/runs/worker-ledger-test/environment', + cleanup_ref: initialized.cleanup_ref, +}, resource); +assert.equal(retainedCleanup.cleaned, false); +assert.equal(read('environment-factory/worker-home-ledger').entries.find( + (entry) => entry.binding.effect_id === missingReservationRequest.effect_id, +).state, 'retained'); +assert.throws( + () => ledger.allocate(durable, missingReservationRequest, 'missing-reservation', {}), + /worker-home allocation has no durable lease or release proof/, +); const workspaceRoot = path.join(root, 'workspaces'); const workspace = path.join(workspaceRoot, 'run-workspace'); fs.mkdirSync(workspace, { recursive: true }); diff --git a/examples/generic-host/tests/workspace_checkout_recovery_test.js b/examples/generic-host/tests/workspace_checkout_recovery_test.js index 43fd6fcc..afe188cf 100644 --- a/examples/generic-host/tests/workspace_checkout_recovery_test.js +++ b/examples/generic-host/tests/workspace_checkout_recovery_test.js @@ -42,6 +42,9 @@ const repository = { url: 'https://example.invalid/testing/generic-host-workspace.git', commit_sha: git(['rev-parse', 'HEAD']), }; +fs.writeFileSync(path.join(sourceRoot, 'fixture.txt'), 'alternate fixture\n'); +git(['commit', '--quiet', '-am', 'alternate fixture']); +const alternateCommit = git(['rev-parse', 'HEAD']); const boundary = { schema: 'testing-host.target-execution-boundary.v1', mode: 'trusted-fixture-exact', @@ -69,6 +72,7 @@ function initializeRun(label) { workspace_root: workspaceRoot, source_root: sourceRoot, commit_sha: repository.commit_sha, + port: 43191, repository, profile: { repository, working_directory: '.' }, target_execution_boundary: boundary, @@ -118,6 +122,82 @@ try { assert.equal(git(['rev-parse', 'HEAD'], config.workspace_root), repository.commit_sha); assert.equal(git(['status', '--porcelain', '--untracked-files=no'], config.workspace_root), ''); } + + const missing = initializeRun('allocated-path-missing'); + assert.throws(() => dispatch('checkout', missing.payload, projectRoot, { + afterWorkspaceResourceRegistered() { + throw new Error('simulated interruption after workspace identity persistence'); + }, + }), /simulated interruption after workspace identity persistence/); + fs.rmSync(missing.config.workspace_root, { recursive: true }); + assert.throws( + () => dispatch('checkout', missing.payload, projectRoot), + /workspace reserved object is missing without release proof/, + ); + + const released = initializeRun('released-before-reallocation'); + assert.throws(() => dispatch('checkout', released.payload, projectRoot, { + afterWorkspaceResourceRegistered() { + throw new Error('simulated interruption after workspace identity persistence'); + }, + }), /simulated interruption after workspace identity persistence/); + assert.throws(() => dispatch('checkout', released.payload, projectRoot, { + afterWorkspaceRecoveryReleased() { + throw new Error('simulated interruption after workspace release proof'); + }, + }), /simulated interruption after workspace release proof/); + assert.equal(fs.existsSync(released.config.workspace_root), false); + const releasedRecovery = dispatch('checkout', released.payload, projectRoot); + assert.equal(releasedRecovery.status, 'passed'); + assert.equal(releasedRecovery.resolved_commit, repository.commit_sha); + + const changedHead = initializeRun('registered-head-changed'); + assert.throws(() => dispatch('checkout', changedHead.payload, projectRoot, { + afterFinalWorkspaceResourceRegistered() { + throw new Error('simulated interruption after final workspace registration'); + }, + }), /simulated interruption after final workspace registration/); + git(['checkout', '--quiet', alternateCommit], changedHead.config.workspace_root); + assert.throws( + () => dispatch('checkout', changedHead.payload, projectRoot), + /workspace resolved commit differs from its durable binding/, + ); + + const changedContent = initializeRun('registered-content-changed'); + assert.throws(() => dispatch('checkout', changedContent.payload, projectRoot, { + afterFinalWorkspaceResourceRegistered() { + throw new Error('simulated interruption after final workspace registration'); + }, + }), /simulated interruption after final workspace registration/); + fs.appendFileSync(path.join(changedContent.config.workspace_root, 'fixture.txt'), 'tampered\n'); + assert.throws( + () => dispatch('checkout', changedContent.payload, projectRoot), + /workspace tracked working tree differs from its durable binding/, + ); + + const application = initializeRun('application-content-changed'); + const applicationCheckout = dispatch('checkout', application.payload, projectRoot); + fs.appendFileSync(path.join(application.config.workspace_root, 'fixture.txt'), 'tampered\n'); + assert.throws(() => dispatch('fixture-start-application', { + ...application.payload, + effect_id: `${application.config.run_id}/application`, + workspace_ref: applicationCheckout.workspace_ref, + cleanup_ref: { kind: 'process-cleanup', ref: `${application.config.run_id}-application` }, + argv: [process.execPath, '-e', 'setInterval(() => {}, 1000)'], + runtime_ports: [{ name: 'application', port: application.config.port }], + }, projectRoot), /workspace tracked working tree differs from its durable binding/); + + const applicationHead = initializeRun('application-head-changed'); + const applicationHeadCheckout = dispatch('checkout', applicationHead.payload, projectRoot); + git(['checkout', '--quiet', alternateCommit], applicationHead.config.workspace_root); + assert.throws(() => dispatch('fixture-start-application', { + ...applicationHead.payload, + effect_id: `${applicationHead.config.run_id}/application`, + workspace_ref: applicationHeadCheckout.workspace_ref, + cleanup_ref: { kind: 'process-cleanup', ref: `${applicationHead.config.run_id}-application` }, + argv: [process.execPath, '-e', 'setInterval(() => {}, 1000)'], + runtime_ports: [{ name: 'application', port: applicationHead.config.port }], + }, projectRoot), /workspace resolved commit differs from its durable binding/); } finally { fs.rmSync(root, { recursive: true, force: true }); } diff --git a/packages/environment-factory/bin/runtime/worker-home-resource.js b/packages/environment-factory/bin/runtime/worker-home-resource.js index 0b1e0479..45e43d9d 100644 --- a/packages/environment-factory/bin/runtime/worker-home-resource.js +++ b/packages/environment-factory/bin/runtime/worker-home-resource.js @@ -164,14 +164,24 @@ function findSlot(ledger, slotId) { return ledger.entries.find((entry) => entry.slot_id === slotId) || null; } +function generationReservationId(baseReservationId, generation) { + if (generation === 1) return baseReservationId; + return sha256(stableStringify({ + schema: 'environment-factory.worker-home-generation-reservation.v1', + base_reservation_id: baseReservationId, + generation, + })).slice(0, 32); +} + function allocateDurableWorkerEnvironment( request, purpose, extra = {}, reservationOverride = null, hooks = {}, ) { const identity = requireLedgerContext(request); const binding = slotBinding(request, purpose, extra); const slotId = sha256(stableStringify(binding)); - const reservationId = reservationOverride === null ? slotId.slice(0, 32) : String(reservationOverride); - if (!/^[0-9a-f]{32}$/.test(reservationId)) { + const baseReservationId = reservationOverride === null + ? slotId.slice(0, 32) : String(reservationOverride); + if (!/^[0-9a-f]{32}$/.test(baseReservationId)) { throw new Error('worker-home slot reservation is invalid'); } const isolation = { @@ -183,7 +193,9 @@ function allocateDurableWorkerEnvironment( purpose, repository: identity.repository, }; - const reservation = workerEnvironmentReservation(extra, isolation, reservationId); + let reservation; + let reservationId; + let generation; let release = acquireLock(`${identity.ledgerPath}.lock`); try { const ledger = verifyLedger(readJson(identity.ledgerPath), identity); @@ -191,12 +203,9 @@ function allocateDurableWorkerEnvironment( if (existing && stableStringify(existing.binding) !== stableStringify(binding)) { throw new Error('worker-home slot binding differs'); } - if (existing && existing.reservation_id !== reservationId) { - throw new Error('worker-home slot reservation differs'); - } - if (existing && (!existing.worker_environment_reservation - || stableStringify(existing.worker_environment_reservation) !== stableStringify(reservation))) { - throw new Error('worker-home slot reservation binding differs'); + if (existing && (!Number.isInteger(existing.generation) || existing.generation < 1 + || !['reserved', 'allocated', 'retained', 'released'].includes(existing.state))) { + throw new Error('worker-home slot lifecycle differs'); } if (!existing) { if (ledger.entries.length >= ledger.max_entries) { @@ -204,10 +213,13 @@ function allocateDurableWorkerEnvironment( error.code = 'WORKER_HOME_LEDGER_CAPACITY_EXCEEDED'; throw error; } + generation = 1; + reservationId = generationReservationId(baseReservationId, generation); + reservation = workerEnvironmentReservation(extra, isolation, reservationId); ledger.entries.push({ slot_id: slotId, reservation_id: reservationId, - generation: 1, + generation, binding, state: 'reserved', release_reason: null, @@ -217,13 +229,34 @@ function allocateDurableWorkerEnvironment( ledger.revision += 1; writeLedger(identity.ledgerPath, ledger); } else if (existing.state === 'released') { - existing.generation += 1; + if (reservationOverride !== null) { + throw new Error('released worker-home slot cannot reuse a supervised reservation'); + } + generation = existing.generation + 1; + reservationId = generationReservationId(baseReservationId, generation); + reservation = workerEnvironmentReservation(extra, isolation, reservationId); + existing.generation = generation; + existing.reservation_id = reservationId; existing.state = 'reserved'; existing.release_reason = null; existing.worker_environment_reservation = reservation; existing.worker_environment_lease = null; ledger.revision += 1; writeLedger(identity.ledgerPath, ledger); + } else { + generation = existing.generation; + reservationId = generationReservationId(baseReservationId, generation); + reservation = workerEnvironmentReservation(extra, isolation, reservationId); + if (existing.reservation_id !== reservationId + || !existing.worker_environment_reservation + || stableStringify(existing.worker_environment_reservation) !== stableStringify(reservation)) { + throw new Error('worker-home slot reservation binding differs'); + } + if (existing.worker_environment_lease) { + verifyWorkerEnvironmentLease(existing.worker_environment_lease); + } else { + throw new Error('worker-home allocation has no durable lease or release proof'); + } } } finally { release(); @@ -240,6 +273,7 @@ function allocateDurableWorkerEnvironment( const ledger = verifyLedger(readJson(identity.ledgerPath), identity); const entry = findSlot(ledger, slotId); if (!entry || stableStringify(entry.binding) !== stableStringify(binding) + || entry.generation !== generation || entry.reservation_id !== reservationId || stableStringify(entry.worker_environment_reservation) !== stableStringify(reservation)) { throw new Error('worker-home slot reservation is unavailable'); } diff --git a/packages/environment-factory/bin/runtime/workspace-reservation.js b/packages/environment-factory/bin/runtime/workspace-reservation.js index 9c1de0a9..b797f0f5 100644 --- a/packages/environment-factory/bin/runtime/workspace-reservation.js +++ b/packages/environment-factory/bin/runtime/workspace-reservation.js @@ -3,6 +3,7 @@ const path = require('path'); const { allocateOwnedDirectory, + ownedDirectoryReleaseProven, pathEntryExists, pathIdentity, removeOwnedDirectory, @@ -55,21 +56,30 @@ function prepareReservedWorkspace(reservation, persistedIdentity, options = {}) verifyReservation(reservation); const hooks = options.hooks || {}; if (persistedIdentity) { - if (pathEntryExists(reservation.path)) { + const recoveryCaptureId = sha256(stableStringify({ + schema: 'environment-factory.workspace-recovery-cleanup.v1', + reservation_id: reservation.reservation_id, + path_identity: persistedIdentity, + cleanup_capture_id: reservation.cleanup_capture_id, + })); + if (!pathEntryExists(reservation.path)) { + if (!ownedDirectoryReleaseProven( + reservation.path, persistedIdentity, reservation.containment_root, recoveryCaptureId, + )) throw new Error('workspace reserved object is missing without release proof'); + } else { if (!samePathIdentity(pathIdentity(reservation.path), persistedIdentity)) { throw new Error('workspace reserved object identity changed'); } - const recoveryCaptureId = sha256(stableStringify({ - schema: 'environment-factory.workspace-recovery-cleanup.v1', - reservation_id: reservation.reservation_id, - path_identity: persistedIdentity, - cleanup_capture_id: reservation.cleanup_capture_id, - })); if (!removeOwnedDirectory( reservation.path, persistedIdentity, reservation.containment_root, recoveryCaptureId, )) throw new Error('workspace reserved object cleanup is unavailable'); } + if (typeof hooks.afterWorkspaceRecoveryReleased === 'function') { + hooks.afterWorkspaceRecoveryReleased({ + reservation: { ...reservation }, recovery_capture_id: recoveryCaptureId, + }); + } } else if (options.reservationWasCreated === true && pathEntryExists(reservation.path)) { throw new Error('workspace path already exists without recoverable reservation ownership'); } else if (options.reservationWasCreated !== true && !pathEntryExists(reservation.path)) { diff --git a/packages/environment-factory/tests/node_runtime_test.js b/packages/environment-factory/tests/node_runtime_test.js index 77802eff..05e02717 100644 --- a/packages/environment-factory/tests/node_runtime_test.js +++ b/packages/environment-factory/tests/node_runtime_test.js @@ -563,9 +563,41 @@ async function main() { }, }), /simulated crash/); assert.strictEqual(fs.existsSync(interruptedWorkerHome), true); + assert.throws(() => allocateDurableWorkerEnvironment({ + ...workerRequest, + effect_id: `node-operation-${process.pid}/interrupted-allocation`, + worker_home_ledger_ref: ledger.cleanup_ref, + }, 'interrupted-allocation'), /worker-home allocation has no durable lease or release proof/); assert.strictEqual(operationWorker.environment.HOME, operationWorkerReplay.environment.HOME); assert.strictEqual(operationWorker.slot_id, operationWorkerReplay.slot_id); assert.notStrictEqual(operationWorker.environment.HOME, readinessWorker.environment.HOME); + const generationRequest = { + ...workerRequest, + effect_id: `node-operation-${process.pid}/generation-retry`, + worker_home_ledger_ref: ledger.cleanup_ref, + }; + const generationOne = allocateDurableWorkerEnvironment(generationRequest, 'generation-retry'); + assert.strictEqual(recordWorkerEnvironmentRelease(generationOne), true); + const generationTwo = allocateDurableWorkerEnvironment(generationRequest, 'generation-retry'); + assert.notStrictEqual(generationTwo.environment.HOME, generationOne.environment.HOME); + assert.notStrictEqual(generationTwo.lease.cleanup_capture_id, generationOne.lease.cleanup_capture_id); + assert.strictEqual(recordWorkerEnvironmentRelease(generationTwo), true); + const supervisedRequest = { + ...workerRequest, + effect_id: `node-operation-${process.pid}/supervised-reservation`, + worker_home_ledger_ref: ledger.cleanup_ref, + }; + const supervisedReservationId = 'b'.repeat(32); + const supervised = allocateDurableWorkerEnvironment( + supervisedRequest, 'supervised-reservation', {}, supervisedReservationId, + ); + assert.strictEqual(recordWorkerEnvironmentRelease(supervised), true); + assert.throws( + () => allocateDurableWorkerEnvironment( + supervisedRequest, 'supervised-reservation', {}, supervisedReservationId, + ), + /released worker-home slot cannot reuse a supervised reservation/, + ); delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER; delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256; assert.strictEqual(recordWorkerEnvironmentRelease(operationWorker), false); @@ -670,6 +702,42 @@ async function main() { assert.strictEqual(fs.existsSync(displacedWorkerHome), true); assert.strictEqual(fs.readFileSync(externalCredential, 'utf8'), 'external-credential-sentinel\n'); + const missingReservationRequest = { + operation_id: `node-worker-home-missing-${process.pid}`, + repository: workerRequest.repository, + artifact_root: artifactRoot, + }; + const missingReservationLedger = initializeWorkerHomeLedger(missingReservationRequest); + let missingReservedHome = null; + const missingAllocation = { + ...missingReservationRequest, + effect_id: `${missingReservationRequest.operation_id}/checkout`, + worker_home_ledger_ref: missingReservationLedger.cleanup_ref, + }; + assert.throws(() => allocateDurableWorkerEnvironment( + missingAllocation, 'checkout', {}, null, { + afterEnvironmentCreated(environment) { + missingReservedHome = environment.HOME; + throw new Error('simulated crash before worker HOME lease persistence'); + }, + }, + ), /simulated crash before worker HOME lease persistence/); + fs.rmSync(missingReservedHome, { recursive: true }); + const missingReservationCleanup = await dispatch('cleanup', { + effect_id: `${missingReservationRequest.operation_id}/cleanup/worker-homes`, + operation_id: missingReservationRequest.operation_id, + artifact_root: artifactRoot, + cleanup_ref: missingReservationLedger.cleanup_ref, + worker_home_ledger_ref: missingReservationLedger.cleanup_ref, + timeout_seconds: 1, + }); + assert.strictEqual(missingReservationCleanup.status, 'blocked'); + assert.strictEqual(missingReservationCleanup.remaining_count, 1); + assert.throws( + () => allocateDurableWorkerEnvironment(missingAllocation, 'checkout'), + /worker-home allocation has no durable lease or release proof/, + ); + const retainedProcessResource = { kind: 'process', pid: 2147483647, pgid: 2147483647, process_start_identity: 'not-running', worker_environment_lease: operationWorker.lease, From 96488d6c6b582cf9e9f57e1a6d7f86c283eb4e57 Mon Sep 17 00:00:00 2001 From: Shaw Zheng Date: Tue, 15 Sep 2026 17:56:01 +0800 Subject: [PATCH 10/11] fix(testing): make durable recovery replayable --- .../tests/worker_home_ledger_test.js | 34 ++++++++ .../bin/fkst-structured-execution-runtime.js | 43 ++++------ .../structured_execution_runtime_test.js | 81 +++++++++++++++++-- .../bin/object-bound-cleanup-broker.py | 60 ++++++++++++-- .../environment-factory/bin/runtime/common.js | 48 ++++++++--- .../tests/node_runtime_test.js | 11 +++ .../object_bound_cleanup_backend_test.py | 58 +++++++++++++ .../testing-runner/structured_execution.lua | 1 + .../tests/structured_execution_test.lua | 6 +- 9 files changed, 290 insertions(+), 52 deletions(-) diff --git a/examples/generic-host/tests/worker_home_ledger_test.js b/examples/generic-host/tests/worker_home_ledger_test.js index baa3aeab..4490d990 100644 --- a/examples/generic-host/tests/worker_home_ledger_test.js +++ b/examples/generic-host/tests/worker_home_ledger_test.js @@ -25,6 +25,7 @@ process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER_SHA256 = common.sha256(fs.readFileS const stable = store.stable; const sha256 = (value) => crypto.createHash('sha256').update(String(value)).digest('hex'); const records = new Map(); +let failReleasedCasForEffect = null; const copy = (value) => value == null ? value : JSON.parse(stable(value)); const read = (key) => copy(records.get(key) || null); const cas = (key, value, version) => { @@ -33,6 +34,11 @@ const cas = (key, value, version) => { if (currentVersion !== version) { return { saved: false, stale: true, version: currentVersion, value: copy(current) }; } + if (failReleasedCasForEffect && value.entries.some((entry) => + entry.binding.effect_id === failReleasedCasForEffect && entry.state === 'released')) { + failReleasedCasForEffect = null; + throw new Error('simulated crash after worker HOME release before ledger CAS'); + } records.set(key, copy(value)); return { saved: true, stale: false, version: value.version, value: copy(value) }; }; @@ -221,6 +227,34 @@ assert.throws( () => ledger.allocate(durable, missingReservationRequest, 'missing-reservation', {}), /worker-home allocation has no durable lease or release proof/, ); + +let replayReleasedHome = null; +const replayReleaseRequest = { + ...base, effect_id: 'replay-reservation-release', worker_home_ledger_ref: initialized.cleanup_ref, +}; +assert.throws(() => ledger.allocate( + durable, replayReleaseRequest, 'replay-reservation-release', {}, null, { + afterEnvironmentCreated(environment) { + replayReleasedHome = environment.HOME; + throw new Error('simulated crash before replayable worker HOME lease persistence'); + }, + }, +), /simulated crash before replayable worker HOME lease persistence/); +failReleasedCasForEffect = replayReleaseRequest.effect_id; +assert.throws(() => ledger.cleanup(durable, root, { + ...base, + artifact_root: '.testing/runs/worker-ledger-test/environment', + cleanup_ref: initialized.cleanup_ref, +}, resource), /simulated crash after worker HOME release before ledger CAS/); +assert.equal(fs.existsSync(replayReleasedHome), false); +ledger.cleanup(durable, root, { + ...base, + artifact_root: '.testing/runs/worker-ledger-test/environment', + cleanup_ref: initialized.cleanup_ref, +}, resource); +assert.equal(read('environment-factory/worker-home-ledger').entries.find( + (entry) => entry.binding.effect_id === replayReleaseRequest.effect_id, +).state, 'released'); const workspaceRoot = path.join(root, 'workspaces'); const workspace = path.join(workspaceRoot, 'run-workspace'); fs.mkdirSync(workspace, { recursive: true }); diff --git a/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js b/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js index 44c2db30..0f72ea1a 100644 --- a/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js +++ b/libraries/testing_runtime/bin/fkst-structured-execution-runtime.js @@ -156,33 +156,21 @@ function writeReplay(config, grantId, value) { } function completedReplayForResult(config, payload) { - const directory = path.join(durableRoot(), 'testing-runner', 'structured-execution'); - if (!fs.existsSync(directory)) throw new Error('authenticated completed replay claim is unavailable'); - const matches = []; - for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { - if (!entry.isFile() || !/^[0-9a-f]{64}\.json$/.test(entry.name)) continue; - const envelope = readJson(path.join(directory, entry.name)); - if (!envelope || envelope.schema !== 'testing-runtime.structured-execution-replay.v1' - || envelope.mac !== replayMac(config, envelope.value)) { - throw new Error('structured execution replay state authentication failed'); - } - const value = envelope.value; - const binding = value && value.binding; - if (!binding || entry.name !== `${sha256(binding.grant_id)}.json`) { - throw new Error('structured execution replay state identity differs'); - } - if (value.status === 'completed' && value.result_ref === payload.result_ref - && value.result_sha256 === payload.result_sha256 - && binding.artifact_root === payload.artifact_root - && binding.operation_id === payload.operation_id - && binding.environment_receipt_sha256 === payload.environment_receipt_sha256 - && sameRepository(binding.repository, payload.repository) - && binding.trace_id === payload.trace_id && binding.dedup_key === payload.dedup_key) { - matches.push(value); - } + const value = readReplay(config, payload.grant_id); + const binding = value && value.binding; + if (!binding || binding.grant_id !== payload.grant_id) { + throw new Error('structured execution replay state identity differs'); } - if (matches.length !== 1) throw new Error('authenticated completed replay claim is unavailable or ambiguous'); - return matches[0]; + if (value.status !== 'completed' || value.result_ref !== payload.result_ref + || value.result_sha256 !== payload.result_sha256 + || binding.artifact_root !== payload.artifact_root + || binding.operation_id !== payload.operation_id + || binding.environment_receipt_sha256 !== payload.environment_receipt_sha256 + || !sameRepository(binding.repository, payload.repository) + || binding.trace_id !== payload.trace_id || binding.dedup_key !== payload.dedup_key) { + throw new Error('authenticated completed replay claim is unavailable'); + } + return value; } function sameRepository(left, right) { @@ -1067,6 +1055,9 @@ function httpRequest(payload) { } function loadResult(payload) { + if (!validString(payload.grant_id, 180)) { + throw new Error('completed execution grant identity is required'); + } if (!/^[0-9a-f]{64}$/.test(String(payload.result_sha256 || ''))) { throw new Error('completed execution result digest is required'); } diff --git a/libraries/testing_runtime/tests/structured_execution_runtime_test.js b/libraries/testing_runtime/tests/structured_execution_runtime_test.js index b7655ddf..dd74f916 100644 --- a/libraries/testing_runtime/tests/structured_execution_runtime_test.js +++ b/libraries/testing_runtime/tests/structured_execution_runtime_test.js @@ -73,6 +73,7 @@ async function main() { const artifactRoot = `.testing/runs/${runId}/execution`; const environmentArtifactRoot = `.testing/runs/${runId}/environment`; const configRef = `.testing/host/structured-execution/${runId}.json`; + const foreignConfigRef = `.testing/host/structured-execution/${runId}-foreign.json`; const environmentConfigRef = `.testing/host/environment-factory/${runId}.json`; const linkPath = `.testing/${runId}-link`; const source = path.join(temp, 'source'); @@ -571,8 +572,70 @@ async function main() { assert.strictEqual(historicalCompletion.completed, true); const historicalReplay = await dispatch('replay-guard', historicalClaimRequest); assert.strictEqual(historicalReplay.status, 'completed'); + + const foreignGrantId = `${runId}-foreign-config-grant`; + const foreignArtifactRoot = `${artifactRoot}/foreign`; + const foreignOperationId = `${operationId}-foreign`; + const foreignTraceId = `${traceId}-foreign`; + const foreignDedupKey = `${dedupKey}-foreign`; + const foreignResultRef = `${foreignArtifactRoot}/execution.json`; + const foreignExecution = { + ...historicalExecution, + operation_id: foreignOperationId, + trace_id: foreignTraceId, + dedup_key: foreignDedupKey, + test_plan_path: `${foreignArtifactRoot}/test-plan.json`, + case_results_path: `${foreignArtifactRoot}/case-results.json`, + execution_path: foreignResultRef, + }; + const foreignExecutionArtifact = persistJson(foreignResultRef, foreignExecution); + const foreignReplayValue = { + status: 'completed', + claim_id: 'foreign-config-claim', + binding: { + grant_id: foreignGrantId, + grant_sha256: '1'.repeat(64), + parent_authorization_sha256: '2'.repeat(64), + plan_sha256: historicalExecution.plan_sha256, + environment_receipt_sha256: common.environment_receipt_sha256, + repository, + operation_id: foreignOperationId, + artifact_root: foreignArtifactRoot, + trace_id: foreignTraceId, + dedup_key: foreignDedupKey, + }, + result_ref: foreignResultRef, + result_sha256: foreignExecutionArtifact.digest, + }; + const foreignReplayEnvelope = { + schema: 'testing-runtime.structured-execution-replay.v1', + value: foreignReplayValue, + mac: crypto.createHmac('sha256', 'foreign-runtime-state-key-000000000000000000000') + .update(`foreign-runtime-v1\0${stableStringify(foreignReplayValue)}`).digest('hex'), + }; + const foreignReplayPath = path.join(process.env.FKST_DURABLE_ROOT, 'testing-runner', + 'structured-execution', `${sha256(foreignGrantId)}.json`); + fs.writeFileSync(foreignReplayPath, `${stableStringify(foreignReplayEnvelope)}\n`, { flag: 'wx' }); + const foreignConfig = JSON.parse(fs.readFileSync(configRef, 'utf8')); + foreignConfig.state_auth_key = 'foreign-runtime-state-key-000000000000000000000'; + foreignConfig.state_mac_generation = 'foreign-runtime-v1'; + fs.writeFileSync(foreignConfigRef, `${stableStringify(foreignConfig)}\n`, { flag: 'wx' }); + const foreignSummary = await dispatch('load-result', { + artifact_root: foreignArtifactRoot, + grant_id: foreignGrantId, + result_ref: foreignResultRef, + result_sha256: foreignExecutionArtifact.digest, + operation_id: foreignOperationId, + repository, + environment_receipt_sha256: common.environment_receipt_sha256, + trace_id: foreignTraceId, + dedup_key: foreignDedupKey, + runtime_config_ref: { kind: 'artifact', ref: foreignConfigRef }, + }); + assert.strictEqual(foreignSummary.status, 'passed'); const historicalSummary = await dispatch('load-result', { - ...common, result_ref: resultRef, result_sha256: historicalReplay.result_sha256, + ...common, grant_id: historicalClaimRequest.grant_id, + result_ref: resultRef, result_sha256: historicalReplay.result_sha256, }); assert.strictEqual(historicalSummary.passed_count, 1); assert.strictEqual(Object.prototype.hasOwnProperty.call( @@ -799,7 +862,8 @@ async function main() { assert.strictEqual(replay.result_ref, resultRef); assert.strictEqual(replay.result_sha256, completion.result_sha256); const summary = await dispatch('load-result', { - ...common, result_ref: resultRef, result_sha256: replay.result_sha256, + ...common, grant_id: `${runId}-canonical-grant`, + result_ref: resultRef, result_sha256: replay.result_sha256, }); assert.strictEqual(summary.passed_count, 1); assert.strictEqual(summary.case_result_set_path, caseResultSetPath); @@ -824,7 +888,8 @@ async function main() { .update(`runtime-test-v1\0${stableStringify(replayEnvelope.value)}`).digest('hex'); fs.writeFileSync(replayPath, `${stableStringify(replayEnvelope)}\n`); await assert.rejects(() => dispatch('load-result', { - ...common, result_ref: resultRef, result_sha256: foreignPlanExecutionArtifact.digest, + ...common, grant_id: `${runId}-canonical-grant`, + result_ref: resultRef, result_sha256: foreignPlanExecutionArtifact.digest, plan_sha256: 'f'.repeat(64), }), /execution result binding is invalid/); fs.writeFileSync(replayPath, replayEnvelopeRaw); @@ -832,17 +897,20 @@ async function main() { persistJson(caseResultSetPath, { ...resultSet, set_id: 'tampered-after-completion' }); await assert.rejects(() => dispatch('load-result', { - ...common, result_ref: resultRef, result_sha256: replay.result_sha256, + ...common, grant_id: `${runId}-canonical-grant`, + result_ref: resultRef, result_sha256: replay.result_sha256, }), /case result set artifact digest differs/); persistJson(caseResultSetPath, resultSet); persistJson(evidenceManifestPath, { ...manifest, manifest_id: 'tampered-after-completion' }); await assert.rejects(() => dispatch('load-result', { - ...common, result_ref: resultRef, result_sha256: replay.result_sha256, + ...common, grant_id: `${runId}-canonical-grant`, + result_ref: resultRef, result_sha256: replay.result_sha256, }), /evidence manifest artifact digest differs/); persistJson(evidenceManifestPath, manifest); fs.writeFileSync(resultRef, '{}\n'); await assert.rejects(() => dispatch('load-result', { - ...common, result_ref: resultRef, result_sha256: replay.result_sha256, + ...common, grant_id: `${runId}-canonical-grant`, + result_ref: resultRef, result_sha256: replay.result_sha256, }), /digest differs/); delete process.env.FKST_OBJECT_BOUND_CLEANUP_BROKER; @@ -878,6 +946,7 @@ async function main() { fs.rmSync(`.testing/runs/${runId}`, { recursive: true, force: true }); fs.rmSync(linkPath, { force: true }); fs.rmSync(configRef, { force: true }); + fs.rmSync(foreignConfigRef, { force: true }); fs.rmSync(environmentConfigRef, { force: true }); fs.rmSync(temp, { recursive: true, force: true }); } diff --git a/packages/environment-factory/bin/object-bound-cleanup-broker.py b/packages/environment-factory/bin/object-bound-cleanup-broker.py index 290879cf..6f38262e 100644 --- a/packages/environment-factory/bin/object-bound-cleanup-broker.py +++ b/packages/environment-factory/bin/object-bound-cleanup-broker.py @@ -21,6 +21,7 @@ CAPTURE_CLEANED_MARKER = "captured-cleaned" CAPTURE_RETAINED_MARKER = "captured-retained" CAPTURE_FINALIZED_MARKER = "finalized" +CAPTURE_RELEASING_MARKER = "releasing" CAPTURE_MARKER_SCHEMA = "environment-factory.object-bound-cleanup-capture.v1" RENAME_NOREPLACE, RENAME_EXCL = 1, 0x00000004 @@ -273,6 +274,10 @@ def capture_marker_body(request): ) +def releasing_marker_body(request): + return b"releasing:" + capture_marker_body(request) + + def write_exclusive_file(directory_fd, name, body): flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if hasattr(os, "O_NOFOLLOW"): @@ -334,7 +339,7 @@ def capture_quarantine_name(request): return QUARANTINE_PREFIX + request["capture_id"] -def open_capture_quarantine(bound, request, create): +def open_capture_quarantine(bound, request, create, allow_releasing=False): name = capture_quarantine_name(request) created = False if create: @@ -365,7 +370,10 @@ def open_capture_quarantine(bound, request, create): if created: write_exclusive_file(directory_fd, CAPTURE_MARKER, expected_marker) os.fsync(directory_fd) - if read_bound_file(directory_fd, CAPTURE_MARKER) != expected_marker: + if marker_exists(directory_fd, CAPTURE_MARKER): + if read_bound_file(directory_fd, CAPTURE_MARKER) != expected_marker: + raise CleanupBlocked("capture-marker-binding-mismatch") + elif not allow_releasing: raise CleanupBlocked("capture-marker-binding-mismatch") return {"name": name, "fd": directory_fd, "stat": opened} except Exception: @@ -794,6 +802,7 @@ def validate_capture_entries(quarantine_fd): CAPTURE_CLEANED_MARKER, CAPTURE_RETAINED_MARKER, CAPTURE_FINALIZED_MARKER, + CAPTURE_RELEASING_MARKER, QUARANTINE_SLOT, } if not names.issubset(allowed): @@ -932,22 +941,57 @@ def release_capture_proof(request): bound = open_bound_root(request) quarantine = None try: - quarantine = open_capture_quarantine(bound, request, False) + quarantine = open_capture_quarantine(bound, request, False, allow_releasing=True) if quarantine is None: return capture_receipt(request, bound, "released") names = validate_capture_entries(quarantine["fd"]) - if names != {CAPTURE_MARKER, CAPTURE_CLEANED_MARKER, CAPTURE_FINALIZED_MARKER}: + initial = {CAPTURE_MARKER, CAPTURE_CLEANED_MARKER, CAPTURE_FINALIZED_MARKER} + replayable = { + frozenset(initial | {CAPTURE_RELEASING_MARKER}), + frozenset({CAPTURE_MARKER, CAPTURE_CLEANED_MARKER, CAPTURE_RELEASING_MARKER}), + frozenset({CAPTURE_MARKER, CAPTURE_RELEASING_MARKER}), + frozenset({CAPTURE_RELEASING_MARKER}), + frozenset(), + } + if names == initial: + if ( + read_bound_file(quarantine["fd"], CAPTURE_CLEANED_MARKER) + != marker_token(request, "cleaned") + or read_bound_file(quarantine["fd"], CAPTURE_FINALIZED_MARKER) + != marker_token(request, "finalized") + ): + raise CleanupBlocked("capture-proof-marker-invalid") + write_exclusive_file( + quarantine["fd"], + CAPTURE_RELEASING_MARKER, + releasing_marker_body(request), + ) + os.fsync(quarantine["fd"]) + names.add(CAPTURE_RELEASING_MARKER) + elif frozenset(names) not in replayable: raise CleanupBlocked("capture-proof-not-releasable") - if ( + if CAPTURE_RELEASING_MARKER in names and ( + read_bound_file(quarantine["fd"], CAPTURE_RELEASING_MARKER) + != releasing_marker_body(request) + ): + raise CleanupBlocked("capture-proof-marker-invalid") + if CAPTURE_CLEANED_MARKER in names and ( read_bound_file(quarantine["fd"], CAPTURE_CLEANED_MARKER) != marker_token(request, "cleaned") - or read_bound_file(quarantine["fd"], CAPTURE_FINALIZED_MARKER) + ): + raise CleanupBlocked("capture-proof-marker-invalid") + if CAPTURE_FINALIZED_MARKER in names and ( + read_bound_file(quarantine["fd"], CAPTURE_FINALIZED_MARKER) != marker_token(request, "finalized") ): raise CleanupBlocked("capture-proof-marker-invalid") for name in (CAPTURE_FINALIZED_MARKER, CAPTURE_CLEANED_MARKER, CAPTURE_MARKER): - os.unlink(name, dir_fd=quarantine["fd"]) - os.fsync(quarantine["fd"]) + if name in names: + os.unlink(name, dir_fd=quarantine["fd"]) + os.fsync(quarantine["fd"]) + if CAPTURE_RELEASING_MARKER in names: + os.unlink(CAPTURE_RELEASING_MARKER, dir_fd=quarantine["fd"]) + os.fsync(quarantine["fd"]) linked = stat_entry(bound["parent_fd"], quarantine["name"]) if not same_object(linked, quarantine["stat"]): raise CleanupBlocked("capture-quarantine-changed") diff --git a/packages/environment-factory/bin/runtime/common.js b/packages/environment-factory/bin/runtime/common.js index ce43b693..3cc4f663 100644 --- a/packages/environment-factory/bin/runtime/common.js +++ b/packages/environment-factory/bin/runtime/common.js @@ -1090,28 +1090,56 @@ function verifyWorkerEnvironmentReservation(reservation) { function releaseWorkerEnvironmentReservation(reservation) { verifyWorkerEnvironmentReservation(reservation); - // A missing pathname is not proof that allocation never published. The - // directory may have been displaced before its inode-bearing lease was - // durably recorded, so cleanup must retain the reservation for audit. - if (!pathEntryExists(reservation.home)) return false; const marker = workerHomeMarker(reservation.identity_sha256, reservation.lease_id); - const markerPath = path.join(reservation.home, '.fkst-worker-home.json'); - if (!pathEntryExists(markerPath) - || readBoundedRegularFile(markerPath, MAX_LOCK_METADATA_BYTES).body !== marker) { - throw new Error('worker environment reservation is not recoverable'); + let homeIdentity; + if (pathEntryExists(reservation.home)) { + const markerPath = path.join(reservation.home, '.fkst-worker-home.json'); + if (!pathEntryExists(markerPath) + || readBoundedRegularFile(markerPath, MAX_LOCK_METADATA_BYTES).body !== marker) { + throw new Error('worker environment reservation is not recoverable'); + } + homeIdentity = pathIdentity(reservation.home); + } else { + const state = readOptionalJson(cleanupCaptureStatePath(reservation.cleanup_capture_id)); + const stateName = state && state.state; + const targetIdentity = state && state.target_identity; + if (state === null) return false; + if (!['pending', 'captured-cleaned', 'finalized', 'released'].includes(stateName) + || !targetIdentity || targetIdentity.realpath !== reservation.home + || typeof targetIdentity.device !== 'string' || targetIdentity.device === '' + || typeof targetIdentity.inode !== 'string' || targetIdentity.inode === '') { + throw new Error('worker environment reservation release proof is invalid'); + } + const expectedState = { + ...cleanupCaptureBinding( + reservation.home, + targetIdentity, + reservation.homes_root, + reservation.homes_root_identity, + reservation.cleanup_capture_id, + ), + state: stateName, + }; + if (stableStringify(state) !== stableStringify(expectedState)) { + throw new Error('worker environment reservation release proof binding differs'); + } + homeIdentity = targetIdentity; } const lease = { schema: 'fkst.worker-home-lease.v1', lease_id: reservation.lease_id, home: reservation.home, - home_identity: pathIdentity(reservation.home), + home_identity: homeIdentity, homes_root: reservation.homes_root, homes_root_identity: reservation.homes_root_identity, marker_sha256: reservation.marker_sha256, identity_sha256: reservation.identity_sha256, cleanup_capture_id: reservation.cleanup_capture_id, }; - verifyWorkerEnvironmentLease(lease); + if (!reservationMatchesLease(reservation, lease)) { + throw new Error('worker environment reservation lease binding differs'); + } + if (pathEntryExists(reservation.home)) verifyWorkerEnvironmentLease(lease); return releaseWorkerEnvironmentLease(lease); } diff --git a/packages/environment-factory/tests/node_runtime_test.js b/packages/environment-factory/tests/node_runtime_test.js index 05e02717..30a0e3e3 100644 --- a/packages/environment-factory/tests/node_runtime_test.js +++ b/packages/environment-factory/tests/node_runtime_test.js @@ -14,6 +14,7 @@ const { pathIdentity, readBoundedRegularFile, removeOwnedDirectory, + releaseWorkerEnvironmentReservation, releaseWorkerEnvironment, releaseWorkerEnvironmentLease, stableStringify, @@ -358,6 +359,16 @@ async function main() { assert.strictEqual(releaseWorkerEnvironment(secondIsolated), true); assert.strictEqual(fs.existsSync(isolatedHome), false); assert.strictEqual(fs.existsSync(secondIsolatedHome), false); + let replayableReservation; + const reservationOnly = minimalEnvironment({}, 'reservation-only-release-replay', null, { + afterHomeDirectoryCreated({ reservation }) { + replayableReservation = reservation; + }, + }); + assert.ok(replayableReservation); + assert.strictEqual(releaseWorkerEnvironmentReservation(replayableReservation), true); + assert.strictEqual(fs.existsSync(reservationOnly.HOME), false); + assert.strictEqual(releaseWorkerEnvironmentReservation(replayableReservation), true); const brokerEnvironment = minimalEnvironment({}, 'node-runtime-broker-cleanup'); const externalDirectory = path.join(temp, 'external-cleanup-sentinel'); fs.mkdirSync(externalDirectory); diff --git a/packages/environment-factory/tests/object_bound_cleanup_backend_test.py b/packages/environment-factory/tests/object_bound_cleanup_backend_test.py index 5733bd93..b24de85d 100644 --- a/packages/environment-factory/tests/object_bound_cleanup_backend_test.py +++ b/packages/environment-factory/tests/object_bound_cleanup_backend_test.py @@ -2,8 +2,10 @@ """Backend-selection tests for atomic object-bound cleanup capture.""" import importlib.util +import os import pathlib import sys +import tempfile import unittest from unittest import mock @@ -73,5 +75,61 @@ def test_unsupported_platform_has_no_ordinary_rename_fallback(self): self.assertEqual(library.rename.calls, []) +class ProofRetirementRecoveryTest(unittest.TestCase): + def test_release_proof_replays_after_first_marker_unlink(self): + with tempfile.TemporaryDirectory() as temporary: + containment_root = pathlib.Path(temporary) / "containment" + target = containment_root / "target" + target.mkdir(parents=True) + (target / "owned.txt").write_text("owned\n", encoding="utf-8") + + def identity(candidate): + linked = os.stat(candidate, follow_symlinks=False) + return { + "realpath": os.path.realpath(candidate), + "device": str(linked.st_dev), + "inode": str(linked.st_ino), + } + + request = { + "schema": BROKER.REQUEST_SCHEMA, + "operation": "capture-delete", + "capture_id": "a" * 64, + "target": os.path.realpath(target), + "target_identity": identity(target), + "containment_root": os.path.realpath(containment_root), + "containment_root_identity": identity(containment_root), + } + self.assertEqual(BROKER.cleanup(request)["status"], "captured-cleaned") + self.assertEqual( + BROKER.cleanup({**request, "operation": "finalize"})["status"], + "finalized", + ) + + real_unlink = BROKER.os.unlink + interrupted = False + + def interrupt_after_first_unlink(name, *args, **kwargs): + nonlocal interrupted + real_unlink(name, *args, **kwargs) + if not interrupted: + interrupted = True + raise RuntimeError("simulated proof retirement interruption") + + with mock.patch.object(BROKER.os, "unlink", side_effect=interrupt_after_first_unlink): + with self.assertRaisesRegex(RuntimeError, "simulated proof retirement interruption"): + BROKER.cleanup({**request, "operation": "release-proof"}) + + proof = pathlib.Path(temporary) / f"{BROKER.QUARANTINE_PREFIX}{request['capture_id']}" + releasing = proof / BROKER.CAPTURE_RELEASING_MARKER + releasing.write_bytes(b"releasing:tampered\n") + with self.assertRaisesRegex(BROKER.CleanupBlocked, "capture-proof-marker-invalid"): + BROKER.cleanup({**request, "operation": "release-proof"}) + releasing.write_bytes(BROKER.releasing_marker_body(request)) + released = BROKER.cleanup({**request, "operation": "release-proof"}) + self.assertEqual(released["status"], "released") + self.assertFalse(proof.exists()) + + if __name__ == "__main__": unittest.main() diff --git a/packages/testing-runner/structured_execution.lua b/packages/testing-runner/structured_execution.lua index a3e94b9c..5dc8a13e 100644 --- a/packages/testing-runner/structured_execution.lua +++ b/packages/testing-runner/structured_execution.lua @@ -727,6 +727,7 @@ function M.run(request, ports) if claim.status == "completed" then local replayed = ports.load_result({ artifact_root = request.artifact_root, + grant_id = grant.value.grant_id, result_ref = claim.result_ref, result_sha256 = claim.result_sha256, operation_id = environment.value.operation_id, diff --git a/packages/testing-runner/tests/structured_execution_test.lua b/packages/testing-runner/tests/structured_execution_test.lua index 94fbf525..80e63a1c 100644 --- a/packages/testing-runner/tests/structured_execution_test.lua +++ b/packages/testing-runner/tests/structured_execution_test.lua @@ -785,12 +785,13 @@ return { test_completed_replay_reuses_result_without_effects_or_writes = function() local request = fixtures.request() - local writes = 0 + local writes, load_request = 0, nil local ports, effects = runtime(fixtures.artifacts(request), { replay_guard = function() return { status = "completed", result_ref = request.artifact_root .. "/execution.json" } end, - load_result = function() + load_result = function(input) + load_request = input return { status = "passed", classification = "passed", case_count = 1, passed_count = 1, failed_count = 0, skipped_count = 0, error_count = 0, @@ -816,6 +817,7 @@ return { t.eq(result.case_result_set_artifact_sha256, string.rep("a", 64)) t.eq(result.evidence_manifest_path, request.artifact_root .. "/evidence-manifest.json") t.eq(result.evidence_manifest_artifact_sha256, string.rep("b", 64)) + t.eq(load_request.grant_id, "grant-110") t.eq(#effects, 0) t.eq(writes, 0) end, From 3823462b3543fd77613dd23bf1da02858fb1b8fc Mon Sep 17 00:00:00 2001 From: Shaw Zheng Date: Tue, 15 Sep 2026 18:07:40 +0800 Subject: [PATCH 11/11] fix(testing): bound proof retirement recovery --- .../bin/object-bound-cleanup-broker.py | 81 +++++-------------- .../object_bound_cleanup_backend_test.py | 6 +- 2 files changed, 21 insertions(+), 66 deletions(-) diff --git a/packages/environment-factory/bin/object-bound-cleanup-broker.py b/packages/environment-factory/bin/object-bound-cleanup-broker.py index 6f38262e..76884ddd 100644 --- a/packages/environment-factory/bin/object-bound-cleanup-broker.py +++ b/packages/environment-factory/bin/object-bound-cleanup-broker.py @@ -21,7 +21,6 @@ CAPTURE_CLEANED_MARKER = "captured-cleaned" CAPTURE_RETAINED_MARKER = "captured-retained" CAPTURE_FINALIZED_MARKER = "finalized" -CAPTURE_RELEASING_MARKER = "releasing" CAPTURE_MARKER_SCHEMA = "environment-factory.object-bound-cleanup-capture.v1" RENAME_NOREPLACE, RENAME_EXCL = 1, 0x00000004 @@ -274,10 +273,6 @@ def capture_marker_body(request): ) -def releasing_marker_body(request): - return b"releasing:" + capture_marker_body(request) - - def write_exclusive_file(directory_fd, name, body): flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if hasattr(os, "O_NOFOLLOW"): @@ -339,7 +334,7 @@ def capture_quarantine_name(request): return QUARANTINE_PREFIX + request["capture_id"] -def open_capture_quarantine(bound, request, create, allow_releasing=False): +def open_capture_quarantine(bound, request, create, require_capture_marker=True): name = capture_quarantine_name(request) created = False if create: @@ -370,10 +365,11 @@ def open_capture_quarantine(bound, request, create, allow_releasing=False): if created: write_exclusive_file(directory_fd, CAPTURE_MARKER, expected_marker) os.fsync(directory_fd) - if marker_exists(directory_fd, CAPTURE_MARKER): - if read_bound_file(directory_fd, CAPTURE_MARKER) != expected_marker: - raise CleanupBlocked("capture-marker-binding-mismatch") - elif not allow_releasing: + if marker_exists(directory_fd, CAPTURE_MARKER) and read_bound_file( + directory_fd, CAPTURE_MARKER + ) != expected_marker: + raise CleanupBlocked("capture-marker-binding-mismatch") + if require_capture_marker and not marker_exists(directory_fd, CAPTURE_MARKER): raise CleanupBlocked("capture-marker-binding-mismatch") return {"name": name, "fd": directory_fd, "stat": opened} except Exception: @@ -802,7 +798,6 @@ def validate_capture_entries(quarantine_fd): CAPTURE_CLEANED_MARKER, CAPTURE_RETAINED_MARKER, CAPTURE_FINALIZED_MARKER, - CAPTURE_RELEASING_MARKER, QUARANTINE_SLOT, } if not names.issubset(allowed): @@ -917,17 +912,10 @@ def finalize_capture(request): ): raise CleanupBlocked("capture-cleaned-marker-invalid") if CAPTURE_FINALIZED_MARKER in names: - if ( - read_bound_file(quarantine["fd"], CAPTURE_FINALIZED_MARKER) - != marker_token(request, "finalized") - ): + if read_bound_file(quarantine["fd"], CAPTURE_FINALIZED_MARKER) != capture_marker_body(request): raise CleanupBlocked("capture-finalized-marker-invalid") else: - write_exclusive_file( - quarantine["fd"], - CAPTURE_FINALIZED_MARKER, - marker_token(request, "finalized"), - ) + write_exclusive_file(quarantine["fd"], CAPTURE_FINALIZED_MARKER, capture_marker_body(request)) os.fsync(quarantine["fd"]) return capture_receipt(request, bound, "finalized") finally: @@ -941,57 +929,24 @@ def release_capture_proof(request): bound = open_bound_root(request) quarantine = None try: - quarantine = open_capture_quarantine(bound, request, False, allow_releasing=True) + quarantine = open_capture_quarantine(bound, request, False, require_capture_marker=False) if quarantine is None: return capture_receipt(request, bound, "released") names = validate_capture_entries(quarantine["fd"]) - initial = {CAPTURE_MARKER, CAPTURE_CLEANED_MARKER, CAPTURE_FINALIZED_MARKER} - replayable = { - frozenset(initial | {CAPTURE_RELEASING_MARKER}), - frozenset({CAPTURE_MARKER, CAPTURE_CLEANED_MARKER, CAPTURE_RELEASING_MARKER}), - frozenset({CAPTURE_MARKER, CAPTURE_RELEASING_MARKER}), - frozenset({CAPTURE_RELEASING_MARKER}), - frozenset(), - } - if names == initial: - if ( - read_bound_file(quarantine["fd"], CAPTURE_CLEANED_MARKER) - != marker_token(request, "cleaned") - or read_bound_file(quarantine["fd"], CAPTURE_FINALIZED_MARKER) - != marker_token(request, "finalized") - ): - raise CleanupBlocked("capture-proof-marker-invalid") - write_exclusive_file( - quarantine["fd"], - CAPTURE_RELEASING_MARKER, - releasing_marker_body(request), - ) - os.fsync(quarantine["fd"]) - names.add(CAPTURE_RELEASING_MARKER) - elif frozenset(names) not in replayable: + allowed = {CAPTURE_MARKER, CAPTURE_CLEANED_MARKER, CAPTURE_FINALIZED_MARKER} + if (not names.issubset(allowed) + or (CAPTURE_MARKER in names and CAPTURE_CLEANED_MARKER not in names) + or (CAPTURE_CLEANED_MARKER in names and CAPTURE_FINALIZED_MARKER not in names)): raise CleanupBlocked("capture-proof-not-releasable") - if CAPTURE_RELEASING_MARKER in names and ( - read_bound_file(quarantine["fd"], CAPTURE_RELEASING_MARKER) - != releasing_marker_body(request) - ): - raise CleanupBlocked("capture-proof-marker-invalid") - if CAPTURE_CLEANED_MARKER in names and ( - read_bound_file(quarantine["fd"], CAPTURE_CLEANED_MARKER) - != marker_token(request, "cleaned") - ): - raise CleanupBlocked("capture-proof-marker-invalid") - if CAPTURE_FINALIZED_MARKER in names and ( - read_bound_file(quarantine["fd"], CAPTURE_FINALIZED_MARKER) - != marker_token(request, "finalized") - ): + markers = ((CAPTURE_CLEANED_MARKER, marker_token(request, "cleaned")), + (CAPTURE_FINALIZED_MARKER, capture_marker_body(request))) + if any(name in names and read_bound_file(quarantine["fd"], name) != body + for name, body in markers): raise CleanupBlocked("capture-proof-marker-invalid") - for name in (CAPTURE_FINALIZED_MARKER, CAPTURE_CLEANED_MARKER, CAPTURE_MARKER): + for name in (CAPTURE_MARKER, CAPTURE_CLEANED_MARKER, CAPTURE_FINALIZED_MARKER): if name in names: os.unlink(name, dir_fd=quarantine["fd"]) os.fsync(quarantine["fd"]) - if CAPTURE_RELEASING_MARKER in names: - os.unlink(CAPTURE_RELEASING_MARKER, dir_fd=quarantine["fd"]) - os.fsync(quarantine["fd"]) linked = stat_entry(bound["parent_fd"], quarantine["name"]) if not same_object(linked, quarantine["stat"]): raise CleanupBlocked("capture-quarantine-changed") diff --git a/packages/environment-factory/tests/object_bound_cleanup_backend_test.py b/packages/environment-factory/tests/object_bound_cleanup_backend_test.py index b24de85d..45c838a3 100644 --- a/packages/environment-factory/tests/object_bound_cleanup_backend_test.py +++ b/packages/environment-factory/tests/object_bound_cleanup_backend_test.py @@ -121,11 +121,11 @@ def interrupt_after_first_unlink(name, *args, **kwargs): BROKER.cleanup({**request, "operation": "release-proof"}) proof = pathlib.Path(temporary) / f"{BROKER.QUARANTINE_PREFIX}{request['capture_id']}" - releasing = proof / BROKER.CAPTURE_RELEASING_MARKER - releasing.write_bytes(b"releasing:tampered\n") + finalized = proof / BROKER.CAPTURE_FINALIZED_MARKER + finalized.write_bytes(b"finalized:tampered\n") with self.assertRaisesRegex(BROKER.CleanupBlocked, "capture-proof-marker-invalid"): BROKER.cleanup({**request, "operation": "release-proof"}) - releasing.write_bytes(BROKER.releasing_marker_body(request)) + finalized.write_bytes(BROKER.capture_marker_body(request)) released = BROKER.cleanup({**request, "operation": "release-proof"}) self.assertEqual(released["status"], "released") self.assertFalse(proof.exists())