fix(hosted): fence artifact abandonment cleanup

.github/workflows/ci.yml

@@ -12,6 +12,20 @@ permissions:
 jobs:
   rust:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:17-alpine
+        env:
+          POSTGRES_DB: call_scribe_test
+          POSTGRES_USER: call_scribe_test
+          POSTGRES_PASSWORD: call_scribe_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U call_scribe_test -d call_scribe_test"
+          --health-interval 2s
+          --health-timeout 5s
+          --health-retries 15
     steps:
       - name: Check out repository
         uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
@@ -32,6 +46,8 @@ jobs:
         run: cargo clippy --all-features --all-targets --locked -- -D warnings
 
       - name: Run tests
+        env:
+          CALL_SCRIBE_TEST_DATABASE_URL: postgres://call_scribe_test:call_scribe_test@127.0.0.1:5432/call_scribe_test
         run: cargo test --all-features --locked
 
       - name: Audit dependencies

Cargo.toml

@@ -35,7 +35,7 @@ rustls = { version = "0.23", default-features = false, features = ["ring"], opti
 serenity = { version = "0.12", default-features = false, features = ["client", "gateway", "model", "native_tls_backend"], optional = true }
 songbird = { version = "0.6", default-features = false, features = ["driver", "gateway", "native", "serenity", "tungstenite", "receive"], optional = true }
 sqlx = { package = "sqlx-core", version = "0.8", default-features = false, features = ["_rt-tokio", "_tls-rustls-ring-webpki", "chrono", "json"] }
-sqlx-postgres = { version = "0.8", features = ["chrono", "json"] }
+sqlx-postgres = { version = "0.8", features = ["chrono", "json", "uuid"] }
 tokio = { version = "1.47", features = ["fs", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] }
 tokio-util = { version = "0.7", features = ["io"] }
 tower-http = { version = "0.6", features = ["fs"] }

docs/deployment/hosted-worker.md

@@ -285,8 +285,53 @@ deletion. The provider `HEAD` must return the exact content length and
 the local WAV, and mark delivery final. It never stores either presigned URL or
 its headers. Failed uploads or verification retain the local WAV for fenced
 retry, and an unsafe delivery older than ten minutes backpressures later hosted
-starts for that organization/guild. After twenty attempts the job is terminal
-and still backpressures capture for operator reconciliation.
+starts for that organization/guild. After twenty attempts the worker clears the
+encrypted reservation lease and durably retries one idempotent abandonment
+notification:
+
+```json
+{
+  "notificationId": "7754ab7e-c0f7-41a1-8373-9c213f27efb5",
+  "operationId": "7f634aa5-f8cb-41b9-8c92-b23e5a44ae53",
+  "generation": 7,
+  "reservationId": "c5596189-da7b-4437-bd21-4541f877ee8a",
+  "recordingId": "recording_01",
+  "artifactId": "artifact_01",
+  "artifactKind": "raw_audio_wav",
+  "segmentIndex": 1,
+  "objectKey": "objects/artifact_01.wav",
+  "destinationId": "84b890cf-88d2-4979-b099-08978d2faedf",
+  "destinationRevision": "c5b70242-8402-4120-a26d-122687ec9c06",
+  "provider": "customer_s3",
+  "allowedUploadHost": "bucket.s3.us-east-1.amazonaws.com",
+  "contentLength": 123456,
+  "sha256": "64-lowercase-hex-characters",
+  "reason": "retry_budget_exhausted"
+}
+```
+
+The endpoint is
+`POST /internal/v1/worker/artifact-deliveries/abandon`. `notificationId` is
+generated once and reused for every retry. Operation ID, generation, and object
+key are either all present or all absent. A null operation tuple also covers a
+lost prepare response: the control plane resolves the exact historical
+reservation/artifact/destination tuple and never reveals the internally
+resolved operation in its echo. The control plane tuple-digest fences
+notification replay and exactly echoes every identity, destination, size, and
+hash field. A `cleanup_pending`/`tombstone_queued` response is only an
+acknowledgement: the worker retains the local WAV and raw locator metadata and
+continues backpressure while the customer-provider object may exist. Only a
+`provider_absent` response with `no_operation`, `not_required`, or
+`provider_absence_verified` authorizes local deletion and terminal metadata
+minimization. Thus neither service may race a delayed upload with a time-only
+deletion.
+
+After verified delivery/local purge or authoritative provider absence/local
+purge, the worker atomically replaces the raw terminal outbox, runtime artifact,
+receipt, provider host/key, filesystem path, and customer identifiers with
+domain-separated SHA-256 audit evidence. A bounded sweep processes 25 legacy
+terminal rows per worker pass. Self-hosted runtime artifacts and audit behavior
+are unchanged.
 
 ## Rollout
 

docs/product/RUNTIME_DATABASE.md

@@ -122,6 +122,15 @@ Stores events such as:
 
 This table provides an append-only operational history for self-hosted deployments.
 
+Hosted raw-audio delivery uses a separate durable outbox while delivery or
+provider cleanup is active. Once verified delivery has removed the local file,
+or an idempotent control-plane abandonment handshake has authoritatively proved
+the provider object absent and the local file is removed, raw outbox, artifact,
+receipt, locator, and customer-identity fields are deleted. The hosted terminal
+audit table retains only domain-separated SHA-256 identity evidence, artifact
+shape, attempt counts, and completion timestamps. Cleanup-pending abandonment
+never deletes or minimizes the remaining local copy.
+
 ## Capture mode
 
 Set `CALL_SCRIBE_CAPTURE_MODE`:

migrations/20260812040000_hosted_delivery_terminal_privacy.sql

@@ -0,0 +1,138 @@
+-- Preserve an authoritative worker-abandonment handshake after upload retry
+-- exhaustion. Local audio remains recoverable until the control plane proves
+-- that the customer provider object is absent.
+ALTER TABLE call_scribe_hosted_artifact_delivery_outbox
+    ADD COLUMN IF NOT EXISTS abandonment_notification_id UUID,
+    ADD COLUMN IF NOT EXISTS abandonment_notification_attempt_count BIGINT NOT NULL DEFAULT 0,
+    ADD COLUMN IF NOT EXISTS abandonment_notified_at TIMESTAMPTZ;
+
+-- Replace the prerelease lifecycle checks additively without depending on the
+-- automatically generated constraint suffixes used by a particular Postgres
+-- version.
+DO $$
+DECLARE
+    constraint_name TEXT;
+BEGIN
+    FOR constraint_name IN
+        SELECT constraints.conname
+        FROM pg_constraint constraints
+        WHERE conrelid = 'call_scribe_hosted_artifact_delivery_outbox'::regclass
+          AND contype = 'c'
+          AND (
+              constraints.conname = 'call_scribe_hosted_artifact_delivery_status_v2_check'
+              OR constraints.conname = 'call_scribe_hosted_artifact_delivery_lifecycle_v2_check'
+              OR pg_get_constraintdef(constraints.oid) LIKE '%status = ANY%pending%in_progress%verified%delivered%failed%'
+              OR pg_get_constraintdef(constraints.oid) LIKE '%status = ''pending''%encrypted_lease_token%status = ''failed''%'
+          )
+    LOOP
+        EXECUTE format(
+            'ALTER TABLE call_scribe_hosted_artifact_delivery_outbox DROP CONSTRAINT %I',
+            constraint_name
+        );
+    END LOOP;
+END $$;
+
+UPDATE call_scribe_hosted_artifact_delivery_outbox
+SET status = 'abandonment_pending',
+    abandonment_notification_id = COALESCE(abandonment_notification_id, gen_random_uuid()),
+    updated_at = now()
+WHERE status = 'failed';
+
+ALTER TABLE call_scribe_hosted_artifact_delivery_outbox
+    ADD CONSTRAINT call_scribe_hosted_artifact_delivery_status_v2_check
+        CHECK (status IN (
+            'pending', 'in_progress', 'verified', 'delivered',
+            'abandonment_pending', 'cleanup_pending', 'abandoned'
+        )),
+    ADD CONSTRAINT call_scribe_hosted_artifact_delivery_lifecycle_v2_check CHECK (
+        (status = 'pending'
+            AND encrypted_lease_token IS NOT NULL
+            AND encryption_nonce IS NOT NULL
+            AND claim_token IS NULL
+            AND claim_until IS NULL
+            AND abandonment_notification_id IS NULL)
+        OR (status = 'in_progress'
+            AND encrypted_lease_token IS NOT NULL
+            AND encryption_nonce IS NOT NULL
+            AND claim_owner IS NOT NULL
+            AND claim_token IS NOT NULL
+            AND claim_until IS NOT NULL
+            AND abandonment_notification_id IS NULL)
+        OR (status = 'verified'
+            AND encrypted_lease_token IS NULL
+            AND encryption_nonce IS NULL
+            AND receipt IS NOT NULL
+            AND verified_at IS NOT NULL
+            AND abandonment_notification_id IS NULL
+            AND ((claim_owner IS NULL AND claim_token IS NULL AND claim_until IS NULL)
+                OR (claim_owner IS NOT NULL AND claim_token IS NOT NULL
+                    AND claim_until IS NOT NULL)))
+        OR (status = 'delivered'
+            AND encrypted_lease_token IS NULL
+            AND encryption_nonce IS NULL
+            AND receipt IS NOT NULL
+            AND verified_at IS NOT NULL
+            AND local_deleted_at IS NOT NULL
+            AND completed_at IS NOT NULL
+            AND claim_token IS NULL
+            AND claim_until IS NULL
+            AND abandonment_notification_id IS NULL)
+        OR (status IN ('abandonment_pending', 'cleanup_pending')
+            AND encrypted_lease_token IS NULL
+            AND encryption_nonce IS NULL
+            AND abandonment_notification_id IS NOT NULL
+            AND ((claim_owner IS NULL AND claim_token IS NULL AND claim_until IS NULL)
+                OR (claim_owner IS NOT NULL AND claim_token IS NOT NULL
+                    AND claim_until IS NOT NULL)))
+        OR (status = 'abandoned'
+            AND encrypted_lease_token IS NULL
+            AND encryption_nonce IS NULL
+            AND abandonment_notification_id IS NOT NULL
+            AND local_deleted_at IS NOT NULL
+            AND completed_at IS NOT NULL
+            AND claim_token IS NULL
+            AND claim_until IS NULL)
+    );
+
+CREATE INDEX IF NOT EXISTS call_scribe_hosted_artifact_abandonment_claim_idx
+    ON call_scribe_hosted_artifact_delivery_outbox(next_attempt_at, claim_until, created_at)
+    WHERE status IN ('abandonment_pending', 'cleanup_pending');
+
+-- Raw tenant, provider-locator, receipt, and filesystem fields live only in the
+-- active outbox. Terminal rows are moved here and retain domain-separated
+-- SHA-256 evidence only.
+CREATE TABLE IF NOT EXISTS call_scribe_hosted_artifact_delivery_terminal_audit (
+    id UUID PRIMARY KEY,
+    notification_id UUID UNIQUE,
+    terminal_state TEXT NOT NULL CHECK (terminal_state IN ('delivered', 'abandoned')),
+    organization_id_sha256 TEXT NOT NULL CHECK (organization_id_sha256 ~ '^[0-9a-f]{64}$'),
+    guild_id_sha256 TEXT NOT NULL CHECK (guild_id_sha256 ~ '^[0-9a-f]{64}$'),
+    recording_id_sha256 TEXT NOT NULL CHECK (recording_id_sha256 ~ '^[0-9a-f]{64}$'),
+    artifact_id_sha256 TEXT NOT NULL CHECK (artifact_id_sha256 ~ '^[0-9a-f]{64}$'),
+    reservation_id_sha256 TEXT NOT NULL CHECK (reservation_id_sha256 ~ '^[0-9a-f]{64}$'),
+    operation_id_sha256 TEXT CHECK (operation_id_sha256 IS NULL OR operation_id_sha256 ~ '^[0-9a-f]{64}$'),
+    object_key_sha256 TEXT CHECK (object_key_sha256 IS NULL OR object_key_sha256 ~ '^[0-9a-f]{64}$'),
+    destination_id_sha256 TEXT NOT NULL CHECK (destination_id_sha256 ~ '^[0-9a-f]{64}$'),
+    destination_revision_sha256 TEXT NOT NULL CHECK (destination_revision_sha256 ~ '^[0-9a-f]{64}$'),
+    allowed_upload_host_sha256 TEXT NOT NULL CHECK (allowed_upload_host_sha256 ~ '^[0-9a-f]{64}$'),
+    provider TEXT NOT NULL CHECK (provider IN ('customer_s3', 'customer_r2')),
+    artifact_kind TEXT NOT NULL CHECK (artifact_kind = 'raw_audio_wav'),
+    segment_index INTEGER NOT NULL CHECK (segment_index BETWEEN 1 AND 16),
+    content_length BIGINT NOT NULL CHECK (content_length > 0),
+    content_sha256 TEXT NOT NULL CHECK (content_sha256 ~ '^[0-9a-f]{64}$'),
+    receipt_sha256 TEXT CHECK (receipt_sha256 IS NULL OR receipt_sha256 ~ '^[0-9a-f]{64}$'),
+    delivery_attempt_count INTEGER NOT NULL CHECK (delivery_attempt_count >= 0),
+    abandonment_notification_attempt_count BIGINT NOT NULL CHECK (abandonment_notification_attempt_count >= 0),
+    provider_absence_verified_at TIMESTAMPTZ,
+    completed_at TIMESTAMPTZ NOT NULL,
+    archived_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    CHECK (
+        (terminal_state = 'delivered' AND notification_id IS NULL
+            AND receipt_sha256 IS NOT NULL AND provider_absence_verified_at IS NULL)
+        OR (terminal_state = 'abandoned' AND notification_id IS NOT NULL
+            AND receipt_sha256 IS NULL AND provider_absence_verified_at IS NOT NULL)
+    )
+);
+
+CREATE INDEX IF NOT EXISTS call_scribe_hosted_terminal_audit_artifact_idx
+    ON call_scribe_hosted_artifact_delivery_terminal_audit(artifact_id_sha256, archived_at);