M3: drift detection — two-tier engine, fail-closed enforcement, exports, five-page dashboard

.github/workflows/ci.yml

@@ -1,5 +1,9 @@
-# M1+M2 CI. Unit tests only — no container services; the live E2E gate
+# M1+M2+M3 CI. Unit tests only — no container services; the live E2E gate
 # is `python scripts/smoke.py`, run locally against the compose stack.
+# Service deps install per-service (drift joined in M3; its psycopg pin is
+# identical to gateway's — the line exists so a future drift dep change
+# reaches CI with the suite that imports it). pyarrow==18.1.0 rides
+# services/gateway/requirements.txt into CI (py3.12 manylinux wheel).
 name: ci
 
 on: [push, pull_request]
@@ -19,6 +23,7 @@ jobs:
                       -r services/pipeline/requirements.txt \
                       -r services/gateway/requirements.txt \
                       -r services/onboarding/requirements.txt \
+                      -r services/drift/requirements.txt \
                       -r simulator/requirements.txt
           pip install pytest~=8.4 ruff==0.16.8
       - name: Lint

.gitignore

@@ -48,3 +48,8 @@ references/
 node_modules/
 web/dist/
 .pytest_cache/
+
+# M3 pre-work: submission artifacts + demo residue (rooted)
+/brag-output/
+/testdata/
+.superpowers/

deploy/.env.example

@@ -15,5 +15,22 @@ ULPF_OLLAMA_URL=http://ollama-4b:11434
 # they are reachable only inside the compose network (onboarding-internal).
 ULPF_OLLAMA_MODEL=qwen3:4b
 
+# Drift engine knobs (M3) — read by services/drift's Config.from_env;
+# docker compose passes them through to the drift service. The acceptance
+# smoke (scripts/smoke.py) exports WINDOW_COUNT=30 plus a huge WINDOW_TIME_S
+# (count-close only) in its own environment before compose up, so demo
+# windows close on deterministic 30-row boundaries (a short time horizon
+# slices the accumulated days-spread history into one-row windows — see the
+# smoke header).
+ULPF_DRIFT_WINDOW_COUNT=1000
+ULPF_DRIFT_WINDOW_TIME_S=600
+ULPF_DRIFT_BASELINE_WINDOWS=10
+ULPF_DRIFT_POLL_S=5
+
+# Onboarding sample-capture cap per fingerprint (pipeline). The accumulating
+# acceptance runs saturate the default 200 — the smoke exports a larger value
+# so a drift-deactivated fingerprint can still show sample growth (K).
+ULPF_SAMPLE_CAPTURE_CAP=200
+
 # Per-service database DSNs (pipeline, gateway, admin) are compose-internal
 # constants defined in docker-compose.yml — M2 adds no user-facing DB knobs.

deploy/docker-compose.yml

@@ -48,6 +48,11 @@ services:
     environment:
       KAFKA_BOOTSTRAP: redpanda:9092
       DATABASE_URL: postgresql://pipeline_role:pipeline_dev@postgres:5432/ulpf
+      # Sample-capture headroom (M3 Task 12): the accumulating multi-run DB
+      # saturates the 200/fingerprint cap — after which a drift-deactivated
+      # fingerprint cannot accumulate onboarding samples and its re-entry is
+      # unobservable. Default stays 200; the smoke exports a larger value.
+      ULPF_SAMPLE_CAPTURE_CAP: ${ULPF_SAMPLE_CAPTURE_CAP:-200}
     restart: unless-stopped
     depends_on: { migrate: { condition: service_completed_successfully }, topics: { condition: service_completed_successfully }, seed: { condition: service_completed_successfully } }
   gateway:
@@ -82,6 +87,21 @@ services:
       OLLAMA_MODEL: ${ULPF_OLLAMA_MODEL:-qwen3:4b}
     restart: unless-stopped
     depends_on: { migrate: { condition: service_completed_successfully } }
+  # M3 drift detection (cold path, Postgres only — no Kafka, no profile so it
+  # is always on): statelessly recomputes per-fingerprint windows from the DB
+  # current view, fires tier-1 invariants from event zero, tier-2 statistics
+  # once a baseline exists; INSERT-only drift writes. Fail-closed: a failed
+  # scan is retried next poll — drift never blocks the hot path.
+  drift:
+    build: { context: .., dockerfile: services/drift/Dockerfile }
+    environment:
+      DATABASE_URL: postgresql://drift_role:drift_dev@postgres:5432/ulpf
+      ULPF_DRIFT_WINDOW_COUNT: ${ULPF_DRIFT_WINDOW_COUNT:-1000}
+      ULPF_DRIFT_WINDOW_TIME_S: ${ULPF_DRIFT_WINDOW_TIME_S:-600}
+      ULPF_DRIFT_BASELINE_WINDOWS: ${ULPF_DRIFT_BASELINE_WINDOWS:-10}
+      ULPF_DRIFT_POLL_S: ${ULPF_DRIFT_POLL_S:-5}
+    restart: unless-stopped
+    depends_on: { migrate: { condition: service_completed_successfully } }
   # Ollama sidecars. No host port by design: only the onboarding service talks
   # to them, inside the compose network. The entrypoint serves, then blocks on
   # `ollama pull` until the tier's model is local, then waits on serve.

deploy/migrations/007_drift_tables.sql

@@ -0,0 +1,50 @@
+-- deploy/migrations/007_drift_tables.sql — M3 drift detection (R-M3-2/3).
+-- Natural PKs ONLY (no serials): the 004/005/006 sequence-grant bug class is
+-- structurally impossible here. Drift writes are INSERT-only.
+DO $$ BEGIN
+  IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='drift_role') THEN
+    CREATE ROLE drift_role LOGIN PASSWORD 'drift_dev'; END IF;
+END $$;
+
+CREATE TABLE drift_windows (
+  fingerprint_id TEXT NOT NULL,
+  rule_version   INT NOT NULL,
+  field          TEXT NOT NULL,        -- OCSF path, 'unmapped.<key>', or '__rule__' sentinel
+  window_start   TIMESTAMPTZ NOT NULL, -- parsed_at of the window's first event (deterministic key)
+  window_end     TIMESTAMPTZ NOT NULL,
+  events_count   INT NOT NULL,
+  null_rate      REAL,
+  match_rate     REAL,                 -- '__rule__' rows: parsed / (parsed + parse_error)
+  violation_rate REAL,                 -- tier-1 invariant violations for this field
+  shape_dist     JSONB,                -- {"ipv4": 12, "int": 3, ...} over non-null scalar values
+  severity       TEXT NOT NULL CHECK (severity IN ('none','minor','moderate','severe')),
+  action_taken   TEXT CHECK (action_taken IS NULL OR action_taken IN
+                ('alert','field_quarantined','rule_deactivated')),
+  scanned_at     TIMESTAMPTZ NOT NULL DEFAULT now(),
+  PRIMARY KEY (fingerprint_id, rule_version, field, window_start)
+);
+CREATE INDEX drift_windows_recent ON drift_windows (fingerprint_id, window_start DESC);
+
+CREATE TABLE baseline_profiles (
+  fingerprint_id TEXT NOT NULL,
+  rule_version   INT NOT NULL,
+  field          TEXT NOT NULL,
+  profile        JSONB NOT NULL,       -- {events_count, null_rate, match_rate, shape_dist}
+  windows_seen   INT NOT NULL,
+  established_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  PRIMARY KEY (fingerprint_id, rule_version, field)
+);
+
+-- drift service (detection + enforcement, R-M3-3)
+GRANT SELECT ON normalized_events, rules TO drift_role;
+GRANT INSERT ON drift_windows, baseline_profiles TO drift_role;
+-- P-2: drift_role self-reads — baseline existence checks + recent-window lookups.
+GRANT SELECT ON drift_windows, baseline_profiles TO drift_role;
+GRANT UPDATE (quarantined_fields) ON rules TO drift_role;
+GRANT UPDATE (status, deactivated_at) ON rules TO drift_role;
+GRANT INSERT ON audit_log TO drift_role;
+GRANT USAGE, SELECT ON SEQUENCE audit_log_id_seq TO drift_role;
+
+-- gateway: human override of quarantine (audited, rules_role write path)
+GRANT UPDATE (quarantined_fields) ON rules TO rules_role;
+-- gateway SELECT on both new tables arrives via 003's ALTER DEFAULT PRIVILEGES.