diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10932e9..d794251 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,9 @@ -# M1+M2 CI. Unit tests only — no container services; the live E2E gate +# M1+M2+M3 CI. Unit tests only — no container services; the live E2E gate # is `python scripts/smoke.py`, run locally against the compose stack. +# Service deps install per-service (drift joined in M3; its psycopg pin is +# identical to gateway's — the line exists so a future drift dep change +# reaches CI with the suite that imports it). pyarrow==18.1.0 rides +# services/gateway/requirements.txt into CI (py3.12 manylinux wheel). name: ci on: [push, pull_request] @@ -19,6 +23,7 @@ jobs: -r services/pipeline/requirements.txt \ -r services/gateway/requirements.txt \ -r services/onboarding/requirements.txt \ + -r services/drift/requirements.txt \ -r simulator/requirements.txt pip install pytest~=8.4 ruff==0.16.8 - name: Lint diff --git a/.gitignore b/.gitignore index 3a5c76e..63f6daa 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,8 @@ references/ node_modules/ web/dist/ .pytest_cache/ + +# M3 pre-work: submission artifacts + demo residue (rooted) +/brag-output/ +/testdata/ +.superpowers/ diff --git a/deploy/.env.example b/deploy/.env.example index c652beb..dfd8701 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -15,5 +15,22 @@ ULPF_OLLAMA_URL=http://ollama-4b:11434 # they are reachable only inside the compose network (onboarding-internal). ULPF_OLLAMA_MODEL=qwen3:4b +# Drift engine knobs (M3) — read by services/drift's Config.from_env; +# docker compose passes them through to the drift service. The acceptance +# smoke (scripts/smoke.py) exports WINDOW_COUNT=30 plus a huge WINDOW_TIME_S +# (count-close only) in its own environment before compose up, so demo +# windows close on deterministic 30-row boundaries (a short time horizon +# slices the accumulated days-spread history into one-row windows — see the +# smoke header). +ULPF_DRIFT_WINDOW_COUNT=1000 +ULPF_DRIFT_WINDOW_TIME_S=600 +ULPF_DRIFT_BASELINE_WINDOWS=10 +ULPF_DRIFT_POLL_S=5 + +# Onboarding sample-capture cap per fingerprint (pipeline). The accumulating +# acceptance runs saturate the default 200 — the smoke exports a larger value +# so a drift-deactivated fingerprint can still show sample growth (K). +ULPF_SAMPLE_CAPTURE_CAP=200 + # Per-service database DSNs (pipeline, gateway, admin) are compose-internal # constants defined in docker-compose.yml — M2 adds no user-facing DB knobs. diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index dc7bae6..532c47f 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -48,6 +48,11 @@ services: environment: KAFKA_BOOTSTRAP: redpanda:9092 DATABASE_URL: postgresql://pipeline_role:pipeline_dev@postgres:5432/ulpf + # Sample-capture headroom (M3 Task 12): the accumulating multi-run DB + # saturates the 200/fingerprint cap — after which a drift-deactivated + # fingerprint cannot accumulate onboarding samples and its re-entry is + # unobservable. Default stays 200; the smoke exports a larger value. + ULPF_SAMPLE_CAPTURE_CAP: ${ULPF_SAMPLE_CAPTURE_CAP:-200} restart: unless-stopped depends_on: { migrate: { condition: service_completed_successfully }, topics: { condition: service_completed_successfully }, seed: { condition: service_completed_successfully } } gateway: @@ -82,6 +87,21 @@ services: OLLAMA_MODEL: ${ULPF_OLLAMA_MODEL:-qwen3:4b} restart: unless-stopped depends_on: { migrate: { condition: service_completed_successfully } } + # M3 drift detection (cold path, Postgres only — no Kafka, no profile so it + # is always on): statelessly recomputes per-fingerprint windows from the DB + # current view, fires tier-1 invariants from event zero, tier-2 statistics + # once a baseline exists; INSERT-only drift writes. Fail-closed: a failed + # scan is retried next poll — drift never blocks the hot path. + drift: + build: { context: .., dockerfile: services/drift/Dockerfile } + environment: + DATABASE_URL: postgresql://drift_role:drift_dev@postgres:5432/ulpf + ULPF_DRIFT_WINDOW_COUNT: ${ULPF_DRIFT_WINDOW_COUNT:-1000} + ULPF_DRIFT_WINDOW_TIME_S: ${ULPF_DRIFT_WINDOW_TIME_S:-600} + ULPF_DRIFT_BASELINE_WINDOWS: ${ULPF_DRIFT_BASELINE_WINDOWS:-10} + ULPF_DRIFT_POLL_S: ${ULPF_DRIFT_POLL_S:-5} + restart: unless-stopped + depends_on: { migrate: { condition: service_completed_successfully } } # Ollama sidecars. No host port by design: only the onboarding service talks # to them, inside the compose network. The entrypoint serves, then blocks on # `ollama pull` until the tier's model is local, then waits on serve. diff --git a/deploy/migrations/007_drift_tables.sql b/deploy/migrations/007_drift_tables.sql new file mode 100644 index 0000000..4957864 --- /dev/null +++ b/deploy/migrations/007_drift_tables.sql @@ -0,0 +1,50 @@ +-- deploy/migrations/007_drift_tables.sql — M3 drift detection (R-M3-2/3). +-- Natural PKs ONLY (no serials): the 004/005/006 sequence-grant bug class is +-- structurally impossible here. Drift writes are INSERT-only. +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='drift_role') THEN + CREATE ROLE drift_role LOGIN PASSWORD 'drift_dev'; END IF; +END $$; + +CREATE TABLE drift_windows ( + fingerprint_id TEXT NOT NULL, + rule_version INT NOT NULL, + field TEXT NOT NULL, -- OCSF path, 'unmapped.', or '__rule__' sentinel + window_start TIMESTAMPTZ NOT NULL, -- parsed_at of the window's first event (deterministic key) + window_end TIMESTAMPTZ NOT NULL, + events_count INT NOT NULL, + null_rate REAL, + match_rate REAL, -- '__rule__' rows: parsed / (parsed + parse_error) + violation_rate REAL, -- tier-1 invariant violations for this field + shape_dist JSONB, -- {"ipv4": 12, "int": 3, ...} over non-null scalar values + severity TEXT NOT NULL CHECK (severity IN ('none','minor','moderate','severe')), + action_taken TEXT CHECK (action_taken IS NULL OR action_taken IN + ('alert','field_quarantined','rule_deactivated')), + scanned_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (fingerprint_id, rule_version, field, window_start) +); +CREATE INDEX drift_windows_recent ON drift_windows (fingerprint_id, window_start DESC); + +CREATE TABLE baseline_profiles ( + fingerprint_id TEXT NOT NULL, + rule_version INT NOT NULL, + field TEXT NOT NULL, + profile JSONB NOT NULL, -- {events_count, null_rate, match_rate, shape_dist} + windows_seen INT NOT NULL, + established_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (fingerprint_id, rule_version, field) +); + +-- drift service (detection + enforcement, R-M3-3) +GRANT SELECT ON normalized_events, rules TO drift_role; +GRANT INSERT ON drift_windows, baseline_profiles TO drift_role; +-- P-2: drift_role self-reads — baseline existence checks + recent-window lookups. +GRANT SELECT ON drift_windows, baseline_profiles TO drift_role; +GRANT UPDATE (quarantined_fields) ON rules TO drift_role; +GRANT UPDATE (status, deactivated_at) ON rules TO drift_role; +GRANT INSERT ON audit_log TO drift_role; +GRANT USAGE, SELECT ON SEQUENCE audit_log_id_seq TO drift_role; + +-- gateway: human override of quarantine (audited, rules_role write path) +GRANT UPDATE (quarantined_fields) ON rules TO rules_role; +-- gateway SELECT on both new tables arrives via 003's ALTER DEFAULT PRIVILEGES. diff --git a/docs/architecture-diagram.png b/docs/architecture-diagram.png new file mode 100644 index 0000000..8a414f2 Binary files /dev/null and b/docs/architecture-diagram.png differ diff --git a/docs/architecture-diagram.svg b/docs/architecture-diagram.svg new file mode 100644 index 0000000..1457f28 --- /dev/null +++ b/docs/architecture-diagram.svg @@ -0,0 +1,213 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + ULPF — Universal Log Pre-processing Framework + lossless ingest · fingerprinting · OCSF normalization · human-gated onboarding · fail-closed drift + + + + HOT PATH — deterministic · no AI per-event + + + + + Firewall + + IDS / IPS + + Any appliance + any perimeter device + + + + + + + + + + Collector + syslog UDP/TCP · :5514 + HTTP POST · :8080 + zero parsing — wraps raw + every line → envelope → topic + + + + + + raw.logs + 6 partitions · replay-safe + Redpanda — Kafka protocol + + + + + + Pipeline workers + + ×N · consumer group + + + 1 + persist raw FIRST + immutable rows · Merkle hash chain + + + 2 + fingerprint — any format + known formats + shape-hash families + + + 3 + apply active rule → OCSF + unknown → samples · errors → DLQ + + + + + + normalized.events + keyed by fingerprint + + + + + + Drift worker + watches rule health per fingerprint + tier-1 invariants — from event 0 + tier-2 statistical vs baseline + + FAIL-CLOSED + field quarantine · rule deactivate + → re-enters onboarding + + + + quarantine field · deactivate rule + + + + raw-first persist + + + + COLD PATH — AI-assisted · human-gated · rules are data, never code + + + + unknown-format samples + + + + Onboarding service + triggers when samples pile up, no active rule + · sample split — 5 prompt / 15 held-out + · SLM drafts candidate rule + · deterministic validation gate + · candidate → pending_review + + Ollama — Qwen3 4B / 8B tiers · offline + cold path only — never per-event + fail-closed: pends on SLM outage + + + + + + + ★ + HUMAN + REVIEW + approve · edit + override + + no auto-approve + approve → rule v2 live + backlog re-parses + + + + + + + PostgreSQL — event store & control plane + least-privilege roles · pipelines are INSERT-only + + + raw_events + INSERT-only · daily partitions + + + raw_batches + per-partition hash chain + + + normalized_events + append-only · versioned + + + rules + immutable versions · 1 active + + + audit_log + every transition + + + onboarding_samples + prompt · held-out · unused + + + + current view · stats · audit + + + + + + + Gateway + Dashboard + FastAPI gateway · React UI + live feed · Review Queue + rules · drift & health · audit + self-hosted assets (air-gap) + + + + Exports + OCSF JSONL · Parquet + → SIEM · data lake + + + + Air-gap native — offline SLM (Ollama) · self-hosted UI assets · docker-compose bundle (tars) · Redpanda speaks the Kafka protocol + + + diff --git a/docs/m3-drift.md b/docs/m3-drift.md new file mode 100644 index 0000000..17226a8 --- /dev/null +++ b/docs/m3-drift.md @@ -0,0 +1,261 @@ +# M3 drift detection — ruling record + +How firmware drift on a live rule is detected, graduated, and recovered, and +why the pieces look the way they do. Migrating decisions from the M3 plan +brainstorm (2026-09-21, rulings R-M3-1..11) are recorded here because they +bind later milestones. Numbers below are measured on the live compose stack +(Task 12 exit gate) unless marked as a default. + +## R-M3-1 — the drift service reads the DB current view, not the topic + +One convergent loop (`services/drift/app.py`, cold path, Postgres only — no +Kafka consumer) polls the active rules and statelessly recomputes each +`(fingerprint_id, rule_version)`'s windows from +`normalized_events WHERE superseded_by_event_id IS NULL AND rule_version = +` (the `ne_current_idx` semantics). + +Why the DB and not `normalized.events`: + +- **Correct by construction.** The topic is at-least-once and the M2 reparse + sweep writes DB rows only — it emits no envelopes (M2 ledger R17). A topic + consumer would need dedupe machinery AND would still miss backlog + re-parses, i.e. it would be wrong in exactly the cases drift exists for. +- **Single replica needs no fencing.** Window math keyed by + `(fingerprint_id, rule_version, field, window_start)` is idempotent under + re-scan (`ON CONFLICT DO NOTHING`); a consumer group adds offsets, commits, + and rebalance races that a single cold-path reader has no use for. +- **Fail-closed posture.** A failed scan is logged and retried next poll + (`ULPF_DRIFT_POLL_S`, default 5 s); drift can never block the hot path. + +Rejected alternatives: **hybrid topic + DB** (two feeds to merge, ~1.5 tasks +more, no correctness gain at MVP scale — the DB already holds the topic's +information plus the reparse rows); **topic-only** (contradicts the R17 +ledger ruling — misses reparse rows). + +**R17 topic note:** `normalized.events` remains produced by the pipeline in +M3 exactly as in M2 — all-status envelopes keyed by `fingerprint_id`, +at-least-once, dedupable by `(raw_id, rule_version)` — but nothing consumes +it in the MVP. It is the documented future streaming path: when drift (or any +other consumer) needs multi-replica scale-out, §7.1's per-partition ownership +applies and the DB poll becomes a topic read plus the same window math. + +## R-M3-2 — stateless windows, INSERT-only + +Each scan slices the version's current view (ordered by `parsed_at`) into +windows that close on **count** (`ULPF_DRIFT_WINDOW_COUNT`, default 1000 +rows) **or time** (`ULPF_DRIFT_WINDOW_TIME_S`, default 600 s), whichever +first; the trailing partial is held back (it may still grow). `window_start` +is the `parsed_at` of the window's first row, so the primary key +`(fingerprint_id, rule_version, field, window_start)` is deterministic from +data: re-scans INSERT nothing new (`ON CONFLICT DO NOTHING`), and no cursor +table or UPDATE grant on drift tables exists. The acceptance smoke runs the +demo with `ULPF_DRIFT_WINDOW_COUNT=30` plus a time horizon no demo gap can +cross (count-close only — deterministic 30-row boundaries the smoke's tuning +replica models exactly). The brief's original 15 s time horizon was measured +unsound on the accumulating DB: it slices the days-spread M1/M2 history into +~255 one-row windows at the first scan, arming an all-zero baseline from ten +of them, after which the tier-2 null ladder deactivates the rule at any 5% +null rate — before the moderate quarantine (Assert J) can ever land +(observed live; recorded in `scripts/smoke.py`'s header). + +Windows are recomputed from the whole current view every scan — including +already-recorded ones, whose findings are re-evaluated. That is what makes +enforcement convergent after a baseline arms (the establishing scan is +tier-1-only by construction: the profile row is inserted only after that +scan's windows are recorded, so its findings predate the profile), and it is +also why a severe window is permanent for its version: recovery is a version +bump, not a wait (see the restoration note under R-M3-11). + +## R-M3-3/R-M3-11 — the grant matrix is the design; enforcement is graduated + +`deploy/migrations/007_drift_tables.sql` (natural PKs only — no serials, so +the 004/005/006 sequence-grant bug class is structurally impossible; the only +serial-backed INSERT drift makes is `audit_log`, whose sequence is granted): + +```sql +-- deploy/migrations/007_drift_tables.sql — M3 drift detection (R-M3-2/3). +-- Natural PKs ONLY (no serials): the 004/005/006 sequence-grant bug class is +-- structurally impossible here. Drift writes are INSERT-only. +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='drift_role') THEN + CREATE ROLE drift_role LOGIN PASSWORD 'drift_dev'; END IF; +END $$; + +CREATE TABLE drift_windows ( + fingerprint_id TEXT NOT NULL, + rule_version INT NOT NULL, + field TEXT NOT NULL, -- OCSF path, 'unmapped.', or '__rule__' sentinel + window_start TIMESTAMPTZ NOT NULL, -- parsed_at of the window's first event (deterministic key) + window_end TIMESTAMPTZ NOT NULL, + events_count INT NOT NULL, + null_rate REAL, + match_rate REAL, -- '__rule__' rows: parsed / (parsed + parse_error) + violation_rate REAL, -- tier-1 invariant violations for this field + shape_dist JSONB, -- {"ipv4": 12, "int": 3, ...} over non-null scalar values + severity TEXT NOT NULL CHECK (severity IN ('none','minor','moderate','severe')), + action_taken TEXT CHECK (action_taken IS NULL OR action_taken IN + ('alert','field_quarantined','rule_deactivated')), + scanned_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (fingerprint_id, rule_version, field, window_start) +); +CREATE INDEX drift_windows_recent ON drift_windows (fingerprint_id, window_start DESC); + +CREATE TABLE baseline_profiles ( + fingerprint_id TEXT NOT NULL, + rule_version INT NOT NULL, + field TEXT NOT NULL, + profile JSONB NOT NULL, -- {events_count, null_rate, match_rate, shape_dist} + windows_seen INT NOT NULL, + established_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (fingerprint_id, rule_version, field) +); + +-- drift service (detection + enforcement, R-M3-3) +GRANT SELECT ON normalized_events, rules TO drift_role; +GRANT INSERT ON drift_windows, baseline_profiles TO drift_role; +-- P-2: drift_role self-reads — baseline existence checks + recent-window lookups. +GRANT SELECT ON drift_windows, baseline_profiles TO drift_role; +GRANT UPDATE (quarantined_fields) ON rules TO drift_role; +GRANT UPDATE (status, deactivated_at) ON rules TO drift_role; +GRANT INSERT ON audit_log TO drift_role; +GRANT USAGE, SELECT ON SEQUENCE audit_log_id_seq TO drift_role; + +-- gateway: human override of quarantine (audited, rules_role write path) +GRANT UPDATE (quarantined_fields) ON rules TO rules_role; +-- gateway SELECT on both new tables arrives via 003's ALTER DEFAULT PRIVILEGES. +``` + +drift_role never INSERTs rules; rule content stays immutable for every role. +Every enforcement mutation carries its expected-state predicate in the WHERE +(`status = 'active'`, `NOT (field = ANY(quarantined_fields))`) with a +rowcount check — a no-op returns False and writes NO audit row — and the +audit INSERT is paired inside the same transaction, actor `drift`. + +Graduated enforcement per (field, window), severity = max(tier-1, tier-2 +signals), enforced worst-first with one deactivate per fingerprint per scan: + +- `minor` → alert only (the `drift_windows` row IS the alert). +- `moderate` → `field_quarantined` (idempotent append to + `rules.quarantined_fields`). +- `severe` → `rule_deactivated`; the fingerprint reverts to raw-only and + re-enters onboarding through the M2 `_READY_SQL` path (samples re-accumulate + — zero onboarding changes needed). +- Quarantined fields are still measured and still escalate: quarantine + suppresses only a redundant re-quarantine, never a finding — a quarantined + field going severe still deactivates the rule. +- `unmapped.*` rows are new-key-emergence signals only: alert-only, + `decide_action` bypassed (binding Task-4 review ruling). + +Humans override any state: `POST /api/rules/{fp}/unquarantine` (rules_role, +audited, `field_unquarantined`) and the M2 +reactivate/deactivate endpoints. Restoration note (measured live in the Task +12 smoke): a reactivated version whose current view still contains severe +windows is re-deactivated by the next scan — recovery from a real drift event +is a **version bump** (approve a successor rule; the M2 backlog sweep +supersedes the poisoned version's rows out of its drift view), which is the +same path a human edit takes. The moderate rung re-arms the same way: +un-quarantining a field whose historical window still evaluates moderate is +undone by the next scan — redundant-quarantine suppression holds only while +the field is IN `quarantined_fields`, so removal re-arms it (findings are +re-evaluated every scan, R-M3-2). An un-quarantine sticks durably only once +the offending window is superseded (the same version bump) or the version's +current view evaluates clean. + +## R-M3-4/R-M3-5 — two detection tiers; the math is pure + +**Tier 1 (from event zero, always on):** invariant-violation rates over the +stored OCSF docs — `src/dst_endpoint.ip` IPv4 shape, ports int 0..65535, +`severity_id` int 0..6, `time` ISO-parsable — mirroring the candidate gate's +`_value_sanity` semantics at drift time (deliberately not an import: +candidate-time vs drift-time). Ladder: `<0.05 none, <0.20 minor, <0.50 +moderate, else severe`. + +**Tier 2 (armed once a baseline exists per `(fingerprint, version)`, +R-M3-7):** the first `ULPF_DRIFT_BASELINE_WINDOWS` (default 10) closed +windows aggregate into `baseline_profiles` (mean null_rate, mean match_rate, +min/max events_count, shape_dist summed and normalized); the profile row is +INSERTed once when the Nth window closes and never re-arms for that version — +a version bump re-arms naturally. Signals: null-rate ratio ladder (the legacy +demo's proven 3x/10x/40x + 0.05-absolute gate), match-rate absolute-drop +ladder (0.10/0.25/0.50), JS-divergence shape ladder (0.15/0.35/0.60, +add-1-smoothed, count-scaled), per-fingerprint volume bounds (silence severe, +outside minor — the silence rung's reachability is qualified in R-M3-10). All +of it lives in `libs/ulpf-core/ulpf_core/drift.py` — +stdlib-only, zero deps, fully unit-tested (Task 3: 37 tests; the drift +service suites add 102). + +## R-M3-6 — quarantine is honored at parse time, new parses only + +`ActiveRule` carries `quarantined_fields`; the pipeline worker builds a +parse-time rule copy with the quarantined mappings removed, so the captured +key lands in `ocsf.unmapped` automatically (spec §7.2 "mapping disabled, +values preserved" — no `parse()` API change). Measured in the Task 12 smoke: +after drift quarantined `src_endpoint.ip`, subsequent parses carried the SRC +value under `ocsf.unmapped.SRC` with `src_endpoint.ip` absent; after the human +un-quarantine, the next clean parses mapped `src_endpoint.ip` again. + +Quarantine affects NEW parses only. The existing backlog is NOT auto +re-parsed — a deliberate scope line: a backlog re-parse requires a version +bump (a human edit/approve rides the M2 sweep, which re-parses stale rows and +supersedes the old version's view). Automatic backlog re-parse would need +either an onboarding-side quarantine awareness (re-parse honoring the +filter — not built, see "Deferred") or churn on every quarantine toggle. + +## R-M3-9 — the demo: deterministic escalating firmware drift + +`simulator/simulate.py --mode firmware-drift --drift-after S --duration D` +keeps the steady corpus set but corrupts syslog lines on the wire after the +deadline: a line is corrupted iff `(index * 2654435761 % 1000)/1000 < +phase`, `phase = clamp(elapsed_since_drift/span, 0, 1)`, +`span = max(duration - drift_after, 1.0)`; among corrupted lines `index % 3 +== 0` renames `SRC` -> `SRCADDR` (a null at parse time) and the rest write +`SRC=uplink-trust-0x4f` (a tier-1 IP violation) — exactly 1:2 nulls to +violations. The corruption is deterministic (the seed IS the corpus line +index; no RNG state), which is what lets the acceptance smoke replicate the +math host-side and pre-tune each run (see `tune_drift_run` in +`scripts/smoke.py`). Only line content mutates — per-corpus counts, the other +corpora, and the ` sent=` stdout contract are unchanged. + +## Audit vocabulary additions + +`action` gains `field_quarantined` and `field_unquarantined`; `rule_deactivated` +is reused with actor `drift` (enforcement) vs a human actor (gateway). The +closed set is now {`samples_split`, `candidate_created`, `candidate_failed`, +`rule_approved`, `rule_rejected`, `rule_deactivated`, `rule_reactivated`, +`field_quarantined`, `field_unquarantined`, `reparse_complete`}; entity = +fingerprint_id. Drift-side rows carry actor `drift` and detail +`{field, rule_id, version, window_start}` (quarantine) / `{rule_id, version, +window_start}` (deactivate). + +## R-M3-10 — per-SOURCE volume deferred to M4 + +Per-fingerprint volume bounds ship now (`__rule__` sentinel rows carry +events_count min/max from the baseline; the ladder maps a zero-count window +to severe — with a reachability caveat, directly below). Per-SOURCE volume +needs a `raw_events` join (`normalized_events` carries no `source_id`); +deferred to M4 as polish — the current-view granularity is what the spec's +silence/flood signal needs at MVP scale. + +Silence honesty note: the severe-on-zero rung cannot fire end-to-end as +shipped. `compute_windows` (`services/drift/windows.py`) closes only +non-empty windows, so a dead feed produces no new rows — no new windows, no +`__rule__` finding, nothing to evaluate; `volume_severity`'s +`count == 0 → severe` branch (`libs/ulpf-core/ulpf_core/drift.py`) is +defensive-only, and the drought rung fires only retroactively, once traffic +resumes and a below-bounds window closes. A wall-clock last-window-age check +is the M4 path — deliberately excluded here by R-M3-2's data-only determinism +(windows derive from the rows' own `parsed_at`, never the clock). Until M4, +the Overview page's Events / min card is the only live silence visibility. + +## Deferred, with numbers + +- **Per-source volume** — see R-M3-10 above. +- **Re-parse honoring quarantine** — the M2 sweep re-parses under the rule's + full mappings (it rebuilds the Rule without the quarantine filter), so a + quarantined field's backlog re-parse maps it again. Measured consequence + (Task 12): after K's deactivation window, the sweep's re-parsed rows + re-introduce the violations at the new version. Bounded by the same + version-bump recovery path above; not worth onboarding-side quarantine + plumbing in M3. +- **Auth** — actor is a request field defaulting to "anonymous" (MVP has no + auth); every mutation is audited regardless. diff --git a/docs/ppt-accuracy-fixes.md b/docs/ppt-accuracy-fixes.md new file mode 100644 index 0000000..07d52a9 --- /dev/null +++ b/docs/ppt-accuracy-fixes.md @@ -0,0 +1,24 @@ +# PPT accuracy fixes (spec §17 — mandated before finals; do them before this submission) + +The deck (`references/sih-ps-26156-ppt-print.pdf`) predates the M2 merge. Three +lines no longer match what shipped — suggested replacements: + +1. **Slide 3, Onboarding engine bullet** — "Local SLM (Ollama + Granite 4.1)" + → **"Local SLM (Ollama, tiered Qwen3 4B/8B — CPU-only, offline)"** + Shipped default tier is qwen3:4b (`deploy/docker-compose.yml`, + `docs/m2-slm-sanity.md`); Granite was a harness alternate, never the default. + +2. **Slide 4, Resource-light bullet** — "billions-of-events/day throughput needs + no GPU or inference" → **"hot-path throughput needs no GPU or inference; + scale-out is Kafka partitions + consumer-group workers (benchmarks in M4)"** + The spec withdrew unmeasured throughput claims (§13 benchmark honesty: + measured numbers only — nothing extrapolated). + +3. **Slide 3, Ingestion bullet** — "buffered via Kafka" → **"buffered via + Redpanda (Kafka-protocol compatible)"** — exact and preempts the judge + question. (The architecture diagram already words it this way.) + +Also worth a glance (not a mismatch, just a risk): slides 2/4 cite Matryoshka +as "UC Berkeley/Google published research" — spec §17 flags this attribution +for re-verification; slide 6 links the julien-piet/matryoshka repo. Confirm +the affiliation wording before submission. diff --git a/docs/ppt-feasibility-viability.md b/docs/ppt-feasibility-viability.md new file mode 100644 index 0000000..690884a --- /dev/null +++ b/docs/ppt-feasibility-viability.md @@ -0,0 +1,69 @@ +# PPT feasibility & industry-viability points (slide 4 replacement copy) + +Companion to `docs/ppt-accuracy-fixes.md`. Replacement text for the deck's +slide 4 ("FEASIBILITY AND VIABILITY"), reworked from idea-stage ("should +work") to prototype-stage ("does work, with numbers"). Every claim is true of +the merged repo (M2, a8f578d). The withdrawn claims stay out: no unmeasured +throughput figures, no Granite naming, no unverified research attribution. + +Usage: drop the three sections into the slide's three template fields. If the +final-round template has a dedicated "viability in the industry" field, the +middle section lifts out as-is. + +## Analysis and Feasibility of the idea + +- **Built and verified, not proposed** — the full pipeline (ingest → + fingerprint → normalize → human review → backlog re-parse) is merged and + green: 244 automated tests, 8/8 end-to-end acceptance checks including a + forced Kafka replay (0 duplicates, 0 losses) and independent re-verification + of every Merkle hash-chain link +- **Proven building blocks only** — Redpanda (Kafka protocol), PostgreSQL, + FastAPI, React, Ollama: no exotic dependencies; the whole stack demos on + one laptop via `docker compose up` +- **Deterministic hot path** — per-event cost is regex + hashing, never + inference; scale-out is Kafka partitions + worker replicas (benchmarks + measured in M4 — no extrapolated throughput claims) +- **Tamper-evidence by construction** — immutable raw store with + per-partition hash chains; the database roles themselves deny update/delete + to the pipeline + +## Viability — in the industry + +- **Attacks the costliest step in log adoption** — per-vendor parser projects + collapse into a review-and-approve cycle; analysts approve rules instead of + writing them +- **Standards-aligned, not proprietary** — normalizes to OCSF (Linux + Foundation; backed by AWS, Splunk, IBM and 30+ vendors) and exports + JSONL/Parquet → integrates with existing SIEMs and data lakes, no + rip-and-replace +- **Serves networks cloud AI cannot** — fully air-gapped operation (offline + SLM, self-hosted UI assets, container bundle) fits government, defense, + banking and OT/ICS deployments +- **Trust survives automation** — every rule versioned with provenance + (slm / slm-edited / human), every transition audited, and no code path can + activate a rule without a human action +- **Reversible adoption** — raw bytes are stored first and never mutated, so + ULPF can sit in front of any existing pipeline; removing it loses nothing + +## Potential Challenges → Strategies (paired) + +| Challenge | Strategy (already built) | +|---|---| +| SLM rule quality varies with hardware | Deterministic 8-point validation gate + human approval **bound** model quality — a weaker model costs review time, never correctness (measured and documented on the default tier) | +| OCSF coverage is a curated subset | Unmapped fields are preserved, never dropped; extending the schema is additive work | +| Vendor formats drift silently | Tier-1 invariant checks fire from event zero (no baseline needed); tier-2 statistical drift arms after baseline; enforcement is fail-closed — quarantine or deactivate, never silently wrong data | +| Low-volume sources are hard to baseline | Windows close on event-count OR time, so a 40-events/day sensor is monitored on its own clock | + +## Speaker notes (don't put on the slide) + +- The strongest single sentence if a judge asks "is this real?": *"You can + delete our consumer group mid-run and replay every message — the store + comes out identical; that check runs in our acceptance gate."* +- If asked about the SLM tier honestly: the default 4B tier on modest CPU + hardware stores zero rules unaided (documented in docs/m2-slm-sanity.md) — + which is exactly why the architecture never lets model quality touch + correctness: the gate + human review bound it, and manual authoring always + works. +- If asked about scale numbers: commit only to "partitions + consumer-group + workers; measured per-worker benchmarks are the M4 deliverable" — the spec + explicitly withdrew unmeasured throughput claims. diff --git a/libs/ulpf-core/tests/test_drift.py b/libs/ulpf-core/tests/test_drift.py new file mode 100644 index 0000000..626dd95 --- /dev/null +++ b/libs/ulpf-core/tests/test_drift.py @@ -0,0 +1,167 @@ +# libs/ulpf-core/tests/test_drift.py — the pure drift-math module, all with concrete data: +import math + +import pytest +from ulpf_core.drift import ( + SEVERITIES, + classify_value, + decide_action, + js_divergence, + match_rate_severity, + null_rate_severity, + shape_histogram, + shape_severity, + tier1_severity, + volume_severity, +) + +# --- classify_value / shape_histogram ---------------------------------------- + +def test_classify_value_classes(): + assert classify_value("198.51.100.4") == "ipv4" + assert classify_value(52114) == "int" and classify_value("52114") == "int" + assert classify_value("2026-09-19T10:00:03Z") == "timestamp" + assert classify_value("2026/09/19 10:00:03") == "timestamp" + assert classify_value("DROP IN=eth0") == "free_text" + assert classify_value(3.14) == "free_text" # no float class + +def test_classify_value_ip_must_be_dotted_quad(): + assert classify_value("1.2.3.4.5") == "free_text" # five groups + assert classify_value("1.2.3") == "free_text" + assert classify_value("999.999.999.999") == "ipv4" # shape-level: 4 digit quads + +def test_classify_value_bool_is_not_int(): + assert classify_value(True) == "free_text" # bool is an int subclass + assert classify_value("007") == "int" # digit-string, leading zeros fine + +def test_classify_value_none_free_contract(): + with pytest.raises(ValueError): # callers filter nulls first + classify_value(None) + +def test_shape_histogram_counts(): + values = ["1.2.3.4", "5", "5", "hello", 7, "2026-09-19T00:00:00Z"] + assert shape_histogram(values) == {"ipv4": 1, "int": 3, "free_text": 1, "timestamp": 1} + +def test_shape_histogram_empty(): + assert shape_histogram([]) == {} + +# --- js_divergence ------------------------------------------------------------ + +def test_js_identity_is_zero(): + assert js_divergence({"ipv4": 90, "int": 10}, {"ipv4": 90, "int": 10}) == 0.0 + +def test_js_symmetric(): + p, q = {"ipv4": 90, "int": 10}, {"ipv4": 10, "int": 90} + assert js_divergence(p, q) == pytest.approx(js_divergence(q, p)) + +def test_js_smoothing_keeps_unseen_key_finite(): + d = js_divergence({"ipv4": 100}, {"int": 100}) # disjoint supports + assert math.isfinite(d) and 0 < d < math.log(2) + +def test_js_disjoint_smoothed_value(): + # add-1 over the union: (11/12, 1/12) vs (1/12, 11/12) -> JS = 0.406311... + assert js_divergence({"a": 10}, {"b": 10}) == pytest.approx(0.4063111975, abs=1e-6) + +def test_js_grows_toward_ln2_as_counts_grow(): + small = js_divergence({"a": 10}, {"b": 10}) + big = js_divergence({"a": 10000}, {"b": 10000}) + assert big > small and big == pytest.approx(math.log(2), abs=5e-3) + +def test_js_empty_histograms(): + assert js_divergence({}, {}) == 0.0 + +# --- null-rate ladder (legacy 3x/10x/40x + 0.05 abs gate) --------------------- +# Dyadic-friendly rates so the ratios land exactly ON the boundaries under test. + +def test_null_ratio_just_below_3x_none(): + assert null_rate_severity(0.0934375, 0.03125) == "none" # 2.99x, diff 0.0622 >= 0.05 + +def test_null_ratio_exactly_3x_minor(): + assert null_rate_severity(0.09375, 0.03125) == "minor" # 3.0x exact, diff 0.0625 + +def test_null_ratio_just_below_10x_minor(): + assert null_rate_severity(0.1546875, 0.015625) == "minor" # 9.9x + +def test_null_ratio_exactly_10x_moderate(): + assert null_rate_severity(0.15625, 0.015625) == "moderate" # 10.0x exact + +def test_null_ratio_just_below_40x_moderate(): + assert null_rate_severity(0.15234375, 0.00390625) == "moderate" # 39x + +def test_null_ratio_exactly_40x_severe(): + assert null_rate_severity(0.15625, 0.00390625) == "severe" # 40.0x exact + +def test_null_abs_gate_suppresses_tiny_baseline_ratio(): + assert null_rate_severity(0.04, 0.004) == "none" # 10x ratio but diff 0.036 < 0.05 + +def test_null_abs_diff_exactly_05_not_suppressed(): + assert null_rate_severity(0.065625, 0.015625) == "minor" # diff exactly 0.05, ratio 4.2x + +def test_null_baseline_zero_current_zero_none(): + assert null_rate_severity(0.0, 0.0) == "none" # ratio defined as 1.0 + +def test_null_baseline_zero_current_positive_severe(): + assert null_rate_severity(0.3, 0.0) == "severe" # inf ratio: field went null + +def test_null_steady_state_none(): + assert null_rate_severity(0.2, 0.2) == "none" # ratio 1, diff 0 + +# --- match-rate drop ladder (<0.10 / <0.25 / <0.50) --------------------------- + +def test_match_drop_below_10pp_none(): + assert match_rate_severity(0.6875, 0.75) == "none" # drop 0.0625 + +def test_match_drop_exactly_10pp_minor(): + assert match_rate_severity(0.9, 1.0) == "minor" # nominal 0.10 (quantized to 6dp) + +def test_match_drop_just_below_25pp_minor(): + assert match_rate_severity(0.25390625, 0.5) == "minor" # drop 0.2461 + +def test_match_drop_exactly_25pp_moderate(): + assert match_rate_severity(0.25, 0.5) == "moderate" # drop 0.25 + +def test_match_drop_just_below_50pp_moderate(): + assert match_rate_severity(0.25390625, 0.75) == "moderate" # drop 0.4961 + +def test_match_drop_exactly_50pp_severe(): + assert match_rate_severity(0.25, 0.75) == "severe" # drop 0.50 + +def test_match_improvement_none(): + assert match_rate_severity(0.9, 0.5) == "none" # negative drop: rule got better + +# --- shape / tier-1 / volume ladders ------------------------------------------ + +def test_shape_severity_bands(): + assert shape_severity(0.0) == "none" and shape_severity(0.149) == "none" + assert shape_severity(0.15) == "minor" and shape_severity(0.34) == "minor" + assert shape_severity(0.35) == "moderate" and shape_severity(0.59) == "moderate" + assert shape_severity(0.60) == "severe" and shape_severity(0.9) == "severe" + +def test_tier1_severity_bands(): + assert tier1_severity(0.0) == "none" and tier1_severity(0.049) == "none" + assert tier1_severity(0.05) == "minor" and tier1_severity(0.19) == "minor" + assert tier1_severity(0.20) == "moderate" and tier1_severity(0.49) == "moderate" + assert tier1_severity(0.50) == "severe" and tier1_severity(1.0) == "severe" + +def test_volume_silence_is_severe(): + assert volume_severity(0, 50, 100) == "severe" # dead feed never looks healthy + assert volume_severity(0, 0, 100) == "severe" # even though 0 is inside bounds + +def test_volume_inside_bounds_none(): + assert volume_severity(50, 50, 100) == "none" # ON p05 + assert volume_severity(100, 50, 100) == "none" # ON p95 + assert volume_severity(75, 50, 100) == "none" + +def test_volume_outside_bounds_minor(): + assert volume_severity(49, 50, 100) == "minor" # drought + assert volume_severity(101, 50, 100) == "minor" # flood + +# --- severity -> action mapping ----------------------------------------------- + +def test_decide_action_full_mapping(): # spec §7.2 graduated enforcement + expected = [None, "alert", "field_quarantined", "rule_deactivated"] + assert [decide_action(s) for s in SEVERITIES] == expected + assert decide_action("none") is None + +def test_severities_tuple(): + assert SEVERITIES == ("none", "minor", "moderate", "severe") diff --git a/libs/ulpf-core/ulpf_core/drift.py b/libs/ulpf-core/ulpf_core/drift.py new file mode 100644 index 0000000..5558b94 --- /dev/null +++ b/libs/ulpf-core/ulpf_core/drift.py @@ -0,0 +1,147 @@ +"""Pure drift-detection math for the M3 drift service (spec §7.1, R-M3-5). + +Everything here is deterministic and dependency-free — stdlib only, no DB, +no model calls — so Tasks 4/5 (window loop, enforcement) can import it +anywhere. Scalar value classification, shape histograms, smoothed JS +divergence, the severity ladders and the severity-to-action mapping. + +Deliberately standalone: the class vocabulary is scalar-level +("ipv4"/"int"/"timestamp"/"free_text"), independent of fingerprint's +token-level ``_classify`` shapes, and must not import it (R-M3-5). +""" + +import math +import re + +SEVERITIES = ("none", "minor", "moderate", "severe") + +# Same dotted-quad shape as validation's IP sanity check (four digit quads, +# no 0-255 range enforcement — this is a class test, not an address parser). +_IPV4_RE = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") +# ISO-8601-ish: a value that OPENS with a YYYY-MM-DD / YYYY/MM/DD date. +_TS_PREFIX_RE = re.compile(r"^\d{4}[-/]\d{2}[-/]\d{2}") +_DIGITS_RE = re.compile(r"^\d+$") + + +def classify_value(v) -> str: + """Scalar class of a mapped value: "ipv4" | "int" | "timestamp" | + "free_text". None-free contract: callers filter nulls before binning + (null-rate is its own tier-2 signal), so None is rejected loudly here + instead of silently classifying as "None" free text.""" + if v is None: + raise ValueError("classify_value is None-free: filter nulls at the caller") + if isinstance(v, str): + if _IPV4_RE.match(v): + return "ipv4" + if _TS_PREFIX_RE.match(v): + return "timestamp" + if _DIGITS_RE.match(v): + return "int" + elif isinstance(v, int) and not isinstance(v, bool): + # bool is an int subclass — a True flag is not an int value + return "int" + return "free_text" + + +def shape_histogram(values: list) -> dict[str, int]: + """Count of values per classify_value class — the tier-2 shape_dist.""" + hist: dict[str, int] = {} + for v in values: + cls = classify_value(v) + hist[cls] = hist.get(cls, 0) + 1 + return hist + + +def js_divergence(p: dict, q: dict) -> float: + """Symmetric Jensen-Shannon divergence (natural log) over two + histograms, add-1 smoothed across the key union so a key unseen on + one side contributes mass instead of a divide-by-zero infinity. + Identical histograms give exactly 0.0; disjoint supports approach + ln 2 (~0.693) as counts grow.""" + keys = set(p) | set(q) + if not keys: + return 0.0 + smoothed_p = {k: p.get(k, 0) + 1 for k in keys} + smoothed_q = {k: q.get(k, 0) + 1 for k in keys} + total_p, total_q = sum(smoothed_p.values()), sum(smoothed_q.values()) + divergence = 0.0 + for k in keys: + pi = smoothed_p[k] / total_p + qi = smoothed_q[k] / total_q + m = (pi + qi) / 2 + divergence += 0.5 * pi * math.log(pi / m) + 0.5 * qi * math.log(qi / m) + return divergence + + +def null_rate_severity(current: float, baseline: float) -> str: + """The legacy demo's proven ladder (legacy-demo/drift_monitor/monitor.py + classify_severity, verbatim semantics): ratio < 3 or absolute diff + < 0.05 -> none; ratio < 10 -> minor; ratio < 40 -> moderate; else + severe. Baseline 0: ratio is inf when current > 0 (never-null field + starting to null is infinitely worse) else 1.0.""" + current = round(current, 6) + baseline = round(baseline, 6) + if baseline > 0: + ratio = current / baseline + else: + ratio = float("inf") if current > 0 else 1.0 + difference = round(current - baseline, 6) + if ratio < 3 or difference < 0.05: + return "none" + if ratio < 10: + return "minor" + if ratio < 40: + return "moderate" + return "severe" + + +def _band_severity(value: float, minor: float, moderate: float, severe: float) -> str: + """Shared ascending-threshold ladder: below `minor` -> none, below + `moderate` -> minor, below `severe` -> moderate, else severe. The + match/shape/tier-1 ladders differ only in their threshold triples.""" + if value < minor: + return "none" + if value < moderate: + return "minor" + if value < severe: + return "moderate" + return "severe" + + +def match_rate_severity(current: float, baseline: float) -> str: + """Absolute match-rate drop ladder: <0.10 none, <0.25 minor, <0.50 + moderate, else severe. The drop is quantized to 6 decimals like the + null ladder so a nominal 1.00 -> 0.90 window sits exactly ON the 0.10 + boundary (1.0 - 0.9 is 0.0999... in binary), not just under it.""" + return _band_severity(round(baseline - current, 6), 0.10, 0.25, 0.50) + + +def shape_severity(divergence: float) -> str: + """JS-divergence ladder: <0.15 none, <0.35 minor, <0.60 moderate, + else severe.""" + return _band_severity(divergence, 0.15, 0.35, 0.60) + + +def tier1_severity(violation_rate: float) -> str: + """Tier-1 invariant-violation rate ladder: <0.05 none, <0.20 minor, + <0.50 moderate, else severe.""" + return _band_severity(violation_rate, 0.05, 0.20, 0.50) + + +def volume_severity(count: int, p05: float, p95: float) -> str: + """Per-source event volume against baseline percentile bounds: + silence (0 events) is severe — a dead feed must never look healthy — + inside [p05, p95] is none, anything else is minor (drought or flood).""" + if count == 0: + return "severe" + if p05 <= count <= p95: + return "none" + return "minor" + + +def decide_action(severity: str) -> str | None: + """Severity -> graduated enforcement step (spec §7.2): minor alerts, + moderate quarantines the field (rule stays live for healthy fields), + severe deactivates the rule; none needs no action.""" + return {"none": None, "minor": "alert", "moderate": "field_quarantined", + "severe": "rule_deactivated"}[severity] diff --git a/pyproject.toml b/pyproject.toml index d12a5a0..1138062 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.pytest.ini_options] testpaths = ["libs", "services", "simulator", "bench"] -pythonpath = [".", "services/collector", "services/pipeline", "services/gateway", "simulator", "services/onboarding"] +pythonpath = [".", "services/collector", "services/pipeline", "services/gateway", "simulator", "services/onboarding", "services/drift"] [tool.ruff] line-length = 100 diff --git a/scripts/smoke.py b/scripts/smoke.py index 2039ae1..a246487 100755 --- a/scripts/smoke.py +++ b/scripts/smoke.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""ULPF acceptance smoke (M1 Task 12 + M2 Task 12, spec section 15) — the -milestone exit gate. +"""ULPF acceptance smoke (M1 Task 12 + M2 Task 12 + M3 Task 12, spec §15) — +the milestone exit gate. End-to-end against the live compose stack: drives the simulator over the golden corpora, then verifies lossless ingest, the parsed/unparsed split, raw @@ -14,7 +14,54 @@ docs/m2-slm-sanity.md), G does the same deterministically for the zenwall corpus via /api/rules/manual, and H re-runs the replay dance with active rules, proving raw/normalized/stats/sample-count idempotency including the -onboarding sample dedup. +onboarding sample dedup. The M3 asserts (I-L) run FIRST, before the A-H block, +and drive the drift engine end to end on the live seeded syslog rule: I proves +tier-1 fires (a fresh drift window with severity >= minor and a nonzero +violation_rate), J that moderate drift quarantines src_endpoint.ip while the +SRC value survives under ocsf.unmapped.SRC (values preserved, mapping +disabled), K that severe drift deactivates the rule (fingerprint re-enters +onboarding — sample_count grows) and a human restores it via reactivate, and +L that a human un-quarantine re-enables the mapping for new parses. + +M3 demo env (exported below BEFORE compose up, per the Task 12 brief): the +drift service gets ULPF_DRIFT_WINDOW_COUNT=30 plus a time horizon no demo +gap can cross, so windows close on COUNT ONLY — deterministic 30-row +boundaries (the brief's 15 s time-close is unsound on the accumulating DB: +it slices the days-spread M1/M2 history into one-row windows and arms a +degenerate all-zero baseline before the demo even starts; see the constant +block below). The pipeline additionally gets ULPF_SAMPLE_CAPTURE_CAP=2000: +the accumulating multi-run DB saturates the 200/fingerprint sample cap, +after which a drift-deactivated fingerprint cannot accumulate onboarding +samples and K's re-entry signal (sample_count grows) is unobservable +(observed live in the Task 12 gate). + +M3 drift-run tuning (verified live, Task 12): the simulator's firmware-drift +corruption is fully deterministic (selector = (index * 2654435761 % 1000) / +1000 vs an escalating phase), so the smoke REPLICATES that math host-side +(tune_drift_run) and plans the drift simulator run(s) such that the J-window +— the 30-row drift window the quarantine decision rides on — carries a +tier-1 violation_rate in [0.27, 0.43]: moderate (>= 0.20 -> field quarantine, +Assert J) with margin on both sides (severe 0.50 would deactivate first and +skip the quarantine entirely). The quarantine lands at the FIRST scan that +sees that window; when that scan is also the baseline-establishing one the +decision is tier-1-only by construction (the profile arms only after the +scan's windows are recorded), which is what keeps the tier-2 null ladder from +ever preempting J. After the quarantine, every further syslog parse is null +at src_endpoint.ip (mapping disabled), so the next full window hits null_rate +1.0 against the fresh baseline — ratio >= 40x, severe -> rule_deactivated +(Assert K). With a view offset > 0 the plan uses a short PRE-RUN to complete +the establishing window before the main run supplies the J-window's rows. + +M3 state restoration (not asserts): severe windows are permanent rows of the +version's current view, and the drift loop re-evaluates the whole view every +scan — so after K's deactivation the reactivated rule would be re-deactivated +by the next scan. The smoke therefore (a) stops the drift container between +K's restore and the end of the M3 block, and (b) finishes with a human +version-bump dance (manual candidate without the SRC mapping -> approve -> the +M2 backlog sweep supersedes every poisoned v1 row at the new version -> +deactivate the new version -> reactivate v1, whose current view is now empty) +before restarting drift. This restores Assert B's seeded-fp precondition for +re-runs on the same accumulating DB (12/12 twice consecutive). Host requirements (Python 3.13 tested): psycopg[binary] plus the two libs importable (pip install -e libs/ulpf-core -e libs/ocsf-schema — the smoke @@ -30,6 +77,9 @@ Usage: python scripts/smoke.py (from the repo root) ULPF_SMOKE_SLM_WAIT_S=300 (seconds to wait for an SLM candidate before the hand-authored fallback) + ULPF_SMOKE_M3_ONLY=1 (DEV: run only the M3 I-L block, skip + the A-H block — the exit gate always + runs the full 12) Exit: 0 all checks PASS, 1 any FAIL. """ @@ -37,6 +87,7 @@ import hashlib import json +import math import os import random import re @@ -45,6 +96,7 @@ import time import urllib.error import urllib.request +from datetime import UTC, datetime from pathlib import Path import psycopg @@ -89,6 +141,7 @@ SAMPLES_WAIT_TIMEOUT_S = 60 # onboarding sample capture for a posted corpus REPARSE_TIMEOUT_S = 120 # approve -> reparse_complete audit row (R-T12-pre-d) AUDIT_PAGE = 200 # /api/audit page size (>= any fingerprint's M2-run trail) +M3_ONLY = os.environ.get("ULPF_SMOKE_M3_ONLY", "") == "1" # dev affordance COLLECTOR = "http://localhost:8080" NEWAPP_SOURCE = "newapp01" @@ -119,6 +172,71 @@ (("rule", "message"),), ) +# --- M3 (Task 12 asserts I-L): the firmware-drift demo on the seeded syslog rule -- + +SYSLOG_FP = "syslog" +QUARANTINE_FIELD = "src_endpoint.ip" # the syslog rule's SRC mapping (seed 001) + +# Demo drift knobs exported before compose up (Task 12 brief Step 1): windows +# close every 30 rows so the demo fits in minutes. The brief pinned +# WINDOW_TIME_S=15 as well, but on the ACCUMULATING DB that is unsound: the +# M1/M2-era rows are spread across days, and a 15 s time-close slices them +# into ~255 one-row windows at the very first scan — arming the syslog +# baseline from ten degenerate 1-row windows (null_rate 0.0), after which the +# tier-2 null ladder fires severe at ANY 5% null rate and deactivates the +# rule before the moderate quarantine (Assert J) can ever land (observed +# live). The smoke therefore pins a time horizon no demo gap can cross and +# lets windows close on COUNT only — deterministic 30-row boundaries the +# tuning math below models exactly. +DRIFT_WINDOW_COUNT = "30" +DRIFT_WINDOW_TIME_S = "1000000000" # ~31.7 years: count-close only + +# The drift-run tuning targets the FIRST FULLY-NEW 30-row window of the run +# (run rows offset+1 .. offset+30, where offset = current v1 view % 30) at a +# tier-1 violation_rate in [MIN, MAX] — comfortably moderate (> 0.20), safely +# below severe (0.50), with margin for +/-2 lines of send-time jitter. +J_WINDOW_MIN_VIOLATION = 0.27 +J_WINDOW_MAX_VIOLATION = 0.43 +J_WINDOW_TARGET_VIOLATION = 0.35 +W_OLD_MAX_VIOLATION = 0.18 # the partial-completing window must stay minor + +DRIFT_I_TIMEOUT_S = 120.0 # I: metrics row appears (tier-1 fires) +DRIFT_J_TIMEOUT_S = 120.0 # J: quarantine lands + a post-quarantine parsed event +DRIFT_K_TIMEOUT_S = 120.0 # K: deactivate + sample growth +DRIFT_L_TIMEOUT_S = 90.0 # L: un-quarantine + a clean mapped event +RULE_REFRESH_MARGIN_S = 6.0 # pipeline reloads active rules every batch (<= 2 s) + +# K-maker: post-quarantine syslog parses are ALL null at src_endpoint.ip (the +# mapping is disabled), so any full 30-row window of them hits null_rate 1.0 +# vs the ~0.017 baseline — ratio >= 40x, severe -> deactivate. Corruption +# content is irrelevant here (clean lines are nulls too once quarantined); the +# firmware-drift mode is used anyway to keep the demo narrative honest. +K_MAKER = {"eps": "25", "drift_after": "1", "duration": "15"} +# Probe/flush: brief steady runs (clean lines only). The probe supplies the +# post-quarantine parsed events J inspects; the flush proves L's mapping came +# back once clean lines flow after the human un-quarantine. +PROBE_RUN = {"eps": "4", "duration": "6"} +FLUSH_RUN = {"eps": "25", "duration": "8"} + +# Restoration dance rule (NOT an assert input): the seed syslog pattern with +# every mapping EXCEPT SRC — sane on any corpus line (corruption only touches +# SRC), so the candidate gate passes even against corrupted onboarding +# samples, and the M2 backlog sweep re-parses the poisoned v1 rows under a +# version that does not even map src_endpoint.ip (no violations recorded). +SYSLOG_NO_SRC_RULE = ( + r"^.*?IPTABLES-\w+:\s*(?P.*)$", + (("SPT", "src_endpoint.port"), ("DST", "dst_endpoint.ip"), + ("DPT", "dst_endpoint.port")), +) +SYSLOG_CORPUS = ROOT / "simulator" / "data" / "raw_logs_syslog.txt" + +# The simulator's deterministic corruption profile, replicated host-side for +# tuning (T11 constants; see simulator/simulate.py corrupt_syslog). The +# selector is a permutation of [0, 1) over any 1000 consecutive indices, so +# `selector < phase` corrupts exactly a `phase` fraction in expectation. +DRIFT_HASH_MULT = 2654435761 +DRIFT_HASH_MOD = 1000 + _SIM_LINE = re.compile(r"^(\w+) sent=(\d+)$", re.MULTILINE) _LAG_LINE = re.compile(r"^TOTAL-LAG\s+(\S+)\s*$", re.MULTILINE) @@ -282,13 +400,32 @@ def step1_up() -> None: sys.exit("FATAL: stack not ready (gateway/web/rules API/kafka group 'pipeline').") -def run_simulator() -> dict[str, int]: - print(f"== Step 2: simulator --eps {SIM_EPS} --duration {SIM_DURATION} ==") +def run_simulator(*, mode: str = "steady", eps: str | None = None, + duration: str | None = None, drift_after: str | None = None, + loop: bool = False, label: str = "Step 2") -> dict[str, int]: + """`docker compose run` the simulator; defaults reproduce the M2 steady + call byte-for-byte. M3 passes mode/eps/duration/drift_after/loop for the + firmware-drift runs and the brief steady probes. Returns the parsed + stdout contract dict ({corpus: sent, total: N}).""" + eps = SIM_EPS if eps is None else eps + duration = SIM_DURATION if duration is None else duration + args = ["--mode", mode, "--eps", eps, "--duration", duration] + if drift_after is not None: + args += ["--drift-after", drift_after] + if loop: + args.append("--loop") # M3 only: repeat corpora so one run yields 30+ syslog rows + print(f"== {label}: simulator --mode {mode} --eps {eps} --duration {duration}" + + (f" --drift-after {drift_after}" if drift_after is not None else "") + + (" --loop" if loop else "") + " ==") # --profile sim BEFORE run: the simulator service is profile-gated # (docker-compose.yml), and `docker compose run` refuses a gated service - # whose profile is not active (R-T12-pre-b). - out = compose("--profile", "sim", "run", "--rm", "-T", "simulator", - "--eps", SIM_EPS, "--duration", SIM_DURATION, timeout=SIM_TIMEOUT_S) + # whose profile is not active (R-T12-pre-b). --build: profile-gated + # services are NOT built by `up --build` (their profile is inactive + # there), so without this the run would use whatever image happened to + # be built last — observed live in M3 Task 12 as an M2-era image without + # firmware-drift mode. Cached layers make the rebuild cheap. + out = compose("--profile", "sim", "run", "--rm", "-T", "--build", "simulator", + *args, timeout=SIM_TIMEOUT_S) sent = {name: int(n) for name, n in _SIM_LINE.findall(out.stdout)} if out.returncode != 0 or "total" not in sent: print("stdout:", out.stdout[-2000:]) @@ -298,6 +435,14 @@ def run_simulator() -> dict[str, int]: missing = {"acmegw", "cef", "json", "syslog"} - set(sent) if missing or corpora_total != sent["total"]: sys.exit(f"FATAL: simulator stdout contract broken: {sent} (missing={missing})") + # Deterministic delivery (live Task 12 finding): the collector's producer + # rides librdkafka defaults, and under this box's load its queue delayed + # messages by many minutes (and retry-duplicated corpora) — breaking K's + # sample poll and G/H's deltas on otherwise-green runs. A graceful + # collector restart runs the app's shutdown flush (uvicorn SIGTERM -> + # lifespan -> producer.flush), draining the queue in seconds. The + # collector is a stateless forwarder; the restart is benign. + compose("restart", "collector", timeout=120) return sent @@ -928,6 +1073,544 @@ def assert_h(fp: str, zfp: str) -> None: + ("" if not problems else "; PROBLEMS: " + "; ".join(problems))) +# --- M3: drift demo asserts I-L (Task 12 Steps 2-5) ------------------------------- + +_SEVERITY_RANK = {name: rank for rank, name in + enumerate(("none", "minor", "moderate", "severe"))} + + +def _parse_ts(value) -> datetime: + """API timestamp (ISO-8601 string or datetime) -> aware UTC datetime.""" + if isinstance(value, datetime): + return value if value.tzinfo else value.replace(tzinfo=UTC) + if not value: + return datetime.min.replace(tzinfo=UTC) + parsed = datetime.fromisoformat(str(value)) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + + +def _selector(index: int) -> float: + """simulator.corrupt_syslog's deterministic per-line corruption selector + (host-side replica for run tuning — the seed IS the corpus line index).""" + return (index * DRIFT_HASH_MULT % DRIFT_HASH_MOD) / DRIFT_HASH_MOD + + +def _row_outcome(k: int, rate: float, drift_after: str, duration: str, + corpus_lines: int) -> tuple[bool, bool]: + """(violates, nulls) for the k-th syslog send (1-based) of ONE simulator + run at `rate` syslog lines/s — replica of simulator.corrupt_syslog: send + time t = k/rate (one syslog line per round-robin pass), phase ramping + from drift_after over the escalation span, corrupted iff + selector((k-1) % corpus_lines) < phase, and among the corrupted 1 in 3 + (by corpus index) renames SRC->SRCADDR (a null) while the rest write the + invalid SRC value (a tier-1 violation).""" + t = k / rate + a = float(drift_after) + if t <= a: + return False, False + span = max(float(duration) - a, 1.0) + phase = min(max((t - a) / span, 0.0), 1.0) + index = (k - 1) % corpus_lines + if _selector(index) < phase: + return index % 3 != 0, index % 3 == 0 + return False, False + + +def _window_counts(segments, corpus_lines: int) -> tuple[int, int]: + """(violations, nulls) over one 30-row window assembled from consecutive + pieces of one or two simulator runs. Each segment is (first_k, n_rows, + eps, drift_after, duration): sends first_k..first_k+n_rows-1 of a run + with those parameters (send numbering restarts per run — each + `compose run` is a fresh process with fresh cursors).""" + viol = nulls = 0 + for first_k, n, eps, drift_after, duration in segments: + rate = float(eps) / 4.0 + for k in range(first_k, first_k + n): + v, nu = _row_outcome(k, rate, drift_after, duration, corpus_lines) + viol += int(v) + nulls += int(nu) + return viol, nulls + + +def tune_drift_run(offset: int, corpus_lines: int) -> dict: + """Plan the firmware-drift simulator run(s) for THIS run's view offset. + + The drift service closes a window every 30 current-view rows of the + ACTIVE version. offset = view%30 decides the shape: with offset == 0 the + run's first 30 rows are themselves the next closed window — which is also + the ESTABLISHING 10th window (the DB history has exactly 9 windows, a + verified M3 starting condition; once a baseline exists this window is + simply the next one and the same math holds, see below). With offset > 0 + the run's first E = 30-offset rows complete the older partial (the + establishing window), and the J-window is the NEXT one. + + The quarantine (J) must land on a window whose window_start is INSIDE + this run (I's delta filter and J's poll both key off fresh rows): with + offset == 0 the run's first 30 rows are that window. With offset > 0 the + run's first E = 30-offset rows complete the OLDER partial — that window + STARTS on a prior-run row, so it must stay MINOR (the quarantine must not + land there: its metrics row is invisible to I's window_start filter and + its scan races the null-severe deactivate) — and the NEXT window, built + from the main run's fresh rows, is the J-window. The plan guarantees: + + - the J-window's deterministic violation_rate lands in + [J_WINDOW_MIN, J_WINDOW_MAX] — moderate with margin on both sides, + robust to +/-2 lines of send-time jitter; + - the establishing window (offset > 0) stays minor (< W_OLD_MAX + violations), so the moderate decision — and I's fresh metrics row — + both arrive at the J-window's scan, before any severe window. + + Returns {"runs": [params...] to execute in order, "j_violation": float, + "j_nulls": int, "establishing_violation": float}; params dicts are + run_simulator kwargs (mode/loop added by the caller). + """ + def rate_of(eps): + return float(eps) / 4.0 + + space = [] + if offset == 0: + # One run: its rows 1..30 are the J-window (and the establishing one). + for eps in ("6", "8", "10"): + rate = rate_of(eps) + for drift_after in ("0", "1", "2"): + duration = 30 / rate + 1.0 + while duration <= 30 / rate + 10.0: + d = f"{duration:.1f}" + viol, nulls = _window_counts( + [(1, 30, eps, drift_after, d)], corpus_lines) + space.append({"runs": [{"eps": eps, "drift_after": drift_after, + "duration": d}], + "j_violation": viol / 30, "j_nulls": nulls, + "est_violation": viol / 30, "est_nulls": nulls, + "segments": [(1, 30, eps, drift_after, d)]}) + duration += 0.5 + else: + est_rows = 30 - offset + # Two runs: a short PRE-RUN completes the establishing window (kept + # minor); the MAIN run (fresh cursors) supplies the J-window's bulk. + # The pre-run may overflow by 0-2 rows into the J-window's head. + for pre_eps in ("4", "6", "8", "10"): + pre_rate = rate_of(pre_eps) + for pre_a in ("0", "1", "2", "3"): + pre_d = est_rows / pre_rate + 0.1 + while pre_d <= est_rows / pre_rate + 0.9: + pd = f"{pre_d:.1f}" + pre_rows = min(max(math.ceil(pre_rate * float(pd)), est_rows), + est_rows + 2) + est_viol, est_nulls = _window_counts( + [(1, est_rows, pre_eps, pre_a, pd)], corpus_lines) + overflow = pre_rows - est_rows + if est_viol >= W_OLD_MAX_VIOLATION * 30: + pre_d += 0.2 + continue + for eps in ("6", "8", "10"): + rate = rate_of(eps) + for drift_after in ("0", "1", "2"): + main_rows = 30 - overflow + duration = main_rows / rate + 1.0 + while duration <= main_rows / rate + 8.0: + d = f"{duration:.1f}" + segments = [] + if overflow: + segments.append((est_rows + 1, overflow, + pre_eps, pre_a, pd)) + segments.append((1, main_rows, eps, drift_after, d)) + viol, nulls = _window_counts(segments, corpus_lines) + space.append({ + "runs": [{"eps": pre_eps, "drift_after": pre_a, + "duration": pd}, + {"eps": eps, "drift_after": drift_after, + "duration": d}], + "j_violation": viol / 30, "j_nulls": nulls, + "est_violation": est_viol / 30, + "est_nulls": est_nulls, "segments": segments}) + duration += 0.5 + pre_d += 0.2 + tiers = [ + lambda c: J_WINDOW_MIN_VIOLATION <= c["j_violation"] <= J_WINDOW_MAX_VIOLATION, + lambda c: 0.22 <= c["j_violation"] <= 0.48, + lambda c: 0.20 <= c["j_violation"] < 0.50, + ] + if not space: + # No pre-run kept the establishing window minor — fall back to any + # pre-run params (the quarantine may land on the old-started window; + # I/J then depend on the main run's own windows). Loud, never silent. + print(" WARNING: no pre-run kept the establishing window minor; " + "falling back to untuned pre-run") + pre_eps, pre_a = "6", "2" + pd = f"{est_rows / rate_of(pre_eps) + 0.5:.1f}" + for eps in ("6", "8", "10"): + rate = rate_of(eps) + for drift_after in ("0", "1", "2"): + d = f"{30 / rate + 5.0:.1f}" + viol, nulls = _window_counts( + [(1, 30, eps, drift_after, d)], corpus_lines) + space.append({"runs": [{"eps": pre_eps, "drift_after": pre_a, + "duration": pd}, + {"eps": eps, "drift_after": drift_after, + "duration": d}], + "j_violation": viol / 30, "j_nulls": nulls, + "est_violation": 1.0, "est_nulls": 0, + "segments": [(1, 30, eps, drift_after, d)]}) + chosen = None + for tier_no, tier in enumerate(tiers): + pool = [c for c in space if tier(c)] + if pool: + chosen = min(pool, key=lambda c: abs(c["j_violation"] - J_WINDOW_TARGET_VIOLATION)) + if tier_no > 0: + print(f" WARNING: drift-run tuning relaxed to tier {tier_no + 1} " + f"(J-window violation={chosen['j_violation']:.3f})") + break + if chosen is None: + chosen = min(space, key=lambda c: abs(c["j_violation"] - J_WINDOW_TARGET_VIOLATION)) + print(f" WARNING: drift-run tuning fell back to closest " + f"violation_rate={chosen['j_violation']:.3f} (no candidate met the bands)") + return chosen + + +def active_rule(fp: str) -> dict | None: + status, body = http_json(f"{GATEWAY}/api/rules?status=active") + if status != 200 or not isinstance(body, dict): + return None + for rule in body.get("rules", []): + if rule.get("fingerprint_id") == fp: + return rule + return None + + +def syslog_view_count(version) -> int: + """Rows of the ACTIVE version's current view — the drift service's window + input; offset = count % 30 decides where this run's rows slice.""" + with psycopg.connect(DSN) as conn, conn.cursor() as cur: + return q1(cur, "SELECT count(*) FROM normalized_events " + "WHERE fingerprint_id=%s AND rule_version=%s " + "AND superseded_by_event_id IS NULL", (SYSLOG_FP, version)) + + +def audit_after(fp: str, action: str, *, actor: str | None = None, + after: datetime | None = None, want: dict | None = None) -> dict | None: + """Newest matching audit row, or None. `after` bounds ts strictly (delta + base: prior-run rows excluded); `want` matches detail key-values.""" + status, body = http_json(f"{GATEWAY}/api/audit?fingerprint={fp}&limit={AUDIT_PAGE}") + if status != 200 or not isinstance(body, dict): + return None + for row in body.get("audit", []): + detail = row.get("detail") or {} + if row.get("action") != action: + continue + if actor is not None and row.get("actor") != actor: + continue + if want and not all(detail.get(k) == v for k, v in want.items()): + continue + if after is not None and _parse_ts(row.get("ts")) <= after: + continue + return row + return None + + +def wait_parsed_event(after_ts: datetime, predicate, timeout_s: float) -> dict | None: + """Poll the newest 50 parsed syslog events until one satisfies + predicate(event, after_ts); None on timeout.""" + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + status, body = http_json( + f"{GATEWAY}/api/events?fingerprint={SYSLOG_FP}&status=parsed&limit=50") + if status == 200 and isinstance(body, dict): + for ev in body.get("events") or []: + try: + if predicate(ev, after_ts): + return ev + except (KeyError, TypeError, ValueError): + continue + time.sleep(POLL_INTERVAL_S) + return None + + +def _j_event_ok(ev: dict, after_ts: datetime) -> bool: + """J's post-quarantine parse: the SRC value preserved under + ocsf.unmapped.SRC, src_endpoint.ip absent (mapping disabled).""" + ocsf = ev.get("ocsf") or {} + unmapped = ocsf.get("unmapped") or {} + src = ocsf.get("src_endpoint") or {} + return (_parse_ts(ev.get("parsed_at")) > after_ts + and unmapped.get("SRC") is not None + and "ip" not in src) + + +def _l_event_ok(ev: dict, after_ts: datetime) -> bool: + """L's post-un-quarantine parse: src_endpoint.ip mapped again, SRC no + longer dangling in unmapped.""" + ocsf = ev.get("ocsf") or {} + unmapped = ocsf.get("unmapped") or {} + src = ocsf.get("src_endpoint") or {} + return (_parse_ts(ev.get("parsed_at")) > after_ts + and isinstance(src.get("ip"), str) and src["ip"] != "" + and "SRC" not in unmapped) + + +def assert_i_tier1(run_start: datetime, plan: dict) -> None: + print("== Assert I: tier-1 fires (fresh syslog drift window with violations) ==") + sent_total = 0 + for i, run in enumerate(plan["runs"], 1): + sent = run_simulator(mode="firmware-drift", eps=run["eps"], + duration=run["duration"], drift_after=run["drift_after"], + loop=True, label=f"I drift run {i}/{len(plan['runs'])}") + sent_total += sent.get("syslog", 0) + deadline = time.monotonic() + DRIFT_I_TIMEOUT_S + hit, last = None, "no metrics response yet" + while time.monotonic() < deadline and hit is None: + status, body = http_json(f"{GATEWAY}/api/drift/metrics") + if status == 200 and isinstance(body, dict): + rows = body.get("metrics") or [] + last = f"{len(rows)} metric row(s), latest syslog window " \ + f"{next((r['window_start'] for r in rows if r['fingerprint_id'] == SYSLOG_FP), 'none')}" + for row in rows: + if (row.get("fingerprint_id") == SYSLOG_FP + and _SEVERITY_RANK.get(row.get("severity"), -1) >= _SEVERITY_RANK["minor"] + and (row.get("violation_rate") or 0) > 0 + and _parse_ts(row.get("window_start")) > run_start): + hit = row + break + if hit is None: + time.sleep(POLL_INTERVAL_S) + if hit is None: + check("I tier-1 fires", False, + f"no syslog metrics row with severity>=minor and violation_rate>0 " + f"within {DRIFT_I_TIMEOUT_S:.0f}s (last poll: {last})") + return + runs_desc = "; ".join(f"eps={r['eps']} drift-after={r['drift_after']} " + f"duration={r['duration']}s" for r in plan["runs"]) + check("I tier-1 fires", True, + f"syslog field {hit['field']!r} window {hit['window_start']}: " + f"severity={hit['severity']} violation_rate={hit['violation_rate']:.3f} " + f"null_rate={(hit['null_rate'] or 0):.3f} events_count={hit['events_count']} " + f"(tuned drift run(s): {runs_desc}; {sent_total} syslog lines sent; " + f"predicted J-window violation {plan['j_violation']:.3f})") + + +def _wait_rule_quarantined(timeout_s: float) -> dict | None: + """Poll GET /api/rules/syslog until the ACTIVE version's quarantined_fields + contains the SRC mapping's target.""" + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + status, body = http_json(f"{GATEWAY}/api/rules/{SYSLOG_FP}") + if status == 200 and isinstance(body, dict): + for rule in body.get("rules") or []: + if rule.get("status") == "active": + if QUARANTINE_FIELD in (rule.get("quarantined_fields") or []): + return rule + break # active version exists, field not quarantined yet + time.sleep(POLL_INTERVAL_S) + return None + + +def assert_j_quarantine(run_start: datetime) -> None: + print("== Assert J: moderate drift quarantines src_endpoint.ip, value preserved ==") + quarantined = _wait_rule_quarantined(DRIFT_J_TIMEOUT_S) + if quarantined is None: + check("J quarantine + values preserved", False, + f"active syslog rule never listed {QUARANTINE_FIELD} in " + f"quarantined_fields within {DRIFT_J_TIMEOUT_S:.0f}s") + return + audit = audit_after(SYSLOG_FP, "field_quarantined", actor="drift", after=run_start) + if audit is None: + check("J quarantine + values preserved", False, + f"quarantined_fields lists {QUARANTINE_FIELD} but no fresh " + f"field_quarantined audit row with actor=drift") + return + quarantine_ts = _parse_ts(audit.get("ts")) + # The pipeline honors quarantine on the next rule refresh (<= 2 s); the + # margin plus a short steady probe supplies post-quarantine parses. + time.sleep(RULE_REFRESH_MARGIN_S) + run_simulator(label="J probe run (steady)", **PROBE_RUN) + event = wait_parsed_event(quarantine_ts, _j_event_ok, DRIFT_J_TIMEOUT_S) + if event is None: + check("J quarantine + values preserved", False, + f"no parsed syslog event after the quarantine audit ({audit.get('ts')}) " + f"carries unmapped.SRC with src_endpoint.ip absent") + return + ocsf = event["ocsf"] + check("J quarantine + values preserved", True, + f"quarantined_fields=['{QUARANTINE_FIELD}'] on v{quarantined['version']} " + f"(audit field_quarantined actor=drift at {audit.get('ts')}); parsed event " + f"{event['event_id']} @ {event['parsed_at']} preserves SRC value " + f"{ocsf['unmapped']['SRC']!r} under ocsf.unmapped.SRC with src_endpoint.ip absent") + + +def assert_k_severe(run_start: datetime, active: dict, pre_samples: int) -> None: + print("== Assert K: severe -> rule deactivated -> re-onboard -> human restore ==") + problems: list[str] = [] + parts: list[str] = [] + run_simulator(mode="firmware-drift", label="K-maker run (post-quarantine null flood)", + loop=True, **K_MAKER) + + deadline = time.monotonic() + DRIFT_K_TIMEOUT_S + absent = False + while time.monotonic() < deadline: + # Only a SUCCESSFUL listing may prove absence (active_rule_fingerprints + # degrades to an empty set on fetch errors — a hiccup must not + # false-pass this check). + status, body = http_json(f"{GATEWAY}/api/rules?status=active") + if status == 200 and isinstance(body, dict): + fps = {r.get("fingerprint_id") for r in body.get("rules") or []} + if SYSLOG_FP not in fps: + absent = True + break + time.sleep(POLL_INTERVAL_S) + if absent: + parts.append("syslog absent from /api/rules?status=active") + else: + problems.append("syslog never left the active rules within " + f"{DRIFT_K_TIMEOUT_S:.0f}s") + + audit = audit_after(SYSLOG_FP, "rule_deactivated", actor="drift", after=run_start, + want={"rule_id": active["id"]}) + if audit: + parts.append(f"audit rule_deactivated actor=drift rule_id={active['id']} " + f"({(audit.get('detail') or {}).get('window_start')})") + else: + problems.append("no fresh rule_deactivated audit row actor=drift " + f"rule_id={active['id']}") + + # Seed a few syslog lines NOW that the rule is provably down: the + # K-maker's own tail is a race (the simulator exits seconds before the + # deactivating scan, so its last rows may all parse while the rule is + # still active), but any line arriving after the deactivation is + # unparsed and captured as an onboarding sample — the re-entry signal. + run_simulator(label="K sample seeder (steady, rule down)", + eps="8", duration="4") + + samples = pre_samples + deadline = time.monotonic() + DRIFT_K_TIMEOUT_S + while time.monotonic() < deadline: + samples = sample_count(SYSLOG_FP) + if samples > pre_samples: + break + time.sleep(POLL_INTERVAL_S) + if samples > pre_samples: + parts.append(f"onboarding re-entry: sample_count {pre_samples}->{samples}") + else: + problems.append(f"sample_count never grew from {pre_samples}") + + # Freeze enforcement before the human restore: severe windows are + # permanent rows of the version's view and the loop re-evaluates the whole + # view every scan, so a live drift service would re-deactivate the + # restored rule within one poll (verified live). Drift restarts after the + # M3 block finishes restoring state. + compose("stop", "drift", timeout=120) + status, body = http_post_json(f"{GATEWAY}/api/rules/{active['id']}/reactivate", + {"actor": "smoke", "reason": "M3 smoke: human restore"}) + if status == 200 and isinstance(body, dict) and body.get("status") == "active": + parts.append(f"reactivate rule_id={active['id']} -> active (drift paused)") + else: + problems.append(f"reactivate rule_id={active['id']} -> {status}: {body!r}") + restored = active_rule(SYSLOG_FP) + if restored is not None and restored.get("id") == active["id"]: + parts.append(f"active again: id={restored['id']} v{restored['version']}") + else: + problems.append(f"active syslog rule after restore: {restored!r}") + reactivated = audit_after(SYSLOG_FP, "rule_reactivated", after=run_start, + want={"rule_id": active["id"]}) + if reactivated: + parts.append(f"audit rule_reactivated rule_id={active['id']}") + else: + problems.append("no fresh rule_reactivated audit row " + f"rule_id={active['id']}") + check("K severe -> deactivate -> re-onboard -> human restore", not problems, + "; ".join(parts) + ("" if not problems else + "; PROBLEMS: " + "; ".join(problems))) + + +def assert_l_unquarantine(run_start: datetime) -> None: + print("== Assert L: human un-quarantine re-enables the mapping ==") + status, body = http_post_json(f"{GATEWAY}/api/rules/{SYSLOG_FP}/unquarantine", + {"field": QUARANTINE_FIELD, "actor": "smoke"}) + if status != 200 or not isinstance(body, dict): + check("L human un-quarantine", False, + f"POST unquarantine -> {status}: {body!r}") + return + fields = body.get("quarantined_fields") + audit = audit_after(SYSLOG_FP, "field_unquarantined", after=run_start, + want={"field": QUARANTINE_FIELD}) + if fields != [] or audit is None: + check("L human un-quarantine", False, + f"quarantined_fields={fields!r} (want []), " + f"audit field_unquarantined={'present' if audit else 'MISSING'}") + return + unquarantine_ts = _parse_ts(audit.get("ts")) + # Clean lines flush the quarantined-parses out; once the pipeline has the + # un-quarantined rule (<= 2 s refresh), parses map src_endpoint.ip again. + run_simulator(label="L flush run (steady)", **FLUSH_RUN) + event = wait_parsed_event(unquarantine_ts, _l_event_ok, DRIFT_L_TIMEOUT_S) + if event is None: + check("L human un-quarantine", False, + f"no parsed syslog event after the un-quarantine ({audit.get('ts')}) " + f"maps src_endpoint.ip again") + return + src_ip = event["ocsf"]["src_endpoint"]["ip"] + check("L human un-quarantine", True, + f"POST unquarantine -> 200 quarantined_fields=[] (audit field_unquarantined " + f"actor=smoke at {audit.get('ts')}); next clean parse {event['event_id']} @ " + f"{event['parsed_at']} maps src_endpoint.ip={src_ip!r}") + + +def restore_syslog_rule(v1: dict) -> None: + """Version-bump dance (restoration, not an assert): severe/null windows are + permanent rows of v1's current view, so v1 can never again survive a drift + scan. Mint a successor version WITHOUT the SRC mapping (its candidate gate + is corruption-proof: corruption only ever touches SRC), approve it — the M2 + backlog sweep then re-parses and supersedes EVERY v1 row under the new + version, emptying v1's drift view — deactivate the successor, and + reactivate v1: active again, empty view, no severe findings possible. The + stats current view keeps every row parsed (B's precondition), and the next + smoke run's I-L block replays the whole cycle on v1.""" + print("== M3 restore: version-bump dance (clears v1's poisoned drift view) ==") + failures: list[str] = [] + lines = read_corpus_lines(SYSLOG_CORPUS) + report = validate_local_rule(SYSLOG_FP, SYSLOG_NO_SRC_RULE, lines) + if not report.passed: + failures.append(f"local gate: {report.checks}") + rule_id = version = None + if report.passed: + status, body = post_manual_candidate(SYSLOG_FP, SYSLOG_NO_SRC_RULE) + if status == 201 and isinstance(body, dict) and "rule_id" in body: + rule_id, version = body["rule_id"], body.get("version") + else: + failures.append(f"manual candidate -> {status}: {body!r}") + if rule_id is not None: + status, body = http_post_json( + f"{GATEWAY}/api/rules/{SYSLOG_FP}/candidates/{rule_id}/approve", + {"actor": "smoke", "reason": "M3 smoke restore"}) + if status != 200: + failures.append(f"approve v{version} -> {status}: {body!r}") + if not failures and version is not None: + reparse = wait_reparse_complete(SYSLOG_FP, version, REPARSE_TIMEOUT_S) + if reparse is None: + failures.append(f"no reparse_complete v{version} within {REPARSE_TIMEOUT_S:.0f}s") + else: + print(f" sweep v{version}: inserted={reparse.get('inserted')} " + f"superseded={reparse.get('superseded')} (v1 drift view now empty)") + if not failures: + status, body = http_post_json(f"{GATEWAY}/api/rules/{SYSLOG_FP}/deactivate", + {"actor": "smoke", "reason": "M3 smoke restore"}) + if status != 200: + failures.append(f"deactivate successor -> {status}: {body!r}") + if not failures: + status, body = http_post_json(f"{GATEWAY}/api/rules/{v1['id']}/reactivate", + {"actor": "smoke", "reason": "M3 smoke restore"}) + if status != 200: + failures.append(f"reactivate v1 id={v1['id']} -> {status}: {body!r}") + started = compose("start", "drift", timeout=120) + if started.returncode != 0: + failures.append(f"compose start drift failed: {started.stderr.strip()[:120]}") + if failures: + sys.exit("FATAL: M3 restoration dance failed: " + "; ".join(failures)) + left = syslog_view_count(v1["version"]) + print(f" restore complete: successor v{version} deactivated (holds the superseded " + f"backlog), seed rule id={v1['id']} active with a {left}-row drift view; " + f"drift service restarted") + + def wait_quiet(window_s: float = 4.0, timeout_s: float = 45.0) -> int: """Wait until raw_events stops moving, then return its count. @@ -949,9 +1632,54 @@ def wait_quiet(window_s: float = 4.0, timeout_s: float = 45.0) -> int: def main() -> int: - print(f"ULPF smoke (M1 A-E + M2 F-H) - {time.strftime('%Y-%m-%d %H:%M:%S')}") + # M3 demo knobs (Task 12 Step 1) — exported BEFORE compose up so the drift + # container is created with demo-scale windows (30 rows; compose + # passthrough verified in Task 4). The window count is load-bearing for + # the drift-run tuning below. + os.environ["ULPF_DRIFT_WINDOW_COUNT"] = DRIFT_WINDOW_COUNT + os.environ["ULPF_DRIFT_WINDOW_TIME_S"] = DRIFT_WINDOW_TIME_S + # Sample-capture headroom (observed live in the Task 12 gate): the + # accumulating DB saturates the 200/fingerprint cap, after which a + # drift-deactivated fingerprint cannot accumulate onboarding samples and + # K's re-entry signal (sample_count grows) is unobservable. 2000 leaves + # headroom for many accumulating runs; the code default stays 200. + os.environ["ULPF_SAMPLE_CAPTURE_CAP"] = "2000" + print(f"ULPF smoke (M1 A-E + M2 F-H + M3 I-L) - {time.strftime('%Y-%m-%d %H:%M:%S')}") step1_up() + # M3 block FIRST (Task 12 Steps 2-5): the drift demo needs the syslog + # baseline to arm DURING the firmware-drift run (the establishing 10th + # window must itself be the moderate one — see the module header), so it + # runs before the A-H steady traffic can pre-arm a clean baseline. + run_start = datetime.now(UTC) + active = active_rule(SYSLOG_FP) + if active is None: + sys.exit("FATAL: no active syslog rule at the M3 block start") + corpus_lines = len(read_corpus_lines(SYSLOG_CORPUS)) + offset = syslog_view_count(active["version"]) % 30 + plan = tune_drift_run(offset, corpus_lines) + runs_desc = " then ".join( + f"eps={r['eps']}/after={r['drift_after']}/dur={r['duration']}s" for r in plan["runs"]) + print(f" syslog seed rule id={active['id']} v{active['version']}; drift view " + f"offset={offset} -> tuned drift run(s): {runs_desc} " + f"(predicted J-window violation {plan['j_violation']:.3f} " + f"({plan['j_nulls']} nulls), establishing window " + f"{plan['est_violation']:.3f})") + pre_samples = sample_count(SYSLOG_FP) + + assert_i_tier1(run_start, plan) + assert_j_quarantine(run_start) + assert_k_severe(run_start, active, pre_samples) + assert_l_unquarantine(run_start) + restore_syslog_rule(active) + + if M3_ONLY: + failed = [name for name, ok, _ in results if not ok] + print(f"\nSMOKE RESULT (M3 I-L only): {'PASS' if not failed else 'FAIL'} " + f"({len(results) - len(failed)}/{len(results)} checks)" + + (f" failed: {', '.join(failed)}" if failed else "")) + return 0 if not failed else 1 + print("== Snapshot before (delta base) ==") before_raw = wait_quiet() before_norm = db_count("SELECT count(*) FROM normalized_events") @@ -959,7 +1687,7 @@ def main() -> int: print(f" raw_events={before_raw} normalized_events={before_norm} " f"fingerprints={len(before_fps)}") - sent = run_simulator() + sent = run_simulator(label="Step 2 (steady)") print(" " + ", ".join(f"{name}={n}" for name, n in sorted(sent.items()))) drained, raw_delta, norm_delta = wait_drain(before_raw, before_norm, sent["total"]) diff --git a/services/drift/Dockerfile b/services/drift/Dockerfile new file mode 100644 index 0000000..d4f4a48 --- /dev/null +++ b/services/drift/Dockerfile @@ -0,0 +1,28 @@ +# services/drift/Dockerfile — M3 drift detection: stateless window recompute, +# tier-1 invariants, INSERT-only drift_windows writes. Postgres only, no Kafka. +# Build context is the repo root (compose build.context: ..; see /.dockerignore). +FROM python:3.12-slim + +WORKDIR /app + +# BOTH libs first (pipeline-image pattern), then deps (layer caching), then +# the service package. /app contains drift/ and libs/ so `python -m drift.app` +# imports cleanly. psycopg is the only direct dep — windows/tier-1 are stdlib +# plus ulpf_core (regex/pydantic arrive via ulpf-core's own pins). +COPY libs/ulpf-core/pyproject.toml /app/libs/ulpf-core/pyproject.toml +COPY libs/ulpf-core/ulpf_core/ /app/libs/ulpf-core/ulpf_core/ +COPY libs/ocsf-schema/pyproject.toml /app/libs/ocsf-schema/pyproject.toml +COPY libs/ocsf-schema/ocsf_schema/ /app/libs/ocsf-schema/ocsf_schema/ +COPY services/drift/requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt \ + && pip install --no-cache-dir -e /app/libs/ulpf-core -e /app/libs/ocsf-schema + +COPY services/drift/__init__.py /app/drift/__init__.py +COPY services/drift/config.py /app/drift/config.py +COPY services/drift/windows.py /app/drift/windows.py +COPY services/drift/detect.py /app/drift/detect.py +COPY services/drift/enforce.py /app/drift/enforce.py +COPY services/drift/store.py /app/drift/store.py +COPY services/drift/app.py /app/drift/app.py + +CMD ["python", "-m", "drift.app"] diff --git a/services/drift/__init__.py b/services/drift/__init__.py new file mode 100644 index 0000000..4af5261 --- /dev/null +++ b/services/drift/__init__.py @@ -0,0 +1,3 @@ +# services/drift — M3 drift-detection service (cold path, Postgres only): +# stateless window recompute from the current view, tier-1 invariants from +# event zero, INSERT-only drift_windows writes (plan Task 4). diff --git a/services/drift/app.py b/services/drift/app.py new file mode 100644 index 0000000..52bfe4e --- /dev/null +++ b/services/drift/app.py @@ -0,0 +1,128 @@ +# services/drift/app.py — the drift service entrypoint: ONE convergent loop +# (cold path, Postgres only — no Kafka). +# +# Per poll: load active rules, then for each rule one stateless scan, in the +# frozen order — fetch_current_view (the DB current view) -> compute_windows +# (pure, data-only horizon) -> evaluate_window (tier-1 always; tier-2 only +# against an existing baseline profile) -> record_windows (idempotent by the +# deterministic window key) -> FIRST baseline insert when the Nth baseline +# window just closed -> enforce_findings worst-first. Ruling: on the +# establishing scan itself the baseline profile does not exist yet at +# evaluation time, so that scan is tier-1-only by construction — the windows +# BEYOND the Nth on that same scan keep their first-recorded tier-1 severity +# (ON CONFLICT DO NOTHING), but the next scan re-evaluates the whole current +# view WITH the profile and enforcement converges. Fail-closed posture: drift +# NEVER blocks anything else — every exception is logged with traceback and +# the loop continues; a failed scan is simply retried on the next poll. +import logging +import threading + +from drift.config import Config +from drift.detect import Finding, aggregate_baseline, evaluate_window +from drift.enforce import enforce_findings, record_windows +from drift.store import ( + ActiveDriftRule, + baseline_for, + connect_db, + fetch_current_view, + fetch_window_history, + first_window_unmapped, + insert_baseline, + load_active_rules, +) +from drift.windows import compute_windows + +log = logging.getLogger("drift.app") + + +def establish_baseline(conn, cfg: Config, rule: ActiveDriftRule) -> bool: + """The FIRST baseline insert, once due (R-M3-7): after this scan's windows + are recorded, aggregate the version's first cfg.baseline_windows closed + windows and INSERT the profiles once. Returns False while fewer than N + windows exist; insert_baseline's existence guard + ON CONFLICT make later + scans no-ops (the 11th scan never re-inserts), and a version bump re-arms + naturally — the new (fingerprint, version) has no profile, so tier-1-only + baseline windows accumulate again.""" + history = fetch_window_history(conn, rule.fingerprint_id, rule.version) + profiles = aggregate_baseline(history, baseline_windows=cfg.baseline_windows) + if profiles is None: + return False + return insert_baseline(conn, rule.fingerprint_id, rule.version, + profiles, cfg.baseline_windows) + + +def scan_rule(cfg: Config, rule: ActiveDriftRule) -> int: + """One scan for one active rule; returns the number of CLOSED windows + found (0 when the current view holds nothing provably complete). One + connection covers the whole scan (the reparse per-sweep pattern).""" + with connect_db(cfg.database_url) as conn: + rows = fetch_current_view(conn, rule.fingerprint_id, rule.version) + baseline = baseline_for(conn, rule.fingerprint_id, rule.version) + known = (first_window_unmapped(conn, rule.fingerprint_id, rule.version) + if baseline is not None else None) + windows = compute_windows( + rows, + count_window=cfg.window_count, + time_window_s=cfg.window_time_s, + mapped_paths=tuple(dict.fromkeys(m.ocsf_path for m in rule.mappings)), + known_unmapped=known, + ) + if not windows: + return 0 + findings: list[Finding] = [] + for win in windows: + window_findings = evaluate_window( + win, baseline, rule_quarantined=rule.quarantined_fields) + record_windows(conn, rule, window_findings, win) + findings.extend(window_findings) + if baseline is None: + establish_baseline(conn, cfg, rule) + enforce_findings(conn, rule, findings) + log.info("scanned %s v%d: %d closed window(s) over %d current-view row(s)", + rule.fingerprint_id, rule.version, len(windows), len(rows)) + return len(windows) + + +def run_drift_loop(cfg: Config, stop: threading.Event) -> None: + """Poll for active rules and scan each, serially (cold path). `stop.set()` + ends the loop promptly (wait() is sliced by the event). Every exception is + logged; the loop continues — windows are recomputed from the DB next poll, + so a failed scan self-heals (fail-closed, never blocks the hot path).""" + log.info("drift loop started (window_count=%d, window_time_s=%d, poll=%.1fs)", + cfg.window_count, cfg.window_time_s, cfg.poll_s) + while not stop.is_set(): + try: + with connect_db(cfg.database_url) as conn: + rules = load_active_rules(conn) + for rule in rules: + if stop.is_set(): + break + try: + scan_rule(cfg, rule) + except Exception: + log.exception("drift scan failed for %s v%d; continuing", + rule.fingerprint_id, rule.version) + except Exception: + log.exception("drift loop iteration failed; will retry") + stop.wait(cfg.poll_s) + log.info("drift loop stopped") + + +def main() -> None: + logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s" + ) + cfg = Config.from_env() + # ONE loop, run in the MAIN thread (controller ruling — unlike onboarding's + # two daemon loops): SIGINT lands here directly and ends the wait-sliced + # loop; run_drift_loop stays daemon-friendly for any hosting thread. + stop = threading.Event() + try: + run_drift_loop(cfg, stop) + except KeyboardInterrupt: + stop.set() + log.info("shutdown requested") + + +if __name__ == "__main__": + main() diff --git a/services/drift/config.py b/services/drift/config.py new file mode 100644 index 0000000..6178925 --- /dev/null +++ b/services/drift/config.py @@ -0,0 +1,31 @@ +# services/drift/config.py — drift service configuration. +# +# Compose env contract (Task 1 pinned these exact names in deploy/.env.example +# and docker-compose passes them through): DATABASE_URL plus the four +# ULPF_DRIFT_* knobs. Defaults equal the .env.example values (binding ruling); +# the DSN follows the per-service constant pattern (env overrides). +import os +from dataclasses import dataclass + + +@dataclass +class Config: + database_url: str = "postgresql://drift_role:drift_dev@postgres:5432/ulpf" + window_count: int = 1000 # rows per window (count-close boundary) + window_time_s: int = 600 # close a window whose FIRST row ages out + baseline_windows: int = 10 # windows aggregated into the baseline (Task 5) + poll_s: float = 5.0 + + @classmethod + def from_env(cls) -> "Config": + base = cls() + return cls( + database_url=os.environ.get("DATABASE_URL", base.database_url), + window_count=int(os.environ.get("ULPF_DRIFT_WINDOW_COUNT", + base.window_count)), + window_time_s=int(os.environ.get("ULPF_DRIFT_WINDOW_TIME_S", + base.window_time_s)), + baseline_windows=int(os.environ.get("ULPF_DRIFT_BASELINE_WINDOWS", + base.baseline_windows)), + poll_s=float(os.environ.get("ULPF_DRIFT_POLL_S", base.poll_s)), + ) diff --git a/services/drift/detect.py b/services/drift/detect.py new file mode 100644 index 0000000..05bbc24 --- /dev/null +++ b/services/drift/detect.py @@ -0,0 +1,298 @@ +# services/drift/detect.py — Window -> findings -> drift_windows rows +# (R-M3-4 measurement + R-M3-11 evaluation). +# +# Two layers over one window: window_rows (measurement — the per-field rates +# and shape histograms, tier-1 severity) and evaluate_window (Task 5 — the +# graduated decision: tier-1 ALWAYS, tier-2 null/match/shape/volume ladders +# only when a baseline profile exists, per-field severity = max across +# signals, decide_action semantics with the two ruling overrides: quarantined +# fields stay measured and still escalate, unmapped.* rows are new-key +# signals only — alert-only, decide_action bypassed). aggregate_baseline is +# the pure FIRST-N-window aggregation behind baseline establishment. +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING + +from ulpf_core.drift import ( + SEVERITIES, + decide_action, + js_divergence, + match_rate_severity, + null_rate_severity, + shape_histogram, + shape_severity, + tier1_severity, + volume_severity, +) + +from drift.windows import FieldStats, Window + +if TYPE_CHECKING: + from drift.store import ActiveDriftRule + +# drift_windows.field vocabulary (migration 007): OCSF path, 'unmapped.', +# or the per-window rule-level sentinel that carries match_rate. +RULE_FIELD = "__rule__" +UNMAPPED_PREFIX = "unmapped." + +# Column order of the drift_windows INSERT in store.insert_windows (this is +# the tuple contract the two modules share; scanned_at uses its DB default). +_ROW_COLUMNS = ( + "fingerprint_id", "rule_version", "field", "window_start", "window_end", + "events_count", "null_rate", "match_rate", "violation_rate", "shape_dist", + "severity", "action_taken", +) + + +@dataclass(frozen=True) +class Finding: + """One (window, field) drift decision: the measured window's per-field + severity (max across tier-1 + tier-2 signals), the graduated action the + severity warrants, the measurements behind it, and the window's start (the + audit detail + the deterministic drift_windows key).""" + + field: str + severity: str + action: str | None + stats: dict + window_start: datetime + + +def shape_dist(stats: FieldStats) -> dict: + """Histogram for one field's stats — EMPTY HISTOGRAM GUARD (Task 3 review + carry, binding): an empty value list short-circuits to {} BEFORE the shape + machinery runs, so js_divergence is never fed an empty histogram and a + field with no observed values stays silent (silence is the volume ladder's + job, never a shape comparison).""" + return shape_histogram(stats.values) if stats.values else {} + + +def window_rows(rule: ActiveDriftRule, windows: list[Window]) -> list[tuple]: + """One drift_windows tuple per (window, field), in _ROW_COLUMNS order. + + A window with zero parsed docs emits ONLY the __rule__ row: with no docs + there is nothing to shape-compare (guard) and no tier-1 denominator. + action_taken stays NULL — Task 5's enforcement decides and audits it. + """ + rows: list[tuple] = [] + for win in windows: + if win.parsed: + for field, stats in win.field_stats.items(): + rows.append(_field_row(rule, win, field, stats)) + rows.append(_rule_row(rule, win)) + return rows + + +def _field_row(rule: ActiveDriftRule, win: Window, field: str, + stats: FieldStats) -> tuple: + is_unmapped = field.startswith(UNMAPPED_PREFIX) + violation_rate = None if is_unmapped else stats.violations / win.parsed + severity = "none" if is_unmapped else tier1_severity(violation_rate) + return ( + rule.fingerprint_id, rule.version, field, + win.start, win.end, win.count, + stats.nulls / win.parsed, # null_rate + None, # match_rate is __rule__-only + violation_rate, + shape_dist(stats), + severity, + None, # action_taken (Task 5 enforcement) + ) + + +def _rule_row(rule: ActiveDriftRule, win: Window) -> tuple: + denominator = win.parsed + win.parse_errors + match_rate = win.parsed / denominator if denominator else None + return ( + rule.fingerprint_id, rule.version, RULE_FIELD, + win.start, win.end, win.count, + None, match_rate, None, None, + # Measurement default: the decided severity/action come from + # evaluate_window and are overlaid by enforce.record_windows. + "none", + None, + ) + + +# --- Task 5: tier-2 evaluation (R-M3-11) ------------------------------------------- + + +def _max_severity(signals: list[str]) -> str: + """Max severity by ladder order (SEVERITIES is ascending none..severe).""" + return max(signals, key=SEVERITIES.index) if signals else "none" + + +def evaluate_window(win: Window, baseline: dict | None, + *, rule_quarantined: list[str] | tuple = ()) -> list[Finding]: + """Evaluate one closed window: tier-1 ALWAYS, tier-2 only when a baseline + profile exists for the field (R-M3-11). Emission mirrors window_rows + exactly — mapped fields only when win.parsed > 0, then the __rule__ + sentinel — so enforce.record_windows can overlay findings onto measured + rows by field. Quarantined fields are STILL measured and STILL escalate: + quarantine only suppresses a redundant re-quarantine action, never a + finding, and a quarantined field going severe still deactivates the rule. + """ + findings: list[Finding] = [] + if win.parsed: + for field_name, stats in win.field_stats.items(): + findings.append(_field_finding(win, field_name, stats, baseline, + tuple(rule_quarantined))) + findings.append(_rule_finding(win, baseline)) + return findings + + +def _field_finding(win: Window, field_name: str, stats: FieldStats, + baseline: dict | None, rule_quarantined: tuple) -> Finding: + null_rate = stats.nulls / win.parsed + hist = shape_dist(stats) # the EMPTY HISTOGRAM GUARD lives in here + + if field_name.startswith(UNMAPPED_PREFIX): + # Binding ruling (Task 4 review carry): unmapped rows are NEW-KEY- + # EMERGENCE signals only — no tier-1 ladder, no tier-2 null/shape + # comparison even if a profile somehow exists, and decide_action is + # bypassed: alert-only. Rates below are the recorded measurement, the + # same numbers window_rows writes, never re-derived into severity. + is_new = field_name[len(UNMAPPED_PREFIX):] in win.new_unmapped + return Finding( + field_name, "minor" if is_new else "none", "alert" if is_new else None, + {"events_count": win.count, "null_rate": null_rate, "shape_dist": hist}, + win.start) + + violation_rate = stats.violations / win.parsed + signals = [tier1_severity(violation_rate)] + profile = baseline.get(field_name) if baseline else None + if profile is not None: # tier-2 arms per field: no profile -> tier-1 only + base_null = profile.get("null_rate") + if base_null is not None: + signals.append(null_rate_severity(null_rate, base_null)) + base_shape = profile.get("shape_dist") + if hist and base_shape: + # Empty-histogram guard: a window with no observed values never + # shape-compares (silence is the volume ladder's job). + signals.append(shape_severity( + js_divergence(hist, _window_scale(base_shape, hist)))) + severity = _max_severity(signals) + action = decide_action(severity) + if action == "field_quarantined" and field_name in rule_quarantined: + action = None # already quarantined: idempotent skip, never a re-audit + return Finding(field_name, severity, action, + {"events_count": win.count, "null_rate": null_rate, + "violation_rate": violation_rate, "shape_dist": hist}, + win.start) + + +def _rule_finding(win: Window, baseline: dict | None) -> Finding: + """The __rule__ sentinel: match_rate_severity + volume_severity, both + tier-2 (no tier-1 signal exists for the rule itself).""" + denominator = win.parsed + win.parse_errors + match_rate = win.parsed / denominator if denominator else None + signals: list[str] = [] + profile = baseline.get(RULE_FIELD) if baseline else None + if profile is not None: + base_match = profile.get("match_rate") + if match_rate is not None and base_match is not None: + signals.append(match_rate_severity(match_rate, base_match)) + bounds = profile.get("events_count") or {} + if "min" in bounds and "max" in bounds: + signals.append(volume_severity(win.count, bounds["min"], bounds["max"])) + severity = _max_severity(signals) + action = decide_action(severity) + if action == "field_quarantined": + # The sentinel is not a mappable field: a moderate match/volume + # decline alerts (humans look); only severe deactivates. + action = "alert" + return Finding(RULE_FIELD, severity, action, + {"events_count": win.count, "match_rate": match_rate}, win.start) + + +def merge_findings(findings: list[Finding]) -> dict[str, str]: + """Per-field max severity across one scan's findings (R-M3-11) — the + worst thing each field did in any window of the scan.""" + merged: dict[str, str] = {} + for finding in findings: + current = merged.get(finding.field, "none") + if SEVERITIES.index(finding.severity) > SEVERITIES.index(current): + merged[finding.field] = finding.severity + return merged + + +# --- Task 5: baseline aggregation (R-M3-7) ----------------------------------------- + + +def aggregate_baseline(rows: list[tuple], *, baseline_windows: int) -> dict | None: + """Aggregate the FIRST `baseline_windows` closed windows of a rule version + into its baseline profiles, or None while fewer windows exist. + + rows: fetch_window_history output — (window_start, field, events_count, + null_rate, match_rate, shape_dist) ordered by window_start. Per mapped + field: mean null_rate + shape_dist SUMMED across the windows then + normalized to proportions. __rule__: mean match_rate + events_count + min/max (the volume bounds, R-M3-10). unmapped.* rows are skipped — they + are alert-only new-key signals, never tier-2 compared (binding ruling). + """ + starts: list = [] + seen: set = set() + for row in rows: + if row[0] not in seen: + seen.add(row[0]) + starts.append(row[0]) + if len(starts) < baseline_windows: + return None + cutoff = starts[baseline_windows - 1] + + buckets: dict[str, dict] = {} + for window_start, field_name, events_count, null_rate, match_rate, shape in rows: + if window_start > cutoff or field_name.startswith(UNMAPPED_PREFIX): + continue + bucket = buckets.setdefault(field_name, {"null_rates": [], "match_rates": [], + "shape": {}, "counts": []}) + if null_rate is not None: + bucket["null_rates"].append(null_rate) + if match_rate is not None: + bucket["match_rates"].append(match_rate) + if shape: + for cls, n in shape.items(): + bucket["shape"][cls] = bucket["shape"].get(cls, 0) + n + bucket["counts"].append(events_count) + + profiles: dict[str, dict] = {} + for field_name, bucket in buckets.items(): + if field_name == RULE_FIELD: + profiles[field_name] = { + "match_rate": (_mean(bucket["match_rates"])), + "events_count": {"min": min(bucket["counts"]), + "max": max(bucket["counts"])}, + } + else: + profiles[field_name] = { + "null_rate": _mean(bucket["null_rates"]), + "shape_dist": _normalized(bucket["shape"]), + } + return profiles + + +def _mean(values: list[float]) -> float | None: + return sum(values) / len(values) if values else None + + +def _normalized(histogram: dict) -> dict: + """Count histogram -> proportions summing to 1.0 ({} stays {}). The STORED + profile is scale-free; comparisons rescale it back (see _window_scale).""" + total = sum(histogram.values()) + return {cls: n / total for cls, n in histogram.items()} if total else {} + + +def _window_scale(baseline_shape: dict, hist: dict) -> dict: + """Project the stored (normalized) baseline shape onto the current + window's observed-value count. R-M3-5's js_divergence add-1 smoothing is + calibrated for COUNT-scale histograms (T3 pins 'grows toward ln2 as counts + grow'); feeding raw proportions would let the +1s dominate the baseline + side and cap every shape comparison at ~0.32 (verified: full flip + converges 0.2404@N=20 -> 0.3178@N=10k; the ~0.057 figure is the cap of a + props-vs-props shape that never occurs), leaving the severe band + unreachable. Rescaling puts both sides at the same count scale, where the + smoothing is negligible and the ladder bands mean what T3 calibrated.""" + total = sum(hist.values()) + return {cls: proportion * total for cls, proportion in baseline_shape.items()} diff --git a/services/drift/enforce.py b/services/drift/enforce.py new file mode 100644 index 0000000..8ad44f9 --- /dev/null +++ b/services/drift/enforce.py @@ -0,0 +1,166 @@ +# services/drift/enforce.py — graduated, fail-closed enforcement (R-M3-11, +# spec §7.2) under drift_role's column-scoped grants (migration 007). +# +# The grant surface IS the design, exactly as in gateway/writes.py: drift_role +# may UPDATE ONLY rules(quarantined_fields) and rules(status, deactivated_at), +# INSERT drift_windows/baseline_profiles/audit_log, and SELECT its own tables. +# Every UPDATE carries its expected-state predicate in the WHERE (TOCTOU +# backstop) and is answered by a rowcount check — 0 rows means another writer +# moved the rule (already quarantined / no longer active): False, never a +# silent no-op that would still write an audit row. On a successful mutation +# the audit INSERT is paired INSIDE the same transaction, actor 'drift' +# (store.ACTOR). +# +# Worst-first scan enforcement (enforce_findings): severe findings before +# moderate; ONE deactivate per fingerprint per scan; after a deactivate the +# remaining actions for that fingerprint are skipped and logged (the rule is +# down — quarantine appends would no-op against status='active' anyway). +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from ulpf_core.drift import SEVERITIES + +from drift.detect import Finding, merge_findings, window_rows +from drift.store import ACTOR, insert_windows + +if TYPE_CHECKING: + import psycopg + + from drift.store import ActiveDriftRule + from drift.windows import Window + +log = logging.getLogger("drift.enforce") + +_AUDIT_SQL = "INSERT INTO audit_log (actor, action, entity, detail) VALUES (%s, %s, %s, %s)" + +# Column-scoped exactly to migration 007's grants; the NOT (... = ANY(...)) +# term is the idempotency predicate — quarantining an already-quarantined +# field matches zero rows and cannot audit. +_QUARANTINE_SQL = ( + "UPDATE rules SET quarantined_fields = array_append(quarantined_fields, %s) " + "WHERE id = %s AND status = 'active' " + "AND NOT (%s = ANY(quarantined_fields))" +) +_DEACTIVATE_SQL = ( + "UPDATE rules SET status = 'deactivated', deactivated_at = now() " + "WHERE id = %s AND status = 'active'" +) + + +def _audit(cur, action: str, entity: str, detail: dict) -> None: + from psycopg.types.json import Json # deferred: keeps psycopg off the test path + + cur.execute(_AUDIT_SQL, (ACTOR, action, entity, Json(detail))) + + +def quarantine_field(conn: psycopg.Connection, rule: ActiveDriftRule, field: str, + window_start) -> bool: + """Append one field to the rule's quarantined_fields (mapping disabled at + parse time; values preserved in unmapped). Idempotent: the NOT (... = ANY + (...)) predicate means an already-quarantined field — or a rule that is no + longer active — updates zero rows, returns False and writes NO audit (a + no-op must never look like an action). True iff the append happened, in + which case the audit row is paired in the same transaction.""" + with conn.transaction(), conn.cursor() as cur: + cur.execute(_QUARANTINE_SQL, (field, rule.id, field)) + if cur.rowcount == 0: + log.info("quarantine no-op for %s v%d field %s (already quarantined " + "or rule inactive)", rule.fingerprint_id, rule.version, field) + return False + # window_start is JSON-serialized as ISO-8601: the audit detail is + # JSONB, and psycopg's Json() does NOT convert datetime (observed live + # in Task 12: the deactivate audit raised TypeError and rolled the + # paired mutation back). + _audit(cur, "field_quarantined", rule.fingerprint_id, + {"field": field, "rule_id": rule.id, "version": rule.version, + "window_start": window_start.isoformat()}) + log.warning("field %s quarantined on %s v%d (window %s)", + field, rule.fingerprint_id, rule.version, window_start) + return True + + +def deactivate_rule(conn: psycopg.Connection, rule: ActiveDriftRule, + window_start) -> bool: + """Deactivate the rule (fingerprint reverts to raw-only and re-enters + onboarding). The status='active' predicate is the expected-state TOCTOU + backstop: zero rows -> False (a human or another scan got there first), + never an audit for something that did not happen.""" + with conn.transaction(), conn.cursor() as cur: + cur.execute(_DEACTIVATE_SQL, (rule.id,)) + if cur.rowcount == 0: + log.info("deactivate no-op for rule %d (no longer active)", rule.id) + return False + _audit(cur, "rule_deactivated", rule.fingerprint_id, + {"rule_id": rule.id, "version": rule.version, + "window_start": window_start.isoformat()}) + log.warning("rule %d (%s v%d) DEACTIVATED by drift (window %s)", + rule.id, rule.fingerprint_id, rule.version, window_start) + return True + + +def record_windows(conn: psycopg.Connection, rule: ActiveDriftRule, + findings: list[Finding], win: Window) -> None: + """INSERT one window's drift_windows rows with the DECIDED severity and + action_taken overlaid onto the measured row (window_rows recomputes the + rates — the exact tier-1 measurement discipline — and the finding supplies + the decision). Idempotent by the deterministic window key through + store.insert_windows' ON CONFLICT DO NOTHING; one transaction per window + batch. The plan's listed form record_windows(conn, findings, win) lacked + the rule identity the INSERT columns need — rule leads, mirroring + window_rows(rule, windows).""" + by_field = {finding.field: finding for finding in findings} + rows = [] + for measured in window_rows(rule, [win]): + finding = by_field.get(measured[2]) + if finding is None: # defensive: emission parity is pinned by test + rows.append(measured) + else: + rows.append(measured[:10] + (finding.severity, finding.action)) + insert_windows(conn, rows) + + +def _rank(severity: str) -> int: + return SEVERITIES.index(severity) + + +def enforce_findings(conn: psycopg.Connection, rule: ActiveDriftRule, + findings: list[Finding]) -> dict: + """Enforce one scan's findings, worst-first (R-M3-11): severe before + moderate (alerts need no enforcement — the drift_windows row IS the + alert), ONE deactivate per fingerprint per scan, and after a deactivate + the remaining actions for this fingerprint are skipped and logged. + Returns {"deactivated": bool, "quarantined": [fields actually appended]}. + """ + merged = merge_findings(findings) # per-field max severity (R-M3-11) + worst: dict[str, Finding] = {} # that max, as the finding that carries it + # (enforcement needs the finding's window_start for the audit detail) + for finding in sorted(findings, + key=lambda f: (-_rank(f.severity), f.window_start, f.field)): + if merged.get(finding.field, "none") == finding.severity: + worst.setdefault(finding.field, finding) + + quarantined: list[str] = [] + deactivated = False + skip_rest = False + for finding in sorted(worst.values(), key=lambda f: (-_rank(f.severity), f.field)): + if skip_rest: + log.info("skipping %s on %s (window %s): the rule was deactivated " + "this scan", finding.action, finding.field, finding.window_start) + continue + if finding.action == "alert": + # minor (or sentinel-moderate): alert + metric logged — no DB action. + log.warning("drift alert: %s v%d field %s severity %s (window %s) " + "stats %s", rule.fingerprint_id, rule.version, finding.field, + finding.severity, finding.window_start, finding.stats) + elif finding.action == "rule_deactivated": + skip_rest = True # one deactivate per fingerprint per scan + deactivated = deactivate_rule(conn, rule, finding.window_start) + if not deactivated: + log.info("rule %d no longer active; drift enforcement stands " + "down for this scan", rule.id) + elif (finding.action == "field_quarantined" + and quarantine_field(conn, rule, finding.field, finding.window_start)): + quarantined.append(finding.field) + return {"deactivated": deactivated, "quarantined": quarantined} diff --git a/services/drift/requirements.txt b/services/drift/requirements.txt new file mode 100644 index 0000000..6b5ba0d --- /dev/null +++ b/services/drift/requirements.txt @@ -0,0 +1 @@ +psycopg[binary]==3.2.1 diff --git a/services/drift/store.py b/services/drift/store.py new file mode 100644 index 0000000..8336895 --- /dev/null +++ b/services/drift/store.py @@ -0,0 +1,188 @@ +# services/drift/store.py — all drift DB access (cold path, Postgres only). +# +# Every function takes a live connection (the app loop opens one per scan) and +# imports psycopg lazily, so unit tests drive everything through fake conns — +# never a live DB. Connections run AUTOCOMMIT (the T7 house style): each +# multi-statement operation wraps exactly one `with conn.transaction():` block, +# single statements rely on autocommit. +# +# Grant shape (migration 007): drift_role has SELECT on normalized_events, +# rules, drift_windows and baseline_profiles, and INSERT on drift_windows, +# baseline_profiles and audit_log. Detection is INSERT-only — no UPDATE +# anywhere in this module (the rules UPDATE grants are Task 5's enforce layer). +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from pydantic import Field +from ulpf_core.models import Mapping, Rule + +if TYPE_CHECKING: + import psycopg + +log = logging.getLogger("drift.store") + +# Audit actor for every drift-side action (Task 5's enforce layer reuses it). +ACTOR = "drift" + + +class ActiveDriftRule(Rule): + """Rule plus the rules-table serial id and the M3 quarantine list (the + pipeline's rules.ActiveRule pattern). pattern is fetched only because Rule + requires it — the scan never compiles it.""" + + id: int + quarantined_fields: list[str] = Field(default_factory=list) + + +def connect_db(url: str) -> psycopg.Connection: + """psycopg connection for drift_role; autocommit per house style.""" + import psycopg # deferred: unit tests never need psycopg + + return psycopg.connect(url, autocommit=True) + + +def load_active_rules(conn: psycopg.Connection) -> list[ActiveDriftRule]: + """All status='active' rules, ordered by fingerprint_id for a deterministic + scan order. mappings arrives as JSONB (list of {source_field, ocsf_path}) + and is rebuilt into the typed Mapping model; quarantined_fields as TEXT[]. + provenance is fetched only because Rule requires it (the reparse ruling). + """ + with conn.cursor() as cur: + cur.execute( + "SELECT id, fingerprint_id, version, pattern, mappings, " + "quarantined_fields, provenance FROM rules WHERE status = 'active' " + "ORDER BY fingerprint_id" + ) + return [ + ActiveDriftRule( + id=rule_id, + fingerprint_id=fingerprint_id, + version=version, + pattern=pattern, + mappings=[Mapping(**m) for m in mappings], + provenance=provenance, + quarantined_fields=list(quarantined or []), + ) + for (rule_id, fingerprint_id, version, pattern, mappings, + quarantined, provenance) in cur.fetchall() + ] + + +def fetch_current_view(conn: psycopg.Connection, fingerprint_id: str, + version: int) -> list[tuple]: + """The DB current view (ne_current_idx semantics): unsuperseded + normalized_events rows parsed under the ACTIVE version, oldest first. A + fingerprint with only unparsed rows (rule_version 0) has no rows here — + and therefore no windows.""" + with conn.cursor() as cur: + cur.execute( + "SELECT parsed_at, status, ocsf FROM normalized_events " + "WHERE fingerprint_id = %s AND rule_version = %s " + "AND superseded_by_event_id IS NULL ORDER BY parsed_at", + (fingerprint_id, version), + ) + return cur.fetchall() + + +def insert_windows(conn: psycopg.Connection, rows: list[tuple]) -> None: + """One INSERT per drift_windows row, idempotent by the deterministic PK + (fingerprint_id, rule_version, field, window_start) — a re-scan of already + closed windows INSERTs nothing. The whole batch is one transaction so a + crash never leaves a half-written window. shape_dist (dict) is wrapped for + JSONB here, at the SQL boundary.""" + from psycopg.types.json import Json # deferred: keeps psycopg off the test path + + sql = ( + "INSERT INTO drift_windows (fingerprint_id, rule_version, field, " + "window_start, window_end, events_count, null_rate, match_rate, " + "violation_rate, shape_dist, severity, action_taken) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) " + "ON CONFLICT (fingerprint_id, rule_version, field, window_start) DO NOTHING" + ) + with conn.transaction(), conn.cursor() as cur: + for row in rows: + shape = row[9] + params = row[:9] + (Json(shape) if shape is not None else None,) + row[10:] + cur.execute(sql, params) + log.info("inserted up to %d drift window rows (conflicts are no-ops)", len(rows)) + + +def baseline_for(conn: psycopg.Connection, fingerprint_id: str, + version: int) -> dict | None: + """{field: profile} for the (fingerprint, version) baseline, or None while + no baseline exists (tier-1-only mode). profile is the JSONB dict written + at establishment (Task 5).""" + with conn.cursor() as cur: + cur.execute( + "SELECT field, profile FROM baseline_profiles " + "WHERE fingerprint_id = %s AND rule_version = %s", + (fingerprint_id, version), + ) + rows = cur.fetchall() + return {field: profile for field, profile in rows} or None + + +def fetch_window_history(conn: psycopg.Connection, fingerprint_id: str, + version: int) -> list[tuple]: + """(window_start, field, events_count, null_rate, match_rate, shape_dist) + rows of the version's drift_windows, oldest window first, field-sorted + within a window — the ordered input detect.aggregate_baseline slices the + FIRST N closed windows from (the whole history is read: it is bounded by + raw retention, and the scan is a cold path).""" + with conn.cursor() as cur: + cur.execute( + "SELECT window_start, field, events_count, null_rate, match_rate, " + "shape_dist FROM drift_windows " + "WHERE fingerprint_id = %s AND rule_version = %s " + "ORDER BY window_start, field", + (fingerprint_id, version), + ) + return cur.fetchall() + + +def first_window_unmapped(conn: psycopg.Connection, fingerprint_id: str, + version: int) -> set[str]: + """Unmapped keys of the baseline's FIRST window — the key set new unmapped + keys are measured against (Step 1: absent from the mapped targets AND from + this set). Keys are returned unprefixed (as doc['unmapped'] keys).""" + with conn.cursor() as cur: + cur.execute( + "SELECT DISTINCT field FROM drift_windows " + "WHERE fingerprint_id = %s AND rule_version = %s " + "AND field LIKE 'unmapped.%%' " + "AND window_start = (SELECT MIN(window_start) FROM drift_windows " + "WHERE fingerprint_id = %s AND rule_version = %s)", + (fingerprint_id, version, fingerprint_id, version), + ) + prefix_len = len("unmapped.") + return {row[0][prefix_len:] for row in cur.fetchall()} + + +def insert_baseline(conn: psycopg.Connection, fingerprint_id: str, version: int, + profiles: dict, windows_seen: int) -> bool: + """Insert the baseline profiles once: an existence SELECT guards the write + (the plan's INSERT-once semantics) and ON CONFLICT DO NOTHING backs it + against a race. One transaction covers guard + inserts. True iff rows were + written; False when a baseline already exists (a no-op, nothing audited).""" + from psycopg.types.json import Json # deferred: keeps psycopg off the test path + + with conn.transaction(), conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM baseline_profiles WHERE fingerprint_id = %s " + "AND rule_version = %s LIMIT 1", + (fingerprint_id, version), + ) + if cur.fetchone(): # exists (None/[] both falsy, [1] truthy) + return False + for field in sorted(profiles): + cur.execute( + "INSERT INTO baseline_profiles (fingerprint_id, rule_version, " + "field, profile, windows_seen) VALUES (%s, %s, %s, %s, %s) " + "ON CONFLICT (fingerprint_id, rule_version, field) DO NOTHING", + (fingerprint_id, version, field, Json(profiles[field]), windows_seen), + ) + log.info("baseline established for %s v%d over %d windows (%d fields)", + fingerprint_id, version, windows_seen, len(profiles)) + return True diff --git a/services/drift/tests/__init__.py b/services/drift/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/drift/tests/test_app.py b/services/drift/tests/test_app.py new file mode 100644 index 0000000..6084ab0 --- /dev/null +++ b/services/drift/tests/test_app.py @@ -0,0 +1,373 @@ +# services/drift/tests/test_app.py — loop wiring tests (onboarding-loop shape): +# fail-closed scan retries, the loop never raises out, scan_rule threading +# (baseline -> known_unmapped), the frozen Task 5 pipeline order (scan -> +# evaluate -> record -> FIRST baseline insert -> enforce worst-first), and the +# single-loop main(). Monkeypatch style: app.py imports its collaborators by +# name, so patching the app module's globals observes the wiring; NEVER a +# live DB. +import threading +from datetime import UTC, datetime, timedelta + +from ulpf_core.models import Mapping + +from drift import app +from drift.config import Config +from drift.store import ActiveDriftRule + + +def _ts(seconds: float) -> datetime: + return datetime(2026, 9, 21, 10, 0, 0, tzinfo=UTC) + timedelta(seconds=seconds) + + +def _rule(fp="fp_a", rule_id=7, version=2, quarantined=()): + return ActiveDriftRule( + id=rule_id, fingerprint_id=fp, version=version, pattern=r"x", + mappings=[Mapping(source_field="SRC", ocsf_path="src_endpoint.ip")], + provenance="human", quarantined_fields=list(quarantined)) + + +def _doc(ip="198.51.100.4"): + return {"src_endpoint": {"ip": ip, "port": 443}, "severity_id": 3, + "time": "2026-09-19T10:00:03Z", "unmapped": {}} + + +class FakeStoreConn: + """Just the context-manager role of a psycopg connection.""" + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def _fast_cfg(**kw): + kw.setdefault("poll_s", 0.0) # stop.wait(0): the loop test never sleeps + return Config(**kw) + + +# --- the loop: fail-closed, never raises out ------------------------------------ + + +def test_loop_continues_after_a_failed_iteration(monkeypatch): + stop = threading.Event() + loads = {"n": 0} + scanned = [] + + def fake_load(conn): + loads["n"] += 1 + if loads["n"] == 1: + raise RuntimeError("db down") # first poll fails... + return [_rule()] # ...the next one scans + + def fake_scan(cfg, rule): + scanned.append(rule.fingerprint_id) + stop.set() # done: end the loop from inside the scan + + monkeypatch.setattr(app, "connect_db", lambda url: FakeStoreConn()) + monkeypatch.setattr(app, "load_active_rules", fake_load) + monkeypatch.setattr(app, "scan_rule", fake_scan) + + app.run_drift_loop(_fast_cfg(), stop) + assert loads["n"] == 2 and scanned == ["fp_a"] + + +def test_loop_never_raises_out_when_everything_fails(monkeypatch): + stop = threading.Event() + loads = {"n": 0} + + def fake_load(conn): + loads["n"] += 1 + if loads["n"] >= 3: + stop.set() + raise RuntimeError("still down") + + monkeypatch.setattr(app, "connect_db", lambda url: FakeStoreConn()) + monkeypatch.setattr(app, "load_active_rules", fake_load) + + app.run_drift_loop(_fast_cfg(), stop) # returns instead of raising + assert loads["n"] == 3 + + +def test_loop_continues_past_a_failed_single_scan(monkeypatch): + stop = threading.Event() + + def fake_load(conn): + return [_rule("fp_a"), _rule("fp_b")] + + def fake_scan(cfg, rule): + if rule.fingerprint_id == "fp_a": + raise RuntimeError("scan blew up") + stop.set() # fp_b still scanned after fp_a failed + + monkeypatch.setattr(app, "connect_db", lambda url: FakeStoreConn()) + monkeypatch.setattr(app, "load_active_rules", fake_load) + monkeypatch.setattr(app, "scan_rule", fake_scan) + + app.run_drift_loop(_fast_cfg(), stop) + assert stop.is_set() # reached fp_b: one failed rule does not stop the sweep + + +# --- scan_rule wiring ------------------------------------------------------------- + + +def test_scan_rule_threads_baseline_into_known_unmapped_and_records(monkeypatch): + view = [(_ts(0), "parsed", _doc()), (_ts(1), "parsed", _doc(ip="SRC=1.2.3.4"))] + baseline = {"src_endpoint.ip": {"null_rate": 0.0, "shape_dist": {"ipv4": 1.0}}, + "unmapped.IN": {"null_rate": 0.0, "shape_dist": {}}} + captured, recorded, enforced = {}, [], [] + + monkeypatch.setattr(app, "connect_db", lambda url: FakeStoreConn()) + monkeypatch.setattr(app, "fetch_current_view", lambda conn, fp, v: view) + monkeypatch.setattr(app, "baseline_for", lambda conn, fp, v: baseline) + monkeypatch.setattr(app, "first_window_unmapped", + lambda conn, fp, v: {"IN", "OUT"}) + + real_compute = app.compute_windows + + def spy(rows, **kw): + captured.update(kw) + return real_compute(rows, **kw) + + monkeypatch.setattr(app, "compute_windows", spy) + monkeypatch.setattr(app, "record_windows", + lambda conn, rule, findings, win: recorded.append(findings)) + monkeypatch.setattr(app, "enforce_findings", + lambda conn, rule, findings: enforced.append(findings)) + + n = app.scan_rule(_fast_cfg(window_count=2), _rule()) + assert n == 1 # one closed window + # Rule context threaded into the pure window math: the rule's mapped + # targets, and the baseline's first-window keys (NOT the raw baseline). + assert captured["mapped_paths"] == ("src_endpoint.ip",) + assert captured["known_unmapped"] == {"IN", "OUT"} + assert captured["count_window"] == 2 and captured["time_window_s"] == 600 + (findings,) = recorded + fields = [f.field for f in findings] + assert "src_endpoint.ip" in fields and "__rule__" in fields + # 1 violation of 2 parsed docs -> violation_rate 0.5 -> tier-1 'severe' + # -> rule_deactivated, and the scan's enforcement saw that finding. + ip = next(f for f in findings if f.field == "src_endpoint.ip") + assert ip.stats["violation_rate"] == 0.5 + assert ip.severity == "severe" and ip.action == "rule_deactivated" + assert enforced == [findings] + + +def test_scan_rule_without_baseline_passes_known_none(monkeypatch): + captured = {} + monkeypatch.setattr(app, "connect_db", lambda url: FakeStoreConn()) + monkeypatch.setattr(app, "fetch_current_view", lambda conn, fp, v: []) + monkeypatch.setattr(app, "baseline_for", lambda conn, fp, v: None) + monkeypatch.setattr(app, "first_window_unmapped", + lambda conn, fp, v: (_ for _ in ()).throw( + AssertionError("must not run pre-baseline"))) + real_compute = app.compute_windows + + def spy(rows, **kw): + captured.update(kw) + return real_compute(rows, **kw) + + monkeypatch.setattr(app, "compute_windows", spy) + + assert app.scan_rule(_fast_cfg(), _rule()) == 0 # no rows -> no windows + assert captured["known_unmapped"] is None # pre-baseline: keys recorded, none new + + +def test_scan_rule_holds_back_trailing_partial(monkeypatch): + # 3 fresh rows with window_count=10: nothing is provably closed, so + # record_windows never runs. + view = [(_ts(i), "parsed", _doc()) for i in range(3)] + recorded = [] + monkeypatch.setattr(app, "connect_db", lambda url: FakeStoreConn()) + monkeypatch.setattr(app, "fetch_current_view", lambda conn, fp, v: view) + monkeypatch.setattr(app, "baseline_for", lambda conn, fp, v: None) + monkeypatch.setattr(app, "record_windows", + lambda conn, rule, findings, win: recorded.append(findings)) + + assert app.scan_rule(_fast_cfg(), _rule()) == 0 + assert recorded == [] + + +# --- scan_rule pipeline: the frozen Task 5 order ---------------------------------- + + +def test_scan_rule_runs_scan_evaluate_record_baseline_enforce_in_order(monkeypatch): + view = [(_ts(i * 10), "parsed", _doc()) for i in range(4)] # 2 windows of 2 + history = [(_ts(0), "__rule__", 2, None, 1.0, None), + (_ts(20), "__rule__", 2, None, 1.0, None)] + order = [] + + real_compute, real_evaluate = app.compute_windows, app.evaluate_window + + def compute_spy(rows, **kw): + order.append("windows") + return real_compute(rows, **kw) + + def evaluate_spy(win, baseline, *, rule_quarantined): + order.append("evaluate") + return real_evaluate(win, baseline, rule_quarantined=rule_quarantined) + + monkeypatch.setattr(app, "connect_db", lambda url: FakeStoreConn()) + monkeypatch.setattr(app, "fetch_current_view", + lambda conn, fp, v: (order.append("scan"), view)[1]) + monkeypatch.setattr(app, "baseline_for", + lambda conn, fp, v: (order.append("baseline-read"), None)[1]) + monkeypatch.setattr(app, "first_window_unmapped", + lambda conn, fp, v: (_ for _ in ()).throw( + AssertionError("must not run pre-baseline"))) + monkeypatch.setattr(app, "compute_windows", compute_spy) + monkeypatch.setattr(app, "evaluate_window", evaluate_spy) + monkeypatch.setattr(app, "record_windows", + lambda conn, rule, findings, win: order.append("record")) + monkeypatch.setattr(app, "fetch_window_history", lambda conn, fp, v: history) + monkeypatch.setattr(app, "insert_baseline", + lambda conn, fp, v, profiles, n: + (order.append("baseline-insert"), True)[1]) + monkeypatch.setattr(app, "enforce_findings", + lambda conn, rule, findings: order.append("enforce")) + + assert app.scan_rule(_fast_cfg(window_count=2, baseline_windows=2), _rule()) == 2 + # evaluate+record run per window; the FIRST baseline insert comes after + # all windows are recorded; enforcement is last, worst-first. + assert order == ["scan", "baseline-read", "windows", "evaluate", "record", + "evaluate", "record", "baseline-insert", "enforce"] + + +def test_scan_rule_establishes_baseline_exactly_at_the_nth_window(monkeypatch): + # 20 rows / window_count 2 -> the 10th window closes on THIS scan: the + # profile INSERTs once, windows_seen == 10, aggregates from the history. + view = [(_ts(i * 10), "parsed", _doc()) for i in range(20)] + history = [(_ts(i * 10), "src_endpoint.ip", 2, 0.0, None, {"ipv4": 2}) + for i in range(0, 20, 2)] + history += [(_ts(i * 10), "__rule__", 2, None, 1.0, None) + for i in range(0, 20, 2)] + inserts = [] + + monkeypatch.setattr(app, "connect_db", lambda url: FakeStoreConn()) + monkeypatch.setattr(app, "fetch_current_view", lambda conn, fp, v: view) + monkeypatch.setattr(app, "baseline_for", lambda conn, fp, v: None) + monkeypatch.setattr(app, "first_window_unmapped", + lambda conn, fp, v: (_ for _ in ()).throw( + AssertionError("must not run pre-baseline"))) + monkeypatch.setattr(app, "record_windows", + lambda conn, rule, findings, win: None) + monkeypatch.setattr(app, "fetch_window_history", lambda conn, fp, v: history) + monkeypatch.setattr(app, "insert_baseline", + lambda conn, fp, v, profiles, n: + inserts.append((fp, v, profiles, n)) or True) + + assert app.scan_rule(_fast_cfg(window_count=2, baseline_windows=10), + _rule()) == 10 + assert len(inserts) == 1 # exactly one insert, this scan + fp, version, profiles, windows_seen = inserts[0] + assert (fp, version, windows_seen) == ("fp_a", 2, 10) + assert set(profiles) == {"src_endpoint.ip", "__rule__"} + assert profiles["src_endpoint.ip"] == {"null_rate": 0.0, + "shape_dist": {"ipv4": 1.0}} + assert profiles["__rule__"]["events_count"] == {"min": 2, "max": 2} + + +def test_scan_rule_below_n_windows_never_inserts_a_baseline(monkeypatch): + view = [(_ts(i * 10), "parsed", _doc()) for i in range(20)] + history = [(_ts(i * 10), "__rule__", 2, None, 1.0, None) + for i in range(0, 20, 2)] # 10 windows, N demands 11 + inserts = [] + monkeypatch.setattr(app, "connect_db", lambda url: FakeStoreConn()) + monkeypatch.setattr(app, "fetch_current_view", lambda conn, fp, v: view) + monkeypatch.setattr(app, "baseline_for", lambda conn, fp, v: None) + monkeypatch.setattr(app, "record_windows", + lambda conn, rule, findings, win: None) + monkeypatch.setattr(app, "fetch_window_history", lambda conn, fp, v: history) + monkeypatch.setattr(app, "insert_baseline", + lambda conn, fp, v, profiles, n: + inserts.append((fp, v, profiles, n)) or True) + + app.scan_rule(_fast_cfg(window_count=2, baseline_windows=11), _rule()) + assert inserts == [] # not due: 10 < 11 + + +def test_scan_rule_post_baseline_never_reinserts_and_runs_tier2(monkeypatch): + # The 11th+ scan: baseline_for returns the profile, so establishment never + # runs, evaluate receives the profile (tier-2 armed) plus the rule's + # quarantine list, and enforcement still sees every finding. + baseline = {"src_endpoint.ip": {"null_rate": 0.0, "shape_dist": {"ipv4": 1.0}}, + "__rule__": {"match_rate": 1.0, + "events_count": {"min": 1, "max": 4}}} + view = [(_ts(i), "parsed", _doc()) for i in range(4)] + evaluated, enforced = [], [] + + real_evaluate = app.evaluate_window + + def evaluate_spy(win, baseline_arg, *, rule_quarantined): + evaluated.append((baseline_arg, rule_quarantined)) + return real_evaluate(win, baseline_arg, rule_quarantined=rule_quarantined) + + monkeypatch.setattr(app, "connect_db", lambda url: FakeStoreConn()) + monkeypatch.setattr(app, "fetch_current_view", lambda conn, fp, v: view) + monkeypatch.setattr(app, "baseline_for", lambda conn, fp, v: baseline) + monkeypatch.setattr(app, "first_window_unmapped", lambda conn, fp, v: set()) + monkeypatch.setattr(app, "evaluate_window", evaluate_spy) + monkeypatch.setattr(app, "record_windows", + lambda conn, rule, findings, win: None) + monkeypatch.setattr(app, "establish_baseline", + lambda conn, cfg, rule: (_ for _ in ()).throw( + AssertionError("establishment must not re-run"))) + monkeypatch.setattr(app, "enforce_findings", + lambda conn, rule, findings: enforced.append(findings)) + + assert app.scan_rule(_fast_cfg(window_count=2), _rule()) == 2 + assert all(b is baseline and q == [] for b, q in evaluated) + assert len(enforced) == 1 and len(enforced[0]) == 4 # 2 windows x 2 fields + + +def test_scan_rule_version_bump_is_tier1_only_then_rebases(monkeypatch): + # A new (fingerprint, version) has no profile: evaluate runs tier-1-only + # (baseline None), establishment is attempted (not due on empty history), + # and enforcement still runs — tier-1 fires from event zero. + view = [(_ts(0), "parsed", _doc()), (_ts(1), "parsed", _doc())] + evaluated, enforced, inserts = [], [], [] + + real_evaluate = app.evaluate_window + + def evaluate_spy(win, baseline_arg, *, rule_quarantined): + evaluated.append(baseline_arg) + return real_evaluate(win, baseline_arg, rule_quarantined=rule_quarantined) + + monkeypatch.setattr(app, "connect_db", lambda url: FakeStoreConn()) + monkeypatch.setattr(app, "fetch_current_view", lambda conn, fp, v: view) + monkeypatch.setattr(app, "baseline_for", lambda conn, fp, v: None) + monkeypatch.setattr(app, "first_window_unmapped", + lambda conn, fp, v: (_ for _ in ()).throw( + AssertionError("must not run pre-baseline"))) + monkeypatch.setattr(app, "evaluate_window", evaluate_spy) + monkeypatch.setattr(app, "record_windows", + lambda conn, rule, findings, win: None) + monkeypatch.setattr(app, "fetch_window_history", lambda conn, fp, v: []) + monkeypatch.setattr(app, "insert_baseline", + lambda conn, fp, v, profiles, n: + inserts.append(n) or True) + monkeypatch.setattr(app, "enforce_findings", + lambda conn, rule, findings: enforced.append(findings)) + + assert app.scan_rule(_fast_cfg(window_count=2), _rule(version=5)) == 1 + assert evaluated == [None] # tier-1-only phase + assert inserts == [] # nothing to aggregate yet + assert len(enforced) == 1 # enforcement ran anyway + + +# --- main(): ONE loop, single-threaded ------------------------------------------- + + +def test_main_runs_the_single_loop_in_the_main_thread(monkeypatch): + invoked = [] + + def fake_loop(cfg, stop): + invoked.append((cfg, stop, threading.current_thread() is threading.main_thread())) + + monkeypatch.setattr(app, "run_drift_loop", fake_loop) + app.main() + + assert len(invoked) == 1 # ONE loop (controller ruling), no daemon threads + cfg, stop, in_main = invoked[0] + assert isinstance(cfg, Config) and isinstance(stop, threading.Event) + assert in_main diff --git a/services/drift/tests/test_config.py b/services/drift/tests/test_config.py new file mode 100644 index 0000000..e82b906 --- /dev/null +++ b/services/drift/tests/test_config.py @@ -0,0 +1,34 @@ +# services/drift/tests/test_config.py — pins the binding env-knob ruling +# (deploy/.env.example names, compose-internal drift_role DSN, defaults equal +# to the .env.example values). +from drift.config import Config + + +def test_defaults_match_the_env_example_ruling(): + cfg = Config() + assert cfg.database_url == "postgresql://drift_role:drift_dev@postgres:5432/ulpf" + assert cfg.window_count == 1000 + assert cfg.window_time_s == 600 + assert cfg.baseline_windows == 10 + assert cfg.poll_s == 5.0 + + +def test_from_env_reads_exactly_the_five_compose_names(monkeypatch): + monkeypatch.setenv("DATABASE_URL", "postgresql://x:x@db:5432/ulpf") + monkeypatch.setenv("ULPF_DRIFT_WINDOW_COUNT", "2500") + monkeypatch.setenv("ULPF_DRIFT_WINDOW_TIME_S", "300") + monkeypatch.setenv("ULPF_DRIFT_BASELINE_WINDOWS", "20") + monkeypatch.setenv("ULPF_DRIFT_POLL_S", "0.5") + cfg = Config.from_env() + assert cfg.database_url == "postgresql://x:x@db:5432/ulpf" + assert cfg.window_count == 2500 + assert cfg.window_time_s == 300 + assert cfg.baseline_windows == 20 + assert cfg.poll_s == 0.5 + + +def test_from_env_without_env_keeps_defaults(monkeypatch): + for name in ("DATABASE_URL", "ULPF_DRIFT_WINDOW_COUNT", "ULPF_DRIFT_WINDOW_TIME_S", + "ULPF_DRIFT_BASELINE_WINDOWS", "ULPF_DRIFT_POLL_S"): + monkeypatch.delenv(name, raising=False) + assert Config.from_env() == Config() diff --git a/services/drift/tests/test_detect.py b/services/drift/tests/test_detect.py new file mode 100644 index 0000000..a49abce --- /dev/null +++ b/services/drift/tests/test_detect.py @@ -0,0 +1,470 @@ +# services/drift/tests/test_detect.py — Window -> drift_windows row assembly +# (tier-1 severity, __rule__ match_rate, the EMPTY HISTOGRAM GUARD from Task 3's +# review) PLUS Task 5's evaluate_window / merge_findings / aggregate_baseline +# (tier-2 ladders, graduated actions, baseline aggregation). Pure functions + +# monkeypatched shape_histogram; NEVER a live DB. +from datetime import UTC, datetime, timedelta + +import pytest +from ulpf_core.models import Mapping + +from drift import detect +from drift.detect import ( + RULE_FIELD, + Finding, + aggregate_baseline, + evaluate_window, + merge_findings, + shape_dist, + window_rows, +) +from drift.store import ActiveDriftRule +from drift.windows import FieldStats, Window, compute_windows + + +def _ts(seconds: float) -> datetime: + return datetime(2026, 9, 21, 10, 0, 0, tzinfo=UTC) + timedelta(seconds=seconds) + + +def _doc(ip="198.51.100.4", unmapped=None): + return {"src_endpoint": {"ip": ip, "port": 443}, "severity_id": 3, + "time": "2026-09-19T10:00:03Z", + "unmapped": unmapped if unmapped is not None else {}} + + +def _rule(): + return ActiveDriftRule( + id=7, fingerprint_id="fp_a", version=2, pattern=r"x", + mappings=[Mapping(source_field="SRC", ocsf_path="src_endpoint.ip")], + provenance="human", quarantined_fields=["dst_endpoint.ip"]) + + +def _windows(rows, rule=None, **kw): + rule = rule or _rule() + return compute_windows(rows, count_window=len(rows), time_window_s=600, + mapped_paths=tuple(m.ocsf_path for m in rule.mappings), + **kw) + + +# --- __rule__ sentinel row: match_rate = parsed / (parsed + parse_errors) ------ + + +def test_rule_row_carries_match_rate(): + rows = ([(_ts(i), "parsed", _doc()) for i in range(3)] + + [(_ts(3), "parse_error", None)]) + (rule_row,) = [r for r in window_rows(_rule(), _windows(rows)) if r[2] == RULE_FIELD] + fp, version, field, start, end, count, null_rate, match_rate, violation_rate, \ + shape, severity, action = rule_row + assert (fp, version, field) == ("fp_a", 2, "__rule__") + assert count == 4 and start == _ts(0) and end == _ts(3) + assert match_rate == pytest.approx(3 / 4) + # No rates, no shape, no tier-1 on the sentinel; the match-rate severity + # ladder needs a baseline (Task 5's evaluate) — recorded 'none' for now. + assert null_rate is None and violation_rate is None and shape is None + assert severity == "none" and action is None + + +def test_rule_row_zero_parsed_docs_match_rate_is_zero(): + rows = [(_ts(0), "parse_error", None), (_ts(1), "parse_error", None)] + (rule_row,) = window_rows(_rule(), _windows(rows)) + assert rule_row[7] == 0.0 + + +def test_rule_row_match_rate_none_when_no_denominator(): + # Defensive: a row with neither parsed nor parse_error status cannot happen + # in the current view (unparsed rows carry rule_version 0), but the rate + # degrades to NULL instead of dividing by zero. + from drift.windows import Window + synthetic = Window(start=_ts(0), end=_ts(0), count=1, parsed=0, parse_errors=0, + field_stats={}, new_unmapped=set()) + (rule_row,) = window_rows(_rule(), [synthetic]) + assert rule_row[7] is None + + +# --- mapped field rows: tier-1 severity ladder --------------------------------- + + +def test_mapped_field_row_tier1_severity_and_rates(): + # 1 violation of 4 parsed docs = 0.25 -> tier1_severity ladder 'moderate'. + rows = ([(_ts(i), "parsed", _doc("SRC=198.51.100.4")) for i in (0,)] + + [(_ts(i), "parsed", _doc()) for i in (1, 2, 3)]) + (ip_row,) = [r for r in window_rows(_rule(), _windows(rows)) + if r[2] == "src_endpoint.ip"] + assert ip_row[5] == 4 # events_count + assert ip_row[6] == 0.0 # null_rate = 0/4 + assert ip_row[7] is None # match_rate is __rule__-only + assert ip_row[8] == pytest.approx(0.25) # violation_rate = 1/4 + assert ip_row[9] == {"ipv4": 3, "free_text": 1} # shape over non-null values + # (the violating SRC=... garbage classifies free_text — violators still bin) + assert ip_row[10] == "moderate" # tier1_severity(0.25) + assert ip_row[11] is None # action_taken: Task 5 enforcement + + +def test_mapped_field_row_zero_violations_is_none_severity(): + rows = [(_ts(i), "parsed", _doc()) for i in range(2)] + (ip_row,) = [r for r in window_rows(_rule(), _windows(rows)) + if r[2] == "src_endpoint.ip"] + assert ip_row[8] == 0.0 and ip_row[10] == "none" + + +def test_mapped_field_row_null_rate_counts_nulls(): + docs = [(_ts(0), "parsed", _doc(None)), (_ts(1), "parsed", _doc())] + (ip_row,) = [r for r in window_rows(_rule(), _windows(docs)) + if r[2] == "src_endpoint.ip"] + assert ip_row[6] == pytest.approx(0.5) + assert ip_row[8] == 0.0 # the null is not a violation + + +# --- unmapped.* rows: recorded always, no tier-1 invariant ---------------------- + + +def test_unmapped_field_rows_have_no_violation_rate(): + rows = [(_ts(0), "parsed", _doc(unmapped={"IN": "eth0"})), + (_ts(1), "parsed", _doc(unmapped={"IN": "eth1"}))] + (in_row,) = [r for r in window_rows(_rule(), _windows(rows)) + if r[2] == "unmapped.IN"] + assert in_row[6] == 0.0 # null_rate still measured + assert in_row[8] is None # no tier-1 invariant on unmapped + assert in_row[9] == {"free_text": 2} # shape still measured + assert in_row[10] == "none" # findings stay alert-only (Task 5) + + +# --- EMPTY HISTOGRAM GUARD (Task 3 review carry, binding) ----------------------- + + +def test_zero_parsed_window_emits_only_the_rule_row(): + # A window with zero parsed docs skips shape comparison ENTIRELY — silence + # is the volume ladder's job (Task 5), never a shape comparison on {}. + rows = [(_ts(i), "parse_error", None) for i in range(3)] + assert window_rows(_rule(), _windows(rows)) == [ + ("fp_a", 2, "__rule__", _ts(0), _ts(2), 3, + None, 0.0, None, None, "none", None)] + + +def test_shape_dist_never_runs_on_an_empty_histogram(monkeypatch): + # The guard itself: an empty value list short-circuits to {} BEFORE the + # shape machinery (and therefore before any js_divergence caller in Task 5) + # ever sees it. + def boom(values): + raise AssertionError("shape machinery called on an empty histogram") + + monkeypatch.setattr(detect, "shape_histogram", boom) + assert shape_dist(FieldStats(nulls=3, violations=0, values=[])) == {} + + +def test_shape_dist_computes_histogram_from_values(): + stats = FieldStats(nulls=0, violations=0, values=["1.2.3.4", "DROP", 7]) + assert shape_dist(stats) == {"ipv4": 1, "free_text": 1, "int": 1} + + +# --- row shape: rule identity, determinism -------------------------------------- + + +def test_rows_are_deterministic_for_the_same_input(): + rows = ([(_ts(0), "parsed", _doc("SRC=1.2.3.4"))] + + [(_ts(i), "parsed", _doc()) for i in (1, 2)]) + assert (window_rows(_rule(), _windows(rows)) + == window_rows(_rule(), _windows(rows))) + + +def test_rows_order_is_mapped_then_sorted_unmapped_then_rule(): + rows = [(_ts(0), "parsed", _doc(unmapped={"Z": "1", "A": "2"}))] + fields = [r[2] for r in window_rows(_rule(), _windows(rows))] + assert fields == ["src_endpoint.ip", "unmapped.A", "unmapped.Z", "__rule__"] + + +# --- Task 5: evaluate_window ------------------------------------------------------ + + +def _win(parsed=10, errors=0, stats=None, new=(), start=None): + """Synthetic Window (parsed>0 keeps field rows; the FieldStats are the + test's chosen measurement, no docs needed — evaluate is pure over stats).""" + return Window(start=start or _ts(0), end=_ts(60), count=parsed + errors, + parsed=parsed, parse_errors=errors, + field_stats=stats or {}, new_unmapped=set(new)) + + +def _ip_stats(nulls=0, violations=0, values=None): + return FieldStats(nulls=nulls, violations=violations, + values=values if values is not None else ["198.51.100.7"]) + + +def _finding_for(win, baseline, field_name, *, quarantined=()): + """The one finding for field_name out of the window's evaluation.""" + return next(f for f in evaluate_window(win, baseline, rule_quarantined=quarantined) + if f.field == field_name) + + +def test_evaluate_tier1_only_without_baseline_ladder_and_actions(): + # violation_rate 0.1 -> tier-1 'minor' -> alert; decide_action semantics. + win = _win(stats={"src_endpoint.ip": _ip_stats(violations=1)}) + finding = _finding_for(win, None, "src_endpoint.ip") + assert (finding.field, finding.severity, finding.action) == ( + "src_endpoint.ip", "minor", "alert") + assert finding.stats["violation_rate"] == pytest.approx(0.1) + assert finding.stats["null_rate"] == 0.0 and finding.stats["events_count"] == 10 + assert finding.window_start == win.start + + +def test_evaluate_tier1_moderate_quarantines_and_severe_deactivates(): + # 0.2 violation-rate -> moderate -> field_quarantined; 0.5 -> severe -> + # rule_deactivated (the full §7.2 ladder through tier-1 alone). + stats_mod = _win(stats={"src_endpoint.ip": _ip_stats(violations=2)}) + assert _finding_for(stats_mod, None, "src_endpoint.ip").action \ + == "field_quarantined" + stats_sev = _win(stats={"src_endpoint.ip": _ip_stats(violations=5)}) + f = _finding_for(stats_sev, None, "src_endpoint.ip") + assert f.severity == "severe" and f.action == "rule_deactivated" + + +def test_evaluate_tier1_only_when_field_missing_from_baseline(): + # A field with no profile in the baseline dict (e.g. never observed during + # the baseline windows) degrades to tier-1-only, never KeyError. + baseline = {"other.field": {"null_rate": 0.0, "shape_dist": {"ipv4": 1.0}}} + win = _win(stats={"src_endpoint.ip": _ip_stats()}) + finding = _finding_for(win, baseline, "src_endpoint.ip") + assert finding.severity == "none" and finding.action is None + + +def test_evaluate_tier2_null_rate_escalates_over_quiet_tier1(): + # null 0.0 -> 0.2 against baseline 0.01: ratio 20, diff 0.19 -> moderate. + baseline = {"src_endpoint.ip": {"null_rate": 0.01, "shape_dist": {"ipv4": 1.0}}} + win = _win(stats={"src_endpoint.ip": _ip_stats( + nulls=2, values=["198.51.100.7"] * 8)}) # null_rate 0.2, shape identical + finding = _finding_for(win, baseline, "src_endpoint.ip") + assert finding.severity == "moderate" and finding.action == "field_quarantined" + + +def test_evaluate_tier2_shape_divergence_alone_reaches_severe(): + # A field with NO tier-1 invariant (shape is its only signal): 1000 + # free-text values against an all-ipv4 baseline rescale to count parity + # ({free_text: 1000} vs {ipv4: 1000}) -> T3's disjoint-count divergence + # ~0.693 -> severe -> rule_deactivated. + baseline = {"metadata.note": {"null_rate": 0.0, "shape_dist": {"ipv4": 1.0}}} + win = _win(stats={"metadata.note": FieldStats(nulls=0, violations=0, + values=["DROP"] * 1000)}) + finding = _finding_for(win, baseline, "metadata.note") + assert finding.severity == "severe" and finding.action == "rule_deactivated" + + +def test_evaluate_tier2_shape_rescales_baseline_to_window_count(monkeypatch): + # The stored profile is normalized, but js_divergence must see it at the + # window's count scale: {ipv4: 1.0} x 4 observed values -> {ipv4: 4.0}. + captured = {} + + def spy(p, q): + captured.update(p=p, q=q) + return 0.0 + + monkeypatch.setattr(detect, "js_divergence", spy) + baseline = {"src_endpoint.ip": {"null_rate": 0.0, "shape_dist": {"ipv4": 1.0}}} + win = _win(stats={"src_endpoint.ip": _ip_stats(values=["198.51.100.7"] * 4)}) + _finding_for(win, baseline, "src_endpoint.ip") + assert captured["p"] == {"ipv4": 4} and captured["q"] == {"ipv4": 4.0} + + +def test_evaluate_severity_is_max_across_signals(): + # tier-1 minor (0.1 violations) + tier-2 null moderate (0.2 vs 0.01): + # the field's severity is the MAX — moderate — not either alone. + baseline = {"src_endpoint.ip": {"null_rate": 0.01, "shape_dist": {"ipv4": 1.0}}} + win = _win(stats={"src_endpoint.ip": _ip_stats(nulls=2, violations=1, + values=["198.51.100.7"] * 7)}) + finding = _finding_for(win, baseline, "src_endpoint.ip") + assert finding.severity == "moderate" and finding.action == "field_quarantined" + + +def test_evaluate_quarantined_field_still_measured_and_skips_requarantine(): + # Already-quarantined field sitting at moderate: STILL measured (the + # finding exists, severity recorded) but the quarantine action is an + # idempotent skip — decide_action('moderate') is suppressed to None. + baseline = {"src_endpoint.ip": {"null_rate": 0.01, "shape_dist": {"ipv4": 1.0}}} + win = _win(stats={"src_endpoint.ip": _ip_stats(nulls=2, + values=["198.51.100.7"] * 8)}) + finding = _finding_for(win, baseline, "src_endpoint.ip", + quarantined=["src_endpoint.ip"]) + assert finding.severity == "moderate" and finding.action is None + + +def test_evaluate_quarantined_field_worsening_to_severe_deactivates(): + # The binding escalation rule: a quarantined field going WORSE must reach + # rule_deactivated — quarantine never hides the field. + baseline = {"src_endpoint.ip": {"null_rate": 0.0, "shape_dist": {"ipv4": 1.0}}} + win = _win(stats={"src_endpoint.ip": _ip_stats(nulls=5, + values=["198.51.100.7"] * 5)}) + finding = _finding_for(win, baseline, "src_endpoint.ip", + quarantined=["src_endpoint.ip"]) + assert finding.severity == "severe" and finding.action == "rule_deactivated" + + +def test_evaluate_never_shape_compares_an_empty_current_histogram(monkeypatch): + # Task 3/4 review carry (binding): a window with no observed values + # short-circuits shape comparison entirely — js_divergence must never see + # an empty histogram. The baseline here is an ALWAYS-NULL field (null + # ladder quiet), so only a broken shape path could fire. + def boom(p, q): + raise AssertionError("js_divergence called on an empty histogram") + + monkeypatch.setattr(detect, "js_divergence", boom) + baseline = {"src_endpoint.ip": {"null_rate": 1.0, "shape_dist": {}}} + win = _win(stats={"src_endpoint.ip": _ip_stats(nulls=10, values=[])}) + finding = _finding_for(win, baseline, "src_endpoint.ip") + assert finding.severity == "none" # null 1.0 vs baseline 1.0: ratio 1.0 + assert finding.stats["shape_dist"] == {} + + +def test_evaluate_rule_row_pre_baseline_is_none(): + win = _win(errors=0, stats={}) # parsed 10, match_rate 1.0, no baseline yet + (finding,) = evaluate_window(win, None, rule_quarantined=[]) + assert finding.field == RULE_FIELD + assert finding.severity == "none" and finding.action is None + assert finding.stats == {"events_count": 10, "match_rate": 1.0} + + +def test_evaluate_rule_row_match_rate_collapse_deactivates(): + # match 1.0 -> 0.3 = 0.7 drop -> severe; volume inside bounds stays none. + baseline = {RULE_FIELD: {"match_rate": 1.0, + "events_count": {"min": 5, "max": 20}}} + win = _win(parsed=3, errors=7, stats={}) + (finding,) = evaluate_window(win, baseline, rule_quarantined=[]) + assert finding.severity == "severe" and finding.action == "rule_deactivated" + assert finding.stats["match_rate"] == pytest.approx(0.3) + + +def test_evaluate_rule_row_volume_outside_bounds_is_minor_alert(): + # count 25 outside [5, 20] -> volume 'minor'; match unchanged -> none. + baseline = {RULE_FIELD: {"match_rate": 1.0, + "events_count": {"min": 5, "max": 20}}} + win = _win(parsed=25, errors=0, stats={}) + (finding,) = evaluate_window(win, baseline, rule_quarantined=[]) + assert finding.severity == "minor" and finding.action == "alert" + + +def test_evaluate_rule_row_moderate_downgrades_quarantine_to_alert(): + # The __rule__ sentinel is NOT a quarantinable field: a moderate match drop + # (1.0 -> 0.7, inside volume bounds) alerts — only severe deactivates. + baseline = {RULE_FIELD: {"match_rate": 1.0, + "events_count": {"min": 5, "max": 20}}} + win = _win(parsed=7, errors=3, stats={}) # match 0.7 -> 0.3 drop: moderate + (finding,) = evaluate_window(win, baseline, rule_quarantined=[]) + assert finding.severity == "moderate" and finding.action == "alert" + + +def test_evaluate_unmapped_new_key_is_minor_alert_only(): + # decide_action is BYPASSED for unmapped.*: a brand-new key surfaces as + # minor/alert (the extension-opportunity card), never quarantine/deactivate. + win = _win(stats={"unmapped.NEW": FieldStats(nulls=0, violations=0, + values=["eth9"])}, + new={"NEW"}) + finding = _finding_for(win, None, "unmapped.NEW") + assert (finding.field, finding.severity, finding.action) == ( + "unmapped.NEW", "minor", "alert") + + +def test_evaluate_unmapped_known_key_stays_none_even_with_terrible_rates(): + # Binding ruling: unmapped rows are new-key-emergence signals ONLY — null + # rates are never re-derived into severity (10/10 nulls here stay 'none'), + # and even a (defensive) baseline profile for an unmapped field is ignored. + baseline = {"unmapped.IN": {"null_rate": 0.0, "shape_dist": {"ipv4": 1.0}}} + win = _win(stats={"unmapped.IN": FieldStats(nulls=10, violations=0, + values=[])}, + new=set()) + finding = _finding_for(win, baseline, "unmapped.IN") + assert finding.severity == "none" and finding.action is None + assert finding.stats["null_rate"] == 1.0 # measured + recorded, never acted on + + +def test_evaluate_zero_parsed_window_emits_only_the_rule_finding(): + win = _win(parsed=0, errors=3, stats={"src_endpoint.ip": _ip_stats()}) + findings = evaluate_window(win, None, rule_quarantined=[]) + assert [f.field for f in findings] == [RULE_FIELD] + + +def test_evaluate_emission_parity_with_window_rows(): + # record_windows overlays findings onto window_rows' measured rows, so the + # two must emit the SAME field set for the same window (mapped, unmapped, + # sentinel; zero-parsed case above). + rows = ([(_ts(i), "parsed", _doc()) for i in range(2)] + + [(_ts(2), "parsed", _doc(unmapped={"IN": "eth0", "Z": "1"}))]) + win = _windows(rows)[0] + measured = [r[2] for r in window_rows(_rule(), [win])] + evaluated = [f.field for f in evaluate_window(win, None, rule_quarantined=[])] + assert evaluated == measured + + +# --- Task 5: merge_findings (R-M3-11 per-field max) ------------------------------- + + +def test_merge_findings_takes_per_field_max_across_windows(): + findings = [ + Finding("src_endpoint.ip", "minor", "alert", {}, _ts(0)), + Finding("src_endpoint.ip", "moderate", "field_quarantined", {}, _ts(60)), + Finding("dst_endpoint.ip", "severe", "rule_deactivated", {}, _ts(60)), + Finding("dst_endpoint.ip", "none", None, {}, _ts(120)), + ] + assert merge_findings(findings) == { + "src_endpoint.ip": "moderate", "dst_endpoint.ip": "severe"} + + +def test_merge_findings_empty_is_empty(): + assert merge_findings([]) == {} + + +# --- Task 5: aggregate_baseline (the FIRST-N window aggregation) ------------------ + + +def _hist_row(start_s, field, count, null_rate, match_rate, shape): + return (_ts(start_s), field, count, null_rate, match_rate, shape) + + +def test_aggregate_baseline_matches_hand_computed_fixture(): + # Hand-computed: 3 windows, ip field nulls 0.1/0.2/0.3 -> mean 0.2; shapes + # summed {ipv4: 18, free_text: 12} -> normalized 0.6/0.4; __rule__ match + # 1.0/0.9/0.8 -> 0.9 with count bounds min 10 / max 14; unmapped excluded. + rows = [ + _hist_row(0, "src_endpoint.ip", 10, 0.1, None, {"ipv4": 8, "free_text": 2}), + _hist_row(0, RULE_FIELD, 10, None, 1.0, None), + _hist_row(60, "src_endpoint.ip", 12, 0.2, None, {"ipv4": 6, "free_text": 4}), + _hist_row(60, RULE_FIELD, 12, None, 0.9, None), + _hist_row(120, "src_endpoint.ip", 14, 0.3, None, {"ipv4": 4, "free_text": 6}), + _hist_row(120, RULE_FIELD, 14, None, 0.8, None), + _hist_row(120, "unmapped.X", 14, 0.5, None, {"free_text": 3}), + ] + profiles = aggregate_baseline(rows, baseline_windows=3) + + assert set(profiles) == {"src_endpoint.ip", RULE_FIELD} # unmapped skipped + ip = profiles["src_endpoint.ip"] + assert ip["null_rate"] == pytest.approx(0.2) + assert ip["shape_dist"]["ipv4"] == pytest.approx(0.6) + assert ip["shape_dist"]["free_text"] == pytest.approx(0.4) + assert sum(ip["shape_dist"].values()) == pytest.approx(1.0) + rule = profiles[RULE_FIELD] + assert rule["match_rate"] == pytest.approx(0.9) + assert rule["events_count"] == {"min": 10, "max": 14} + + +def test_aggregate_baseline_none_before_n_windows_exist(): + rows = [_hist_row(0, RULE_FIELD, 10, None, 1.0, None)] + assert aggregate_baseline(rows, baseline_windows=10) is None + assert aggregate_baseline([], baseline_windows=10) is None + + +def test_aggregate_baseline_uses_only_the_first_n_windows(): + # Windows 4 and 5 exist (null 0.9) but must NOT enter a 3-window baseline. + rows = ([_hist_row(i * 60, "src_endpoint.ip", 10 + i, 0.1 + 0.1 * i, None, + {"ipv4": 10}) + for i in range(3)] + + [_hist_row(240, "src_endpoint.ip", 99, 0.9, None, {"free_text": 10}), + _hist_row(300, "src_endpoint.ip", 99, 0.9, None, {"free_text": 10})]) + profiles = aggregate_baseline(rows, baseline_windows=3) + assert profiles["src_endpoint.ip"]["null_rate"] == pytest.approx(0.2) + assert profiles["src_endpoint.ip"]["shape_dist"] == {"ipv4": 1.0} + + +def test_aggregate_baseline_skips_none_match_rates_in_mean(): + rows = [ + _hist_row(0, RULE_FIELD, 10, None, 1.0, None), + _hist_row(60, RULE_FIELD, 10, None, 0.8, None), + _hist_row(120, RULE_FIELD, 10, None, None, None), # defensive NULL + ] + profiles = aggregate_baseline(rows, baseline_windows=3) + assert profiles[RULE_FIELD]["match_rate"] == pytest.approx(0.9) diff --git a/services/drift/tests/test_enforce.py b/services/drift/tests/test_enforce.py new file mode 100644 index 0000000..bc0d751 --- /dev/null +++ b/services/drift/tests/test_enforce.py @@ -0,0 +1,341 @@ +# services/drift/tests/test_enforce.py — Task 5 enforcement: quarantine / +# deactivate SQL (TOCTOU expected-status predicates + rowcount checks, the +# gateway writes.py discipline under drift_role's column-scoped grants), +# record_windows row assembly, and worst-first scan enforcement. Everything is +# pinned through a recording fake conn/cursor or monkeypatched collaborators; +# NEVER a live DB. +import json +from datetime import UTC, datetime, timedelta + +import pytest +from ulpf_core.models import Mapping + +from drift import enforce +from drift.detect import RULE_FIELD, Finding, evaluate_window +from drift.enforce import ( + deactivate_rule, + enforce_findings, + quarantine_field, + record_windows, +) +from drift.store import ActiveDriftRule +from drift.windows import FieldStats, Window + + +def _ts(seconds: float) -> datetime: + return datetime(2026, 9, 21, 10, 0, 0, tzinfo=UTC) + timedelta(seconds=seconds) + + +def _rule(fp="fp_a", rule_id=7, version=2, quarantined=()): + return ActiveDriftRule( + id=rule_id, fingerprint_id=fp, version=version, pattern=r"x", + mappings=[Mapping(source_field="SRC", ocsf_path="src_endpoint.ip")], + provenance="human", quarantined_fields=list(quarantined)) + + +class FakeConn: + """Fake psycopg connection: records SQL + params, replays scripted + rowcounts (drift enforcement's TOCTOU answers) and fetch results.""" + + def __init__(self, rowcounts=(), fetchone_results=(), fetchall_results=()): + self.sql: list = [] + self.params: list = [] + self.txns_opened = 0 + self._rowcounts = list(rowcounts) + self._fetchone = list(fetchone_results) + self._fetchall = list(fetchall_results) + + def transaction(self): + conn = self + + class Txn: + def __enter__(self): + conn.txns_opened += 1 + conn.sql.append("TXN-START") + conn.params.append(None) # keep sql/params index-aligned + + def __exit__(self, exc_type, exc, tb): + conn.sql.append("TXN-ROLLBACK" if exc_type is not None else "TXN-COMMIT") + conn.params.append(None) + return False + + return Txn() + + def cursor(self): + conn = self + + class Cur: + rowcount = None + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params=None): + conn.sql.append(sql) + conn.params.append(params) + self.rowcount = (conn._rowcounts.pop(0) + if conn._rowcounts else 1) + + def fetchone(self): + return list(conn._fetchone.pop(0)) if conn._fetchone else [1] + + def fetchall(self): + return list(conn._fetchall.pop(0)) if conn._fetchall else [] + + return Cur() + + +def _unwrap(value): + """psycopg's Json adapter -> plain Python (duck-typed: no psycopg here).""" + return value.obj if hasattr(value, "obj") else value + + +def _statements(conn): + return [(s, p) for s, p in zip(conn.sql, conn.params) if isinstance(s, str)] + + +# --- quarantine_field: array_append + NOT ANY + expected status ------------------- + + +def test_quarantine_field_sql_and_params(): + conn = FakeConn(rowcounts=[1]) + assert quarantine_field(conn, _rule(), "src_endpoint.ip", _ts(0)) is True + + (sql, params), = [sp for sp in _statements(conn) if sp[0].startswith("UPDATE")] + assert sql == ( + "UPDATE rules SET quarantined_fields = array_append(quarantined_fields, %s) " + "WHERE id = %s AND status = 'active' " + "AND NOT (%s = ANY(quarantined_fields))") + # (field for array_append, rule id, field again for the ANY guard) + assert params == ("src_endpoint.ip", 7, "src_endpoint.ip") + + +def test_quarantine_field_rowcount_zero_is_false_and_never_audited(): + # Already quarantined OR rule inactive: the WHERE matches nothing -> 0 + # rows -> False, idempotent, and NO audit row on the no-op. + conn = FakeConn(rowcounts=[0]) + assert quarantine_field(conn, _rule(), "src_endpoint.ip", _ts(0)) is False + assert not [s for s, _ in _statements(conn) if s.startswith("INSERT")] + assert conn.txns_opened == 1 # the UPDATE still ran inside its transaction + + +def test_quarantine_field_true_audits_with_actor_drift_and_detail(): + conn = FakeConn(rowcounts=[1]) + rule = _rule(rule_id=9, version=3) + assert quarantine_field(conn, rule, "dst_endpoint.ip", _ts(90)) is True + + audits = [(s, p) for s, p in _statements(conn) if s.startswith("INSERT INTO audit_log")] + (sql, params), = audits + assert sql == ("INSERT INTO audit_log (actor, action, entity, detail) " + "VALUES (%s, %s, %s, %s)") + assert params[0] == "drift" and params[1] == "field_quarantined" + assert params[2] == "fp_a" + assert _unwrap(params[3]) == {"field": "dst_endpoint.ip", "rule_id": 9, + "version": 3, "window_start": _ts(90).isoformat()} + # The detail is JSONB: every value must survive plain json.dumps — a raw + # datetime raised TypeError inside psycopg's Json() and rolled the paired + # mutation back (observed live in Task 12; fake conns never serialize, so + # this pin is the regression guard for that exact bug class). + json.dumps(_unwrap(params[3])) + # UPDATE and its audit pair inside ONE transaction (the writes.py pairing). + assert conn.txns_opened == 1 + assert conn.sql[0] == "TXN-START" and conn.sql[-1] == "TXN-COMMIT" + + +def test_quarantine_field_touches_only_the_granted_column(): + conn = FakeConn(rowcounts=[1]) + quarantine_field(conn, _rule(), "f", _ts(0)) + (sql, _), = [sp for sp in _statements(conn) if sp[0].startswith("UPDATE")] + # Only quarantined_fields is SET — migration 007 grants UPDATE on exactly + # that column (plus status/deactivated_at for deactivate). + assert sql.startswith("UPDATE rules SET quarantined_fields = array_append") + + +# --- deactivate_rule: expected-status predicate ----------------------------------- + + +def test_deactivate_rule_sql_expected_status_predicate(): + conn = FakeConn(rowcounts=[1]) + assert deactivate_rule(conn, _rule(), _ts(0)) is True + + (sql, params), = [sp for sp in _statements(conn) if sp[0].startswith("UPDATE")] + assert sql == ("UPDATE rules SET status = 'deactivated', deactivated_at = now() " + "WHERE id = %s AND status = 'active'") + assert params == (7,) + + +def test_deactivate_rule_false_on_non_active_and_never_audited(): + # The 409-equivalent under drift_role: status moved concurrently -> 0 rows + # -> False, no audit (a silent no-op that still audits would lie). + conn = FakeConn(rowcounts=[0]) + assert deactivate_rule(conn, _rule(), _ts(0)) is False + assert not [s for s, _ in _statements(conn) if s.startswith("INSERT")] + + +def test_deactivate_rule_true_audits_with_actor_drift(): + conn = FakeConn(rowcounts=[1]) + rule = _rule(rule_id=11, version=4) + assert deactivate_rule(conn, rule, _ts(30)) is True + + (_, params), = [(s, p) for s, p in _statements(conn) + if s.startswith("INSERT INTO audit_log")] + assert params[0] == "drift" and params[1] == "rule_deactivated" + assert params[2] == "fp_a" + # window_start rides as ISO-8601 (JSONB detail — see the quarantine test). + detail = _unwrap(params[3]) + assert detail == {"rule_id": 11, "version": 4, + "window_start": _ts(30).isoformat()} + json.dumps(detail) + + +# --- record_windows: findings overlaid onto the measured rows --------------------- + + +def _win(): + return Window(start=_ts(0), end=_ts(9), count=10, parsed=10, parse_errors=0, + field_stats={"src_endpoint.ip": FieldStats( + nulls=0, violations=3, values=["198.51.100.7"] * 7)}, + new_unmapped=set()) + + +def test_record_windows_writes_severity_and_action_taken(): + conn = FakeConn() + rule, win = _rule(), _win() + findings = evaluate_window(win, None, rule_quarantined=[]) + record_windows(conn, rule, findings, win) + + inserts = [(s, p) for s, p in _statements(conn) + if s.startswith("INSERT INTO drift_windows")] + assert len(inserts) == 2 # field row + __rule__ row + assert all("ON CONFLICT (fingerprint_id, rule_version, field, window_start) " + "DO NOTHING" in s for s, _ in inserts) + by_field = {p[2]: p for _, p in inserts} + ip = by_field["src_endpoint.ip"] + # 3 violations of 10 parsed = 0.3 -> tier-1 moderate -> field_quarantined. + assert ip[8] == pytest.approx(0.3) and ip[10] == "moderate" + assert ip[11] == "field_quarantined" + assert _unwrap(ip[9]) == {"ipv4": 7} + assert by_field[RULE_FIELD][10] == "none" and by_field[RULE_FIELD][11] is None + + +def test_record_windows_keeps_zero_parsed_and_unmapped_row_shapes(): + conn = FakeConn() + rule = _rule() + win = Window(start=_ts(0), end=_ts(2), count=3, parsed=3, parse_errors=0, + field_stats={"unmapped.NEW": FieldStats(nulls=0, violations=0, + values=["eth9"])}, + new_unmapped={"NEW"}) + record_windows(conn, rule, evaluate_window(win, None, rule_quarantined=[]), win) + + by_field = {p[2]: p for s, p in _statements(conn) + if s.startswith("INSERT INTO drift_windows")} + new = by_field["unmapped.NEW"] + assert new[8] is None # no tier-1 invariant on unmapped + assert new[10] == "minor" and new[11] == "alert" + + zero = Window(start=_ts(0), end=_ts(2), count=3, parsed=0, parse_errors=3, + field_stats={}, new_unmapped=set()) + conn2 = FakeConn() + record_windows(conn2, rule, evaluate_window(zero, None, rule_quarantined=[]), + zero) + fields = [p[2] for s, p in _statements(conn2) + if s.startswith("INSERT INTO drift_windows")] + assert fields == [RULE_FIELD] # zero-parsed: only the sentinel row + + +# --- enforce_findings: worst-first, one deactivate per scan ----------------------- + + +def _finding(field, severity, action, start=0): + return Finding(field, severity, action, {}, _ts(start)) + + +class _Spy: + def __init__(self, results=()): + self.calls = [] + self._results = list(results) + + def __call__(self, conn, rule, *args): + self.calls.append(args if args else rule.fingerprint_id) + return self._results.pop(0) if self._results else True + + +def test_enforce_findings_severe_first_skips_moderate_after_deactivate(monkeypatch): + deact, quar = _Spy(), _Spy() + monkeypatch.setattr(enforce, "deactivate_rule", deact) + monkeypatch.setattr(enforce, "quarantine_field", quar) + findings = [ + _finding("src_endpoint.ip", "moderate", "field_quarantined", 60), + _finding("dst_endpoint.ip", "severe", "rule_deactivated", 0), + ] + result = enforce_findings(FakeConn(), _rule(), findings) + assert result == {"deactivated": True, "quarantined": []} + assert len(deact.calls) == 1 and quar.calls == [] # worst-first: severe won + + +def test_enforce_findings_one_deactivate_per_scan(monkeypatch): + deact = _Spy() + monkeypatch.setattr(enforce, "deactivate_rule", deact) + findings = [ + _finding("src_endpoint.ip", "severe", "rule_deactivated", 0), + _finding(RULE_FIELD, "severe", "rule_deactivated", 60), + _finding("dst_endpoint.ip", "severe", "rule_deactivated", 120), + ] + result = enforce_findings(FakeConn(), _rule(), findings) + assert result["deactivated"] is True + assert len(deact.calls) == 1 # the first worst finding deactivates, rest skip + + +def test_enforce_findings_quarantines_each_moderate_field(monkeypatch): + quar = _Spy() + monkeypatch.setattr(enforce, "deactivate_rule", _Spy()) + monkeypatch.setattr(enforce, "quarantine_field", quar) + findings = [ + _finding("src_endpoint.ip", "moderate", "field_quarantined", 60), + _finding("dst_endpoint.ip", "moderate", "field_quarantined", 0), + ] + result = enforce_findings(FakeConn(), _rule(), findings) + assert result == {"deactivated": False, + "quarantined": ["dst_endpoint.ip", "src_endpoint.ip"]} + assert quar.calls == [("dst_endpoint.ip", _ts(0)), ("src_endpoint.ip", _ts(60))] + + +def test_enforce_findings_ignores_none_minor_and_none_actions(monkeypatch): + deact = _Spy() + quar = _Spy() + monkeypatch.setattr(enforce, "deactivate_rule", deact) + monkeypatch.setattr(enforce, "quarantine_field", quar) + findings = [ + _finding("src_endpoint.ip", "none", None), + _finding("unmapped.NEW", "minor", "alert"), + _finding("dst_endpoint.ip", "moderate", None), # already quarantined + ] + assert enforce_findings(FakeConn(), _rule(), findings) == { + "deactivated": False, "quarantined": []} + assert deact.calls == [] and quar.calls == [] + + +def test_enforce_findings_deactivate_false_still_ends_the_scan(monkeypatch): + # deactivate_rule False (rule no longer active, TOCTOU): remaining actions + # are skipped too — every later UPDATE would no-op anyway. + deact, quar = _Spy(results=[False]), _Spy() + monkeypatch.setattr(enforce, "deactivate_rule", deact) + monkeypatch.setattr(enforce, "quarantine_field", quar) + findings = [ + _finding("src_endpoint.ip", "severe", "rule_deactivated"), + _finding("dst_endpoint.ip", "moderate", "field_quarantined"), + ] + result = enforce_findings(FakeConn(), _rule(), findings) + assert result == {"deactivated": False, "quarantined": []} + assert quar.calls == [] + + +def test_enforce_findings_alerts_never_touch_the_db(monkeypatch): + monkeypatch.setattr(enforce, "deactivate_rule", _Spy()) + conn = FakeConn() + findings = [_finding("src_endpoint.ip", "minor", "alert")] + enforce_findings(conn, _rule(), findings) + assert _statements(conn) == [] # an alert IS the drift_windows row, no SQL diff --git a/services/drift/tests/test_migration_007_pin.py b/services/drift/tests/test_migration_007_pin.py new file mode 100644 index 0000000..569bc84 --- /dev/null +++ b/services/drift/tests/test_migration_007_pin.py @@ -0,0 +1,61 @@ +"""services/drift/tests/test_migration_007_pin.py — pins the FILE +deploy/migrations/007_drift_tables.sql, not service behavior (the M1 +migration-pin style: the SQL file is the contract, so any edit to its grants +or keys must consciously update this test too). + +Pinned, per R-M3-3's "the grant matrix is the design": (a) both column-level +GRANT UPDATE (quarantined_fields) lines — drift enforcement and the gateway's +human override; (b) natural composite PKs on both tables (R-M3-2's +INSERT-only, ON CONFLICT DO NOTHING idempotence); (c) no SERIAL-family token +anywhere in the SQL — the 004/005/006 sequence-grant bug class stays +structurally impossible, the one allowed serial context being the audit_log +sequence grant line (pinned verbatim below). +""" + +import re +from pathlib import Path + +# Repo root regardless of CWD: tests/ -> drift/ -> services/ -> root. +MIGRATION = ( + Path(__file__).resolve().parents[3] / "deploy" / "migrations" / "007_drift_tables.sql" +) + +# The only sequence context drift legitimately touches: audit_log's serial id +# backing sequence, granted to drift_role. +AUDIT_SEQUENCE_GRANT = "GRANT USAGE, SELECT ON SEQUENCE audit_log_id_seq TO drift_role;" + + +def test_migration_007_grants_keys_and_no_serials_pinned(): + """One pin over the whole file (a)/(b)/(c): + + (a) both column-level UPDATE grants on rules.quarantined_fields exist + verbatim — drift_role enforces (R-M3-3), rules_role is the gateway's + un-quarantine write path; losing either surfaces as a live + permissions error, not a test failure; + (b) both CREATE TABLE statements key on natural composite PKs derived + from data (R-M3-2) — a surrogate key would need a sequence grant and + break ON CONFLICT DO NOTHING re-scan idempotence; + (c) a case-insensitive serial grep over the file's SQL stays empty — + comment prose is not SQL (the header's '(no serials)' is stripped + before matching), and any surviving SERIAL/SMALLSERIAL/BIGSERIAL + token fails unless it sits on the pinned audit_log sequence grant + line, the 004/005/006 bug class locked out by construction. + """ + sql = MIGRATION.read_text(encoding="utf-8") + + # (a) the grant matrix + assert "GRANT UPDATE (quarantined_fields) ON rules TO drift_role;" in sql + assert "GRANT UPDATE (quarantined_fields) ON rules TO rules_role;" in sql + + # (b) natural composite PKs, verbatim clauses + assert "PRIMARY KEY (fingerprint_id, rule_version, field, window_start)" in sql + assert "PRIMARY KEY (fingerprint_id, rule_version, field)" in sql + + # (c) no serial tokens in SQL outside the audit sequence grant line + code = "\n".join(line.split("--", 1)[0] for line in sql.splitlines()) + offenders = [ + line.strip() + for line in code.splitlines() + if re.search(r"(?i)serial", line) and line.strip() != AUDIT_SEQUENCE_GRANT + ] + assert offenders == [] diff --git a/services/drift/tests/test_store.py b/services/drift/tests/test_store.py new file mode 100644 index 0000000..eb70f55 --- /dev/null +++ b/services/drift/tests/test_store.py @@ -0,0 +1,245 @@ +# services/drift/tests/test_store.py — store SQL pinned through a recording +# fake conn/cursor (the onboarding/pipeline house style; NEVER a live DB). +# +# The fake records (sql, params) in execution order and brackets each +# `with conn.transaction():` block with TXN-START / TXN-COMMIT (or +# TXN-ROLLBACK) markers. Fetch results are consumed FIFO. psycopg is not on +# the test path, so its Json wrapper is unwrapped duck-typed via .obj. +from ulpf_core.models import Mapping + +from drift.store import ( + ActiveDriftRule, + baseline_for, + fetch_current_view, + fetch_window_history, + first_window_unmapped, + insert_baseline, + insert_windows, + load_active_rules, +) + + +class FakeConn: + """Fake psycopg connection: records SQL, replays scripted fetch results.""" + + def __init__(self, fetchone_results=(), fetchall_results=()): + self.sql: list = [] + self.params: list = [] + self.txns_opened = 0 + self._fetchone = list(fetchone_results) + self._fetchall = list(fetchall_results) + + def transaction(self): + conn = self + + class Txn: + def __enter__(self): + conn.txns_opened += 1 + conn.sql.append("TXN-START") + conn.params.append(None) # keep sql/params index-aligned + + def __exit__(self, exc_type, exc, tb): + conn.sql.append("TXN-ROLLBACK" if exc_type is not None else "TXN-COMMIT") + conn.params.append(None) + return False + + return Txn() + + def cursor(self): + conn = self + + class Cur: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params=None): + conn.sql.append(sql) + conn.params.append(params) + + def fetchone(self): + return list(conn._fetchone.pop(0)) if conn._fetchone else [1] + + def fetchall(self): + return list(conn._fetchall.pop(0)) if conn._fetchall else [] + + return Cur() + + +def _unwrap(value): + """psycopg's Json adapter -> plain Python (duck-typed: no psycopg here).""" + return value.obj if hasattr(value, "obj") else value + + +def _rule_row(fp="fp_a", rule_id=7, version=2, quarantined=("dst_endpoint.ip",)): + return (rule_id, fp, version, r"x", [ + {"source_field": "SRC", "ocsf_path": "src_endpoint.ip"}], + list(quarantined) if quarantined is not None else None, "human") + + +# --- load_active_rules: id + mappings + quarantined_fields --------------------- + + +def test_load_active_rules_rebuilds_typed_rules_with_quarantine(): + conn = FakeConn(fetchall_results=[[_rule_row()]]) + rules = load_active_rules(conn) + + assert len(rules) == 1 and isinstance(rules[0], ActiveDriftRule) + rule = rules[0] + assert rule.id == 7 and rule.fingerprint_id == "fp_a" and rule.version == 2 + assert rule.mappings == [Mapping(source_field="SRC", ocsf_path="src_endpoint.ip")] + assert rule.quarantined_fields == ["dst_endpoint.ip"] + sql = next(s for s in conn.sql if isinstance(s, str)) + assert "FROM rules WHERE status = 'active'" in sql + assert "quarantined_fields" in sql and "mappings" in sql + assert "ORDER BY fingerprint_id" in sql # deterministic scan order + + +def test_load_active_rules_returns_rules_in_fingerprint_order(): + conn = FakeConn(fetchall_results=[[_rule_row("fp_b", 9), _rule_row("fp_a", 7)]]) + rules = load_active_rules(conn) + assert [r.fingerprint_id for r in rules] == ["fp_b", "fp_a"] # DB orders; passthrough + + +def test_load_active_rules_null_quarantine_becomes_empty_list(): + conn = FakeConn(fetchall_results=[[_rule_row(quarantined=None)]]) + assert load_active_rules(conn)[0].quarantined_fields == [] + + +# --- fetch_current_view: the DB current view, oldest first ---------------------- + + +def test_fetch_current_view_selects_unsuperseded_active_version_rows(): + rows = [("2026-09-21T10:00:00Z", "parsed", {"a": 1})] + conn = FakeConn(fetchall_results=[rows]) + assert fetch_current_view(conn, "fp_a", 2) == rows + + sql = next(s for s in conn.sql if isinstance(s, str)) + assert sql == ("SELECT parsed_at, status, ocsf FROM normalized_events " + "WHERE fingerprint_id = %s AND rule_version = %s " + "AND superseded_by_event_id IS NULL ORDER BY parsed_at") + assert conn.params[conn.sql.index(sql)] == ("fp_a", 2) + + +# --- insert_windows: idempotent by deterministic PK, one txn per batch ---------- + + +def test_insert_windows_on_conflict_do_nothing_in_one_transaction(): + row = ("fp_a", 2, "src_endpoint.ip", "t0", "t1", 4, 0.0, None, 0.25, + {"ipv4": 3}, "moderate", None) + conn = FakeConn() + insert_windows(conn, [row, row]) + + inserts = [(s, p) for s, p in zip(conn.sql, conn.params) + if isinstance(s, str) and s.startswith("INSERT INTO drift_windows")] + assert len(inserts) == 2 + sql, params = inserts[0] + assert "ON CONFLICT (fingerprint_id, rule_version, field, window_start) DO NOTHING" in sql + assert params[:3] == ("fp_a", 2, "src_endpoint.ip") + assert params[6] == 0.0 and params[8] == 0.25 and params[10] == "moderate" + assert params[11] is None # action_taken NULL until Task 5 + assert _unwrap(params[9]) == {"ipv4": 3} # shape_dist rides as JSONB + # One txn around the whole batch: a crash never leaves a half-written window. + assert conn.txns_opened == 1 + assert conn.sql[0] == "TXN-START" and conn.sql[-1] == "TXN-COMMIT" + + +def test_insert_windows_null_shape_dist_stays_null(): + row = ("fp_a", 2, "__rule__", "t0", "t1", 4, None, 1.0, None, None, "none", None) + conn = FakeConn() + insert_windows(conn, [row]) + params = next(p for s, p in zip(conn.sql, conn.params) + if isinstance(s, str) and s.startswith("INSERT INTO drift_windows")) + assert params[9] is None + + +def test_insert_windows_never_updates_anything(): + conn = FakeConn() + insert_windows(conn, [("fp_a", 2, "__rule__", "t0", "t1", 1, None, 1.0, + None, None, "none", None)]) + assert not [s for s in conn.sql if isinstance(s, str) + and s.upper().startswith("UPDATE")] # INSERT-only grants honored + + +# --- baseline_for: dict of profiles or None ------------------------------------- + + +def test_baseline_for_none_when_no_rows(): + conn = FakeConn(fetchall_results=[[]]) + assert baseline_for(conn, "fp_a", 2) is None + sql = next(s for s in conn.sql if isinstance(s, str)) + assert sql == ("SELECT field, profile FROM baseline_profiles " + "WHERE fingerprint_id = %s AND rule_version = %s") + assert conn.params[conn.sql.index(sql)] == ("fp_a", 2) + + +def test_baseline_for_maps_fields_to_profiles(): + profiles = [("src_endpoint.ip", {"null_rate": 0.1}), ("__rule__", {"match_rate": 1.0})] + conn = FakeConn(fetchall_results=[profiles]) + assert baseline_for(conn, "fp_a", 2) == { + "src_endpoint.ip": {"null_rate": 0.1}, "__rule__": {"match_rate": 1.0}} + + +# --- fetch_window_history: the baseline aggregation input (Task 5) --------------- + + +def test_fetch_window_history_reads_version_rows_oldest_first(): + rows = [("w0", "src_endpoint.ip", 10, 0.1, None, {"ipv4": 10})] + conn = FakeConn(fetchall_results=[rows]) + assert fetch_window_history(conn, "fp_a", 2) == rows + + sql = next(s for s in conn.sql if isinstance(s, str)) + assert sql == ("SELECT window_start, field, events_count, null_rate, match_rate, " + "shape_dist FROM drift_windows " + "WHERE fingerprint_id = %s AND rule_version = %s " + "ORDER BY window_start, field") + assert conn.params[conn.sql.index(sql)] == ("fp_a", 2) + + +# --- first_window_unmapped: the baseline's first-window key set ------------------ + + +def test_first_window_unmapped_reads_earliest_window_keys(): + conn = FakeConn(fetchall_results=[[("unmapped.IN",), ("unmapped.OUT",)]]) + assert first_window_unmapped(conn, "fp_a", 2) == {"IN", "OUT"} + sql = next(s for s in conn.sql if isinstance(s, str)) + assert "SELECT DISTINCT field FROM drift_windows" in sql + assert "field LIKE 'unmapped.%%'" in sql + assert "(SELECT MIN(window_start) FROM drift_windows" in sql + assert conn.params[conn.sql.index(sql)] == ("fp_a", 2, "fp_a", 2) + + +# --- insert_baseline: existence-guarded, once ----------------------------------- + + +def test_insert_baseline_skips_when_one_already_exists(): + conn = FakeConn() # default fetchone -> [1]: a row exists + inserted = insert_baseline(conn, "fp_a", 2, {"__rule__": {"match_rate": 1.0}}, 10) + + assert inserted is False + sql = [s for s in conn.sql if isinstance(s, str)] + assert any("SELECT 1 FROM baseline_profiles" in s for s in sql) + assert not [s for s in sql if s.startswith("INSERT")] # no write on the no-op + assert conn.txns_opened == 1 # guard still ran inside one txn + + +def test_insert_baseline_writes_each_field_once_with_conflict_noop(): + conn = FakeConn(fetchone_results=[[]]) # fetchone -> []: no baseline yet + profiles = {"src_endpoint.ip": {"null_rate": 0.1}, "__rule__": {"match_rate": 1.0}} + inserted = insert_baseline(conn, "fp_a", 2, profiles, 10) + + assert inserted is True + sql = [s for s in conn.sql if isinstance(s, str)] + guard = next(s for s in sql if "SELECT 1 FROM baseline_profiles" in s) + inserts = [s for s in sql if s.startswith("INSERT INTO baseline_profiles")] + assert len(inserts) == 2 # one row per field, sorted for determinism + assert all("ON CONFLICT (fingerprint_id, rule_version, field) DO NOTHING" in s + for s in inserts) + assert conn.txns_opened == 1 + assert sql.index(guard) < sql.index(inserts[0]) + first_params = conn.params[sql.index(inserts[0])] + assert first_params[:3] == ("fp_a", 2, "__rule__") # sorted() puts __rule__ first + assert first_params[4] == 10 # windows_seen + assert _unwrap(first_params[3]) == {"match_rate": 1.0} diff --git a/services/drift/tests/test_windows.py b/services/drift/tests/test_windows.py new file mode 100644 index 0000000..8b2e7e4 --- /dev/null +++ b/services/drift/tests/test_windows.py @@ -0,0 +1,234 @@ +# services/drift/tests/test_windows.py — pure window math + tier-1 semantics +# (plan Task 4 Steps 1/2), all with concrete rows/docs; NEVER a live DB. +from datetime import UTC, datetime, timedelta + +from drift.windows import FieldStats, Window, compute_windows, tier1_violation + + +def _ts(seconds: float) -> datetime: + """Aware UTC timestamp (the shape psycopg returns for TIMESTAMPTZ).""" + return datetime(2026, 9, 21, 10, 0, 0, tzinfo=UTC) + timedelta(seconds=seconds) + + +def _row(seconds: float, status: str = "parsed", ocsf: dict | None = None): + """One current-view row: (parsed_at, status, ocsf|None).""" + return (_ts(seconds), status, ocsf) + + +def _doc(ip="198.51.100.4", port=443, sev=3, time="2026-09-19T10:00:03Z", + unmapped: dict | None = None): + return { + "src_endpoint": {"ip": ip, "port": port}, + "severity_id": sev, + "time": time, + "unmapped": unmapped if unmapped is not None else {}, + } + + +# --- compute_windows: count-close / time-close / trailing held back ------------ + + +def _fast_row(millis: int, status: str = "parsed", ocsf: dict | None = None): + """Row packed milliseconds apart — 1000 rows stay well inside window_time_s, + so only the COUNT boundary can close these windows.""" + return (_ts(0) + timedelta(milliseconds=millis), status, ocsf) + + +def test_count_close_exactly_at_boundary(): + # The 1000th row CLOSES the window; the 1001st STARTS a new one — which is + # a trailing partial and therefore held back (never emitted). + rows = [_fast_row(i) for i in range(1001)] + windows = compute_windows(rows, count_window=1000, time_window_s=600) + assert len(windows) == 1 + assert windows[0].count == 1000 + assert windows[0].start == rows[0][0] and windows[0].end == rows[999][0] + + +def test_two_full_windows_close_cleanly(): + rows = [_fast_row(i) for i in range(2000)] + windows = compute_windows(rows, count_window=1000, time_window_s=600) + assert [w.count for w in windows] == [1000, 1000] + assert windows[1].start == rows[1000][0] and windows[1].end == rows[1999][0] + + +def test_time_close_low_volume_without_reaching_count(): + # 3 rows spread beyond window_time_s: the first row is older than the + # horizon of the newest row, so its window closes on TIME at count 1; the + # fresh tail [700, 800] is the trailing partial and is held back. + rows = [_row(0), _row(700), _row(800)] + windows = compute_windows(rows, count_window=1000, time_window_s=600) + assert len(windows) == 1 + assert windows[0].count == 1 + assert windows[0].start == _ts(0) and windows[0].end == _ts(0) + + +def test_trailing_partial_window_is_never_emitted(): + rows = [_row(i) for i in range(5)] + assert compute_windows(rows, count_window=10, time_window_s=600) == [] + + +def test_window_start_and_end_are_first_and_last_parsed_at(): + rows = [_row(i) for i in range(7)] + windows = compute_windows(rows, count_window=3, time_window_s=600) + # [0,1,2] and [3,4,5] close on count; [6] is trailing and held back. + assert len(windows) == 2 + assert windows[0].start == _ts(0) and windows[0].end == _ts(2) + assert windows[1].start == _ts(3) and windows[1].end == _ts(5) + + +def test_same_rows_twice_produce_identical_windows(): + rows = [_row(i, "parsed" if i % 3 else "parse_error", + _doc() if i % 3 else None) for i in range(50)] + first = compute_windows(rows, count_window=7, time_window_s=600, + mapped_paths=("src_endpoint.ip",)) + second = compute_windows(rows, count_window=7, time_window_s=600, + mapped_paths=("src_endpoint.ip",)) + assert first == second # determinism: data only, never wall-clock + + +def test_parsed_and_parse_errors_partition_the_window(): + rows = [_row(0, "parsed", _doc()), _row(1, "parsed", _doc()), + _row(2, "parse_error", None)] + windows = compute_windows(rows, count_window=3, time_window_s=600) + assert windows[0].count == 3 + assert windows[0].parsed == 2 and windows[0].parse_errors == 1 + + +def test_empty_rows_return_no_windows(): + assert compute_windows([], count_window=1000, time_window_s=600) == [] + + +# --- field stats: nulls / violations / values ---------------------------------- + + +def test_nulls_counted_not_tier1_violations(): + doc = _doc(ip=None) # mapped field present but None: a null, never a violation + windows = compute_windows([_row(0, "parsed", doc)], + count_window=1, time_window_s=600, + mapped_paths=("src_endpoint.ip",)) + assert windows[0].field_stats["src_endpoint.ip"] == FieldStats( + nulls=1, violations=0, values=[]) + + +def test_field_stats_values_collected_for_the_histogram(): + docs = [_row(0, "parsed", _doc(ip="198.51.100.4")), + _row(1, "parsed", _doc(ip="198.51.100.5"))] + windows = compute_windows(docs, count_window=2, time_window_s=600, + mapped_paths=("src_endpoint.ip",)) + assert windows[0].field_stats["src_endpoint.ip"].values == [ + "198.51.100.4", "198.51.100.5"] + + +def test_field_stats_covers_mapped_paths_that_never_fire(): + # A mapping whose group never matches: every parsed doc counts a null. + docs = [_row(0, "parsed", _doc()), _row(1, "parsed", _doc())] + windows = compute_windows(docs, count_window=2, time_window_s=600, + mapped_paths=("dst_endpoint.ip", "src_endpoint.ip")) + assert windows[0].field_stats["dst_endpoint.ip"].nulls == 2 + assert "dst_endpoint.ip" in windows[0].field_stats # present even at zero + + +def test_tier1_violations_counted_per_field_in_window(): + docs = [_row(0, "parsed", _doc(ip="SRC=198.51.100.4")), # firmware-drift garbage + _row(1, "parsed", _doc(ip="198.51.100.5")), + _row(2, "parsed", _doc(ip="DST=203.0.113.9")), + _row(3, "parsed", _doc(ip="198.51.100.6"))] + windows = compute_windows(docs, count_window=4, time_window_s=600, + mapped_paths=("src_endpoint.ip",)) + assert windows[0].field_stats["src_endpoint.ip"].violations == 2 + + +# --- unmapped keys: recorded always, "new" only against a baseline ------------- + + +def test_unmapped_keys_recorded_as_field_stats(): + docs = [_row(0, "parsed", _doc(unmapped={"IN": "eth0", "OUT": None})), + _row(1, "parsed", _doc(unmapped={"IN": "eth1"}))] + windows = compute_windows(docs, count_window=2, time_window_s=600) + stats = windows[0].field_stats + assert stats["unmapped.IN"].values == ["eth0", "eth1"] + assert stats["unmapped.IN"].nulls == 0 + assert stats["unmapped.OUT"].nulls == 1 and stats["unmapped.OUT"].values == [] + + +def test_new_unmapped_empty_without_baseline(): + # Before a baseline exists, keys are just RECORDED — none are "new". + rows = [_row(0, "parsed", _doc(unmapped={"IN": "eth0"}))] + windows = compute_windows(rows, count_window=1, time_window_s=600) + assert windows[0].new_unmapped == set() + + +def test_new_unmapped_flagged_against_known_set(): + rows = [_row(0, "parsed", _doc(unmapped={"IN": "eth0", "OUT": "wan"}))] + windows = compute_windows(rows, count_window=1, time_window_s=600, + known_unmapped=frozenset({"IN"})) + assert windows[0].new_unmapped == {"OUT"} + + +def test_unmapped_key_equal_to_a_mapped_target_never_new(): + # A quarantined mapping relocates its value into `unmapped` under the + # SOURCE key (Task 6); a key equal to a mapped OCSF target is never "new". + rows = [_row(0, "parsed", _doc(unmapped={"src_endpoint.ip": "1.2.3.4"}))] + windows = compute_windows(rows, count_window=1, time_window_s=600, + mapped_paths=("src_endpoint.ip",), + known_unmapped=frozenset()) + assert windows[0].new_unmapped == set() + + +# --- tier1_violation: mirrors validation._value_sanity, window-aggregated ------- + + +def test_ip_fields_require_dotted_quad_string(): + assert tier1_violation("src_endpoint.ip", "SRC=198.51.100.4") is True + assert tier1_violation("dst_endpoint.ip", "198.51.100.4") is False + assert tier1_violation("src_endpoint.ip", "999.999.999.999") is False # shape-level + assert tier1_violation("src_endpoint.ip", 123) is True # must be a str + assert tier1_violation("dst_endpoint.ip", "1.2.3.4.5") is True + + +def test_ports_must_be_int_in_0_65535(): + assert tier1_violation("src_endpoint.port", 70000) is True + assert tier1_violation("dst_endpoint.port", "443") is True # a str port is a failure + assert tier1_violation("src_endpoint.port", True) is True # bool is not an int + assert tier1_violation("src_endpoint.port", 0) is False + assert tier1_violation("dst_endpoint.port", 65535) is False + assert tier1_violation("src_endpoint.port", 65536) is True + assert tier1_violation("dst_endpoint.port", -1) is True + + +def test_severity_id_must_be_int_0_to_6(): + assert tier1_violation("severity_id", 9) is True + assert tier1_violation("severity_id", 3) is False + assert tier1_violation("severity_id", 0) is False + assert tier1_violation("severity_id", False) is True # bool is not an int + assert tier1_violation("severity_id", "5") is True # a str severity fails + + +def test_time_must_be_iso_parsable_string(): + assert tier1_violation("time", "not-a-ts") is True + assert tier1_violation("time", "2026-09-19T10:00:03Z") is False + assert tier1_violation("time", "2026-09-19") is False + assert tier1_violation("time", 12345) is True # must be a str + + +def test_none_is_a_null_not_a_violation(): + for path in ("src_endpoint.ip", "dst_endpoint.ip", "src_endpoint.port", + "dst_endpoint.port", "severity_id", "time"): + assert tier1_violation(path, None) is False + + +def test_unregulated_fields_never_violate(): + assert tier1_violation("message", "anything") is False + assert tier1_violation("action", 12345) is False + assert tier1_violation("unmapped.IN", "DROP") is False + + +def test_window_dataclass_is_frozen_shape(): + win = Window(start=_ts(0), end=_ts(1), count=1, parsed=1, parse_errors=0, + field_stats={}, new_unmapped=set()) + assert win.count == 1 + import dataclasses + + import pytest + with pytest.raises(dataclasses.FrozenInstanceError): + win.count = 2 diff --git a/services/drift/windows.py b/services/drift/windows.py new file mode 100644 index 0000000..4c5125a --- /dev/null +++ b/services/drift/windows.py @@ -0,0 +1,157 @@ +# services/drift/windows.py — pure window math over the current view (R-M3-4). +# +# Stateless recompute contract (spec §7.1): windows are derived ONLY from the +# rows handed in — ordered by parsed_at — and never from the wall clock, so +# the same rows always produce identical windows (idempotent INSERT keys). +# Detection math beyond counting (JS divergence, ladders) lives in +# ulpf_core.drift; this module owns slicing and the per-field aggregation. +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import datetime, timedelta + +# Same dotted-quad shape as ulpf_core.validation._IP_RE / drift._IPV4_RE +# (four digit quads, no 0-255 enforcement — a shape test, not a parser). +_IPV4_RE = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") + + +@dataclass(frozen=True) +class FieldStats: + """Per-field aggregates for one window: null count, tier-1 invariant + violation count, and the non-null values (input of the shape histogram).""" + + nulls: int + violations: int + values: list + + +@dataclass(frozen=True) +class Window: + """One closed window of the current view (a trailing partial is never + emitted — it may still grow before the next scan).""" + + start: datetime # parsed_at of the window's first event + end: datetime # parsed_at of the window's last event + count: int # parsed + parse_error rows in the window + parsed: int + parse_errors: int + field_stats: dict[str, FieldStats] + new_unmapped: set[str] + + +def compute_windows(rows, *, count_window, time_window_s, + mapped_paths=(), known_unmapped=None) -> list[Window]: + """Slice current-view rows into closed windows. + + rows: (parsed_at, status, ocsf|None) ordered by parsed_at. A window closes + when it holds count_window rows OR when its FIRST row is older than + window_time_s — checked at scan time against the NEWEST row's parsed_at + (the data's own horizon, never the wall clock; determinism). The trailing + partial window is held back: it may still grow before the next scan. + + mapped_paths: the rule's mapped OCSF targets — the fields field_stats + covers (a mapped-but-None value is a null, never a tier-1 violation). + known_unmapped: None while no baseline exists (unmapped keys are recorded + but none are "new"); afterwards, the baseline's first-window key set, and + keys absent from BOTH it and the mapped targets are new (Step 1 contract). + + The frozen call form compute_windows(rows, count_window=..., + time_window_s=...) is unchanged; the two rule-context keywords default to + the pre-baseline, no-mappings behavior. + """ + if not rows: + return [] + horizon = rows[-1][0] - timedelta(seconds=time_window_s) + windows: list[Window] = [] + current: list = [] + for row in rows: + current.append(row) + if len(current) >= count_window or current[0][0] < horizon: + windows.append(_close_window(current, mapped_paths, known_unmapped)) + current = [] + return windows + + +def _close_window(rows, mapped_paths, known_unmapped) -> Window: + """Aggregate one closed window: counts, per-field stats, new unmapped.""" + docs = [ocsf for (_, status, ocsf) in rows if status == "parsed" and ocsf is not None] + parse_errors = sum(1 for (_, status, _) in rows if status == "parse_error") + + nulls = {path: 0 for path in mapped_paths} + violations = {path: 0 for path in mapped_paths} + values: dict[str, list] = {path: [] for path in mapped_paths} + un_nulls: dict[str, int] = {} + un_values: dict[str, list] = {} + for doc in docs: + for path in mapped_paths: + value = _doc_get(doc, path) + if value is None: + nulls[path] += 1 + else: + values[path].append(value) + if tier1_violation(path, value): + violations[path] += 1 + for key, value in (doc.get("unmapped") or {}).items(): + if value is None: + un_nulls[key] = un_nulls.get(key, 0) + 1 + else: + un_values.setdefault(key, []).append(value) + + field_stats = {path: FieldStats(nulls[path], violations[path], values[path]) + for path in mapped_paths} + for key in sorted(set(un_nulls) | set(un_values)): # sorted: deterministic order + field_stats[f"unmapped.{key}"] = FieldStats( + un_nulls.get(key, 0), 0, un_values.get(key, [])) + + if known_unmapped is None: + new_unmapped: set[str] = set() # pre-baseline: recorded, never "new" + else: + targets = set(mapped_paths) + new_unmapped = {key for key in set(un_nulls) | set(un_values) + if key not in targets and key not in known_unmapped} + + return Window(start=rows[0][0], end=rows[-1][0], count=len(rows), + parsed=len(docs), parse_errors=parse_errors, + field_stats=field_stats, new_unmapped=new_unmapped) + + +def tier1_violation(field: str, value) -> bool: + """Tier-1 invariant for one mapped value (spec §7.1), mirroring + validation._value_sanity's checks as window-aggregated drift-time counts — + deliberately NOT an import: validation is candidate-time, this is + drift-time. None is NOT a violation (nulls are tier-2's signal); fields + with no invariant (message, unmapped.*) never violate.""" + if value is None: + return False + if field in ("src_endpoint.ip", "dst_endpoint.ip"): + return not (isinstance(value, str) and _IPV4_RE.match(value)) + if field in ("src_endpoint.port", "dst_endpoint.port"): + return not _is_int_in(value, 0, 65535) + if field == "severity_id": + return not _is_int_in(value, 0, 6) + if field == "time": + if not isinstance(value, str): + return True + try: + datetime.fromisoformat(value) + except ValueError: + return True + return False + return False + + +def _is_int_in(v, lo: int, hi: int) -> bool: + # Mirrors validation._is_int_in: bool is an int subclass — a True flag is + # neither a port nor a severity. + return isinstance(v, int) and not isinstance(v, bool) and lo <= v <= hi + + +def _doc_get(doc: dict, dotted_path: str): + """Walk a document by dotted OCSF path ('src_endpoint.ip'); missing -> None.""" + current = doc + for part in dotted_path.split("."): + if not isinstance(current, dict) or part not in current: + return None + current = current[part] + return current diff --git a/services/gateway/app.py b/services/gateway/app.py index 374ca99..a476bbe 100644 --- a/services/gateway/app.py +++ b/services/gateway/app.py @@ -18,7 +18,7 @@ import uvicorn from fastapi import FastAPI, HTTPException, Query -from fastapi.responses import StreamingResponse +from fastapi.responses import Response, StreamingResponse from pydantic import BaseModel from gateway import queries, writes @@ -41,6 +41,14 @@ def _json_default(value): raise TypeError(f"unserializable SSE value: {type(value).__name__}") +def _wire_str(value) -> str: + """Scalar wire form shared by BOTH exports: exactly the serialization + _json_default applies inside json.dumps (UUID -> str, datetime -> + ISO-8601), applied to Parquet columns BEFORE the arrow table is built so + every non-int column is a plain string and the file schema stays stable.""" + return value.isoformat() if isinstance(value, datetime) else str(value) + + async def sse_events(queries_module, poll_seconds, heartbeat_seconds, max_polls): """Yield one `data:` line per new current-view row, then poll. @@ -99,6 +107,61 @@ class ManualBody(BaseModel): confidence: float | None = None +class UnquarantineBody(BaseModel): + field: str + actor: str = "anonymous" + reason: str | None = None + + +# --- M3 export bodies (Task 8; spec §11 / PS g+h — SIEM/data-lake feeds). ------ +# Both builders are module-level like sse_events so the routes can run them in +# a worker thread; both share fetch_export_rows (parsed-only current view, +# newest first, EXPORT_MAX-capped) so the two formats carry identical rows. + +def _export_jsonl_lines(queries_module, fingerprint, status, limit) -> list[str]: + """OCSF JSONL export body, assembled in the caller's worker thread (the + route runs this via asyncio.to_thread so the fetch + serialization never + touch the event loop): ONE OCSF document per line, newest first. Rows + arrive parsed-only from fetch_export_rows, so no line is ever "null". + Memory bound = the EXPORT_MAX cap: the row list and the line list are both + capped by the same LIMIT.""" + rows = queries_module.fetch_export_rows(fingerprint, status, limit) + return [json.dumps(row["ocsf"], default=_json_default) for row in rows] + + +def _export_parquet_bytes(queries_module, fingerprint, limit) -> bytes: + """Parquet export body, built in the caller's worker thread. pyarrow is + imported lazily (mirrors queries.py's lazy psycopg) so importing + gateway.app stays cheap for non-export callers. The ocsf column is stored + as a JSON-serialized STRING per row: nested OCSF documents vary row to row + (optional attributes come and go), so a struct schema would churn with + every producer change — a flat string column keeps the file schema + constant for data-lake ingestion (consumers json-parse it on read). + Timestamps/UUIDs are wire-serialized BEFORE the table (_wire_str: the same + serialization as the JSONL export) so both formats carry identical values + and the arrow types are plain strings. Memory bound = the EXPORT_MAX cap + (row list + column arrays + the in-memory file).""" + import io + + import pyarrow as pa + import pyarrow.parquet as pq + + rows = queries_module.fetch_export_rows(fingerprint, None, limit) + table = pa.table({ + "event_id": pa.array([_wire_str(r["event_id"]) for r in rows], type=pa.string()), + "raw_id": pa.array([_wire_str(r["raw_id"]) for r in rows], type=pa.string()), + "fingerprint_id": pa.array([r["fingerprint_id"] for r in rows], type=pa.string()), + "rule_version": pa.array([r["rule_version"] for r in rows], type=pa.int32()), + "status": pa.array([r["status"] for r in rows], type=pa.string()), + "parsed_at": pa.array([_wire_str(r["parsed_at"]) for r in rows], type=pa.string()), + "ocsf": pa.array([json.dumps(r["ocsf"], default=_json_default) for r in rows], + type=pa.string()), + }) + sink = io.BytesIO() + pq.write_table(table, sink, compression="snappy") + return sink.getvalue() + + def _write_http_error(exc: writes.WriteError) -> HTTPException: """writes.* typed errors -> the API contract: 404 unknown, 409 conflict, 422 failed validation carrying the gate's checks + notes (the Review UI @@ -237,6 +300,68 @@ async def reactivate(rule_id: int, body: RejectBody | None = None): except writes.WriteError as exc: raise _write_http_error(exc) from exc + # ---------- M3: drift surfaces + human un-quarantine (Task 7) ---------- + + @app.get("/api/drift/metrics") + async def drift_metrics(): + return {"metrics": await asyncio.to_thread(queries_module.fetch_drift_metrics)} + + @app.get("/api/drift/alerts") + async def drift_alerts(): + # P-4: one kind-discriminated list — "window" rows (drift_windows with + # minor/moderate/severe severity) merged with "audit" rows (quarantine/ + # un-quarantine/deactivate actions, any actor), latest first. + return {"alerts": await asyncio.to_thread(queries_module.fetch_drift_alerts)} + + @app.post("/api/rules/{fingerprint_id}/unquarantine") + async def unquarantine(fingerprint_id: str, body: UnquarantineBody): + try: + return await asyncio.to_thread( + writes_module.unquarantine_field, fingerprint_id, body.field, + actor=body.actor, reason=body.reason) + except writes.WriteError as exc: + raise _write_http_error(exc) from exc + + # ---------- M3: export endpoints (Task 8; spec §11 / PS g+h) ---------- + + @app.get("/api/export/ocsf") + async def export_ocsf( + fingerprint: str | None = None, + status: str | None = None, + limit: int = Query(default=queries.EXPORT_MAX), + ): + """OCSF JSONL export: application/x-ndjson, ONE OCSF document per + line, newest first. Parsed-status DEFAULT — the export carries OCSF + documents and only parsed rows have one, so the status filter narrows + WITHIN parsed: non-parsed rows are SKIPPED (status=unparsed/ + parse_error/quarantined matches nothing and yields a 200 with an empty + stream — never a "null" line). limit deliberately has NO le= bound: + a bulk export must not 422; the fetch clamps it to EXPORT_MAX.""" + lines = await asyncio.to_thread( + _export_jsonl_lines, queries_module, fingerprint, status, limit) + return StreamingResponse((line + "\n" for line in lines), + media_type="application/x-ndjson") + + @app.get("/api/export/parquet") + async def export_parquet( + fingerprint: str | None = None, + limit: int = Query(default=queries.EXPORT_MAX), + ): + """Parquet export: the SAME fetch as the JSONL endpoint (EventRow + columns + ocsf, parsed-only current view, newest first, EXPORT_MAX- + capped; no status param in the frozen signature — the parsed default + applies), returned as a snappy-compressed attachment. The ocsf column + rides as a JSON string (flat, stable schema) and timestamps are + ISO-8601 strings — see _export_parquet_bytes.""" + data = await asyncio.to_thread( + _export_parquet_bytes, queries_module, fingerprint, limit) + return Response( + content=data, + media_type="application/octet-stream", + headers={"Content-Disposition": + 'attachment; filename="ulpf-events.parquet"'}, + ) + return app diff --git a/services/gateway/queries.py b/services/gateway/queries.py index 8420052..65cb064 100644 --- a/services/gateway/queries.py +++ b/services/gateway/queries.py @@ -22,6 +22,10 @@ DEFAULT_LIMIT = 50 MAX_LIMIT = 500 SNAPSHOT_LIMIT = 100 # SSE initial snapshot: latest N rows, emitted oldest-first +# Export rows are bulk (SIEM/data-lake integration), so the export endpoints +# get their OWN cap — 100x MAX_LIMIT — kept separate so the interactive LIMIT +# clamp can never silently widen (and vice versa). +EXPORT_MAX = 50_000 _STATUS_KEYS = ("parsed", "unparsed", "parse_error", "quarantined") @@ -166,8 +170,15 @@ def fetch_chain_heads() -> list[dict]: _RULE_COLUMNS = ( "id, fingerprint_id, version, pattern, mappings, provenance, confidence, " - "status, created_by, created_at, activated_at, deactivated_at, validation" + "status, quarantined_fields, created_by, created_at, activated_at, " + "deactivated_at, validation" ) +# quarantined_fields (ruling P-8): additive 14th key, column order mirroring +# migration 001's rules table (status, quarantined_fields, created_by). The +# column has existed since 001 (TEXT[] NOT NULL DEFAULT '{}') and gateway_role +# holds table-level SELECT, so both rule surfaces gain the key with no grant +# change — the Drift & Health page gates un-quarantine on the active +# version's list from GET /api/rules/{fp}. def fetch_rules(status=None): @@ -245,3 +256,99 @@ def fetch_audit(fingerprint=None, limit=DEFAULT_LIMIT): (fingerprint, fingerprint, limit), ) return cur.fetchall() + + +# --- M3 drift surfaces (Task 7; SELECT arrives via 003's default privileges) ---- + +_DRIFT_WINDOW_COLUMNS = ( + "fingerprint_id, rule_version, field, window_start, window_end, events_count, " + "null_rate, match_rate, violation_rate, shape_dist, severity, action_taken" +) + + +def fetch_drift_metrics(limit=DEFAULT_LIMIT) -> list[dict]: + """GET /api/drift/metrics rows: the LATEST closed window per + (fingerprint_id, field) — DISTINCT ON over window_start DESC — clamp-capped. + shape_dist is JSONB, so dict_row hands back a dict natively (no unwrap).""" + limit = clamp_limit(limit) + with _connect() as conn, conn.cursor() as cur: + cur.execute( + f"SELECT DISTINCT ON (fingerprint_id, field) {_DRIFT_WINDOW_COLUMNS} " + "FROM drift_windows " + "ORDER BY fingerprint_id, field, window_start DESC LIMIT %s", + (limit,), + ) + return cur.fetchall() + + +def _alert_recency(row: dict): + """Latest-first merge key across the two alert sources (P-4): a window row + is as recent as its window_start, an audit row as its ts.""" + return row["window_start"] if "window_start" in row else row["ts"] + + +def fetch_drift_alerts(limit=DEFAULT_LIMIT) -> list[dict]: + """GET /api/drift/alerts rows — controller ruling P-4's kind-discriminated + union, rendered by the web feed as ONE list: "window" rows are + drift_windows with severity minor/moderate/severe (the row IS the alert); + "audit" rows are audit_log actions field_quarantined/field_unquarantined/ + rule_deactivated from ANY actor (drift's enforcement or a human override). + Both sources are fetched latest-first and clamp-capped, then merged into a + single latest-first list capped at the same limit. The severity/action + IN-lists are module-controlled vocabulary (never request input), so they + stay literals — the _STATUS_KEYS / samples-role style.""" + limit = clamp_limit(limit) + with _connect() as conn, conn.cursor() as cur: + cur.execute( + f"SELECT {_DRIFT_WINDOW_COLUMNS} FROM drift_windows " + "WHERE severity IN ('minor', 'moderate', 'severe') " + "ORDER BY window_start DESC LIMIT %s", + (limit,), + ) + windows = [dict(row, kind="window") for row in cur.fetchall()] + cur.execute( + "SELECT id, ts, actor, action, entity, detail FROM audit_log " + "WHERE action IN ('field_quarantined', 'field_unquarantined', " + "'rule_deactivated') " + "ORDER BY id DESC LIMIT %s", + (limit,), + ) + audits = [dict(row, kind="audit") for row in cur.fetchall()] + return sorted(windows + audits, key=_alert_recency, reverse=True)[:limit] + + +# --- M3 exports (Task 8; spec §11 / PS g+h — SIEM/data-lake integration) -------- + +def clamp_export_limit(limit: int) -> int: + """Export LIMIT clamp: 1..EXPORT_MAX regardless of caller input. The export + endpoints take an UNBOUNDED limit param (no FastAPI le= — a bulk export + must not 422) and clamp here, in the query: one enforcement point, pinned + by the fake-conn tests.""" + return max(1, min(int(limit), EXPORT_MAX)) + + +def fetch_export_rows(fingerprint=None, status=None, limit=EXPORT_MAX): + """Export fetch shared by GET /api/export/{ocsf,parquet}: current view + (R9 predicate), EventRow columns + ocsf, newest first, clamp-capped at + EXPORT_MAX. + + PARSED-ONLY by design: the exports carry OCSF documents, and only parsed + rows have one. The `status = 'parsed'` literal is module-controlled + vocabulary (never request input), so it stays a literal like the + drift-severity lists; the status param narrows WITHIN parsed — it arrives + parameterized and INTERSECTS, so status=unparsed/parse_error/quarantined + selects nothing (non-parsed rows are skipped here, in SQL — the JSONL + stream never carries a "null" line) rather than erroring.""" + limit = clamp_export_limit(limit) + with _connect() as conn, conn.cursor() as cur: + cur.execute( + "SELECT event_id, raw_id, fingerprint_id, rule_version, status, parsed_at, ocsf " + "FROM normalized_events " + "WHERE superseded_by_event_id IS NULL " + "AND status = 'parsed' " + "AND (%s::text IS NULL OR status = %s) " + "AND (%s::text IS NULL OR fingerprint_id = %s) " + "ORDER BY parsed_at DESC LIMIT %s", + (status, status, fingerprint, fingerprint, limit), + ) + return cur.fetchall() diff --git a/services/gateway/requirements.txt b/services/gateway/requirements.txt index c20a40c..53ef28c 100644 --- a/services/gateway/requirements.txt +++ b/services/gateway/requirements.txt @@ -1,4 +1,8 @@ fastapi==0.111.0 uvicorn[standard]==0.30.1 psycopg[binary]==3.2.1 +# Controller P-6: 18.1.0, NOT 17.0.0 — 17.0.0 has no cp313 wheel and the +# py3.13 host must run the same version as CI (py3.12) and the image; 18.1.0 +# ships both cp312 and cp313 wheels. +pyarrow==18.1.0 httpx # tests (FastAPI TestClient), dev diff --git a/services/gateway/tests/test_gateway.py b/services/gateway/tests/test_gateway.py index 66b13b0..a1da393 100644 --- a/services/gateway/tests/test_gateway.py +++ b/services/gateway/tests/test_gateway.py @@ -263,11 +263,13 @@ def test_clamp_limit(queries): def test_r9_current_view_predicate_in_every_normalized_query(queries): # Controller R9: EVERY query over normalized_events carries # `superseded_by_event_id IS NULL` — fetch_stats (3 normalized queries), - # fetch_events, fetch_raw_trace (join), poll_events. fetch_chain_heads is - # deliberately absent (raw_batches has no supersede column). + # fetch_events, fetch_raw_trace (join), poll_events, fetch_export_rows + # (M3 export fetch). fetch_chain_heads is deliberately absent + # (raw_batches has no supersede column). import inspect - for fn_name in ("fetch_stats", "fetch_events", "fetch_raw_trace", "poll_events"): + for fn_name in ("fetch_stats", "fetch_events", "fetch_raw_trace", "poll_events", + "fetch_export_rows"): src = inspect.getsource(getattr(queries, fn_name)) assert "superseded_by_event_id IS NULL" in src, f"{fn_name} missing R9 predicate" @@ -281,13 +283,25 @@ def test_r9_current_view_predicate_in_every_normalized_query(queries): RULE_ROW_KEYS = { "id", "fingerprint_id", "version", "pattern", "mappings", "provenance", "confidence", - "status", "created_by", "created_at", "activated_at", "deactivated_at", "validation", + "status", "quarantined_fields", "created_by", "created_at", "activated_at", + "deactivated_at", "validation", } AUDIT_ROW_KEYS = {"id", "ts", "actor", "action", "entity", "detail"} +# P-8: the additive quarantined_fields column rides BOTH rule surfaces +# (fetch_rules, fetch_rule_history) — the full SELECT list is pinned literally +# (the services/pipeline/tests/test_rules.py style) so a regression that drops +# the key fails here, not in the Drift & Health page's gating. +RULE_COLUMNS_SQL = ( + "SELECT id, fingerprint_id, version, pattern, mappings, provenance, " + "confidence, status, quarantined_fields, created_by, created_at, " + "activated_at, deactivated_at, validation FROM rules" +) + def rule_row(**overrides) -> dict: - """One RuleRow as psycopg dict_row returns it (JSONB loaded, real datetimes).""" + """One RuleRow as psycopg dict_row returns it (JSONB loaded, TEXT[] as a + Python list, real datetimes).""" row = { "id": 7, "fingerprint_id": "fp_ssh_denied", @@ -297,6 +311,7 @@ def rule_row(**overrides) -> dict: "provenance": "slm", "confidence": 0.93, "status": "active", + "quarantined_fields": [], "created_by": "onboarding", "created_at": datetime(2026, 9, 19, 9, 0, 0, tzinfo=UTC), "activated_at": datetime(2026, 9, 19, 9, 5, 0, tzinfo=UTC), @@ -395,7 +410,8 @@ def fake(status=None): def test_rule_history_shape(client, queries, monkeypatch): - body = {"rules": [rule_row(version=3), rule_row(version=2, status="superseded")], + body = {"rules": [rule_row(version=3, quarantined_fields=["dst_endpoint.ip"]), + rule_row(version=2, status="superseded")], "audit": [audit_row()]} monkeypatch.setattr(queries, "fetch_rule_history", lambda fp: body) r = client.get("/api/rules/fp_ssh_denied") @@ -404,6 +420,9 @@ def test_rule_history_shape(client, queries, monkeypatch): "rules": [jsonable(row) for row in body["rules"]], "audit": [jsonable(row) for row in body["audit"]], } + # P-8: TEXT[] reaches the wire as a JSON array of strings — the Drift & + # Health page's un-quarantine gate consumes this exact value. + assert r.json()["rules"][0]["quarantined_fields"] == ["dst_endpoint.ip"] # ---------- GET /api/onboarding/samples ---------- @@ -464,6 +483,7 @@ def test_fetch_rules_filters_status_and_orders(queries, monkeypatch): conn = _FakeRowsConn(fetchall=[[rule_row()]]) monkeypatch.setattr(queries, "_connect", lambda: conn) assert queries.fetch_rules("pending_review") == [rule_row()] + assert RULE_COLUMNS_SQL in conn.sql[0] # P-8: quarantined_fields rides the list surface assert "WHERE (%s::text IS NULL OR status = %s)" in conn.sql[0] assert "ORDER BY fingerprint_id, version DESC" in conn.sql[0] assert conn.params[0] == ("pending_review", "pending_review") @@ -478,6 +498,7 @@ def test_fetch_rule_history_returns_rules_and_scoped_audit(queries, monkeypatch) body = queries.fetch_rule_history("fp_ssh_denied") assert body["rules"][0]["version"] == 3 # versions newest-first assert body["audit"] == [audit_row()] + assert RULE_COLUMNS_SQL in conn.sql[0] # P-8: ...and the history surface assert "ORDER BY version DESC" in conn.sql[0] assert "FROM audit_log" in conn.sql[1] and "ORDER BY id DESC LIMIT 200" in conn.sql[1] assert conn.params[1] == ("fp_ssh_denied",) @@ -708,3 +729,384 @@ def fake_reactivate(rule_id, **kwargs): assert r.json() == {"rule_id": 5, "version": 2, "status": "active"} assert seen["reactivate"][0] == 5 assert seen["reactivate"][1]["actor"] == "anonymous" # default actor + + +# ================= M3 (Task 7): drift surfaces + human un-quarantine ========== +# Everything below is ADDITIVE — the frozen M1+M2 contracts above are untouched. +# Same patterns: GETs return the queries-module fixtures verbatim (shapes +# mirrored key-for-key by Task 10's types.ts); the un-quarantine POST forwards +# to gateway.writes off the event loop; the fake-conn tests pin the real SQL. + +DRIFT_WINDOW_KEYS = { + "fingerprint_id", "rule_version", "field", "window_start", "window_end", + "events_count", "null_rate", "match_rate", "violation_rate", "shape_dist", + "severity", "action_taken", +} + + +def drift_window_row(**overrides) -> dict: + """One drift_windows row as psycopg dict_row returns it (JSONB shape_dist + loaded as a dict, real timestamptz datetimes).""" + row = { + "fingerprint_id": "fp_ssh_denied", + "rule_version": 3, + "field": "src_endpoint.ip", + "window_start": datetime(2026, 9, 21, 12, 0, 0, tzinfo=UTC), + "window_end": datetime(2026, 9, 21, 12, 15, 0, tzinfo=UTC), + "events_count": 240, + "null_rate": 0.012, + "match_rate": 0.988, + "violation_rate": 0.0, + "shape_dist": {"ipv4": 230, "int": 8}, + "severity": "minor", + "action_taken": "alert", + } + row.update(overrides) + return row + + +# ---------- GET /api/drift/metrics ---------- + +def test_drift_metrics_shape(client, queries, monkeypatch): + rows = [drift_window_row()] + monkeypatch.setattr(queries, "fetch_drift_metrics", lambda: rows) + r = client.get("/api/drift/metrics") + assert r.status_code == 200 + assert r.json() == {"metrics": [jsonable(rows[0])]} + assert set(r.json()["metrics"][0]) == DRIFT_WINDOW_KEYS # exact keys + + +# ---------- GET /api/drift/alerts (controller P-4: one discriminated list) ---- + +def test_drift_alerts_shape_both_kinds(client, queries, monkeypatch): + alerts = [ + dict(drift_window_row(severity="severe", action_taken="field_quarantined"), + kind="window"), + dict(audit_row(action="field_quarantined", actor="drift"), kind="audit"), + dict(audit_row(action="field_unquarantined", actor="amy", id=42), kind="audit"), + ] + monkeypatch.setattr(queries, "fetch_drift_alerts", lambda: alerts) + r = client.get("/api/drift/alerts") + assert r.status_code == 200 + assert r.json() == {"alerts": [jsonable(a) for a in alerts]} + # window rows: metric shape + the P-4 discriminator; audit rows: audit shape + it + assert set(r.json()["alerts"][0]) == DRIFT_WINDOW_KEYS | {"kind"} + assert r.json()["alerts"][0]["kind"] == "window" + assert set(r.json()["alerts"][1]) == AUDIT_ROW_KEYS | {"kind"} + assert r.json()["alerts"][1]["kind"] == "audit" + assert r.json()["alerts"][2]["kind"] == "audit" # any actor, per P-4 + + +# ---------- drift read helpers over a fake conn (real SQL pinned) ---------- + +def test_fetch_drift_metrics_latest_window_per_field(queries, monkeypatch): + conn = _FakeRowsConn(fetchall=[[drift_window_row()]]) + monkeypatch.setattr(queries, "_connect", lambda: conn) + assert queries.fetch_drift_metrics() == [drift_window_row()] + # LATEST window per (fingerprint_id, field): DISTINCT ON over window_start DESC + assert "DISTINCT ON (fingerprint_id, field)" in conn.sql[0] + assert "FROM drift_windows" in conn.sql[0] + assert "ORDER BY fingerprint_id, field, window_start DESC LIMIT %s" in conn.sql[0] + assert conn.params[0] == (50,) # DEFAULT_LIMIT through the clamp + + +def test_fetch_drift_alerts_queries_both_sources_and_clamps(queries, monkeypatch): + conn = _FakeRowsConn(fetchall=[ + [drift_window_row()], + [audit_row(action="field_quarantined", actor="drift")], + ]) + monkeypatch.setattr(queries, "_connect", lambda: conn) + out = queries.fetch_drift_alerts(limit=10**9) # nonsense limit -> clamped + assert len(conn.sql) == 2 # BOTH sources: drift_windows, then audit_log + assert "FROM drift_windows" in conn.sql[0] + assert "severity IN ('minor', 'moderate', 'severe')" in conn.sql[0] + assert "ORDER BY window_start DESC LIMIT %s" in conn.sql[0] + assert "FROM audit_log" in conn.sql[1] + assert ("action IN ('field_quarantined', 'field_unquarantined', " + "'rule_deactivated')") in conn.sql[1] + assert "ORDER BY id DESC LIMIT %s" in conn.sql[1] + assert conn.params[0] == (500,) and conn.params[1] == (500,) + assert [row["kind"] for row in out] == ["window", "audit"] # P-4 discriminator + + +def test_fetch_drift_alerts_merges_latest_first_across_sources(queries, monkeypatch): + new_window = drift_window_row(window_start=datetime(2026, 9, 21, 13, 0, tzinfo=UTC)) + old_window = drift_window_row(field="dst_endpoint.ip", + window_start=datetime(2026, 9, 20, 13, 0, tzinfo=UTC)) + drift_audit = audit_row(action="field_quarantined", actor="drift", + ts=datetime(2026, 9, 21, 12, 30, tzinfo=UTC)) + human_audit = audit_row(action="field_unquarantined", actor="amy", + ts=datetime(2026, 9, 19, 9, 0, tzinfo=UTC)) + conn = _FakeRowsConn(fetchall=[[new_window, old_window], + [drift_audit, human_audit]]) + monkeypatch.setattr(queries, "_connect", lambda: conn) + out = queries.fetch_drift_alerts() + # ONE list, latest first across both sources (window_start vs ts as the key) + assert [row["kind"] for row in out] == ["window", "audit", "window", "audit"] + assert out[0]["window_start"] == new_window["window_start"] + assert out[1]["ts"] == drift_audit["ts"] + assert out[3]["ts"] == human_audit["ts"] + + capped = _FakeRowsConn(fetchall=[[new_window, old_window], + [drift_audit, human_audit]]) + monkeypatch.setattr(queries, "_connect", lambda: capped) + assert len(queries.fetch_drift_alerts(limit=1)) == 1 # merged list is capped too + + +# ---------- POST /api/rules/{fp}/unquarantine (human override) ---------- + +def test_unquarantine_endpoint_forwards_and_runs_in_worker_thread(client, monkeypatch): + import threading + + from gateway import writes + + seen = {} + + def fake(fp, field, **kwargs): + seen["fp"], seen["field"], seen["kwargs"] = fp, field, kwargs + seen["main_thread"] = threading.current_thread() is threading.main_thread() + return {"fingerprint_id": fp, "field": field, "quarantined_fields": []} + + monkeypatch.setattr(writes, "unquarantine_field", fake) + r = client.post("/api/rules/fp_ssh_denied/unquarantine", + json={"field": "src_endpoint.ip", "actor": "amy", + "reason": "false positive"}) + assert r.status_code == 200 + assert r.json() == {"fingerprint_id": "fp_ssh_denied", + "field": "src_endpoint.ip", "quarantined_fields": []} + assert (seen["fp"], seen["field"]) == ("fp_ssh_denied", "src_endpoint.ip") + assert seen["kwargs"] == {"actor": "amy", "reason": "false positive"} + assert seen["main_thread"] is False # asyncio.to_thread, not the event loop + + +def test_unquarantine_defaults_actor_and_reason(client, monkeypatch): + from gateway import writes + + seen = {} + + def fake(fp, field, **kwargs): + seen.update(fp=fp, field=field, **kwargs) + return {"fingerprint_id": fp, "field": field, "quarantined_fields": ["other"]} + + monkeypatch.setattr(writes, "unquarantine_field", fake) + r = client.post("/api/rules/fp_a/unquarantine", json={"field": "src_endpoint.ip"}) + assert r.status_code == 200 + assert seen["actor"] == "anonymous" and seen["reason"] is None # body defaults + + +def test_unquarantine_error_mapping_404_and_409(client, monkeypatch): + from gateway import writes + + monkeypatch.setattr(writes, "unquarantine_field", + _raises(writes.NotFoundError("unknown fingerprint"))) + assert client.post("/api/rules/fp_none/unquarantine", + json={"field": "x"}).status_code == 404 + + monkeypatch.setattr(writes, "unquarantine_field", + _raises(writes.ConflictError("not quarantined"))) + assert client.post("/api/rules/fp_a/unquarantine", + json={"field": "x"}).status_code == 409 + + +# ============== M3 (Task 8): export endpoints (OCSF JSONL + Parquet) =========== +# Both exports share fetch_export_rows (parsed-only current view, newest +# first, EXPORT_MAX-capped). Route tests monkeypatch the fetch and pin the +# wire bytes; fake-conn tests pin the real SQL — the parsed-only literal, the +# intersecting status filter, the R9 predicate, and the cap clamp (the ONE +# enforcement point). The Parquet tests round-trip the ACTUAL response bytes +# through pq.read_table — the same path a data lake takes. + +PARQUET_COLUMNS = ["event_id", "raw_id", "fingerprint_id", + "rule_version", "status", "parsed_at", "ocsf"] + + +def ocsf_doc(**overrides) -> dict: + """One realistic parsed-row OCSF document (class_uid 4001 = File Activity, + nested attributes exactly as the JSONB column loads them).""" + doc = {"class_uid": 4001, "activity_name": "Open", + "severity_id": 99, "src_endpoint": {"ip": "10.1.2.3", "port": 51000}} + doc.update(overrides) + return doc + + +def _read_parquet(content): + import io + + import pyarrow.parquet as pq + + return pq.read_table(io.BytesIO(content)) + + +# ---------- GET /api/export/ocsf (JSONL) ---------- + +def test_export_ocsf_one_ocsf_doc_per_line(client, queries, monkeypatch): + rows = [event_row(ocsf=ocsf_doc(activity_name="Open"), + event_id=uuid.UUID("11111111-1111-1111-1111-111111111111")), + event_row(ocsf=ocsf_doc(activity_name="Close", + dst_endpoint={"ip": "192.0.2.9"}))] + monkeypatch.setattr(queries, "fetch_export_rows", lambda *a, **k: rows) + r = client.get("/api/export/ocsf") + assert r.status_code == 200 + assert r.headers["content-type"].startswith("application/x-ndjson") + lines = r.content.decode("utf-8").splitlines() + assert len(lines) == 2 # one line per row: the OCSF doc alone, not EventRow + docs = [json.loads(line) for line in lines] + assert all(doc["class_uid"] == 4001 for doc in docs) + assert docs == [rows[0]["ocsf"], rows[1]["ocsf"]] # faithful, unkeyed + assert r.content.decode("utf-8").endswith("\n") # NDJSON trailing newline + + +def test_export_ocsf_newest_first(client, queries, monkeypatch): + # fetch_export_rows orders newest first (pinned in the SQL below); the + # stream must emit that order unchanged — line 0 is the newest row's doc. + newest = event_row(parsed_at=datetime(2026, 9, 21, 12, 0, tzinfo=UTC), + ocsf=ocsf_doc(time=2)) + oldest = event_row(parsed_at=datetime(2026, 9, 21, 11, 0, tzinfo=UTC), + ocsf=ocsf_doc(time=1)) + monkeypatch.setattr(queries, "fetch_export_rows", lambda *a, **k: [newest, oldest]) + r = client.get("/api/export/ocsf") + docs = [json.loads(line) for line in r.content.decode("utf-8").splitlines()] + assert [doc["time"] for doc in docs] == [2, 1] + + +def test_export_ocsf_forwards_filters_runs_off_loop_and_defaults( + client, queries, monkeypatch): + import threading + + seen = {} + + def fake(fingerprint=None, status=None, limit=None): + seen.update(fingerprint=fingerprint, status=status, limit=limit) + seen["main_thread"] = threading.current_thread() is threading.main_thread() + return [] + + monkeypatch.setattr(queries, "fetch_export_rows", fake) + assert client.get("/api/export/ocsf").status_code == 200 + # defaults: no fingerprint/status filters, bulk default limit = EXPORT_MAX + assert (seen["fingerprint"], seen["status"]) == (None, None) + assert seen["limit"] == queries.EXPORT_MAX + assert seen["main_thread"] is False # fetch + line assembly in a worker thread + + assert client.get("/api/export/ocsf", params={ + "fingerprint": "fp_ssh_denied", "status": "parsed", "limit": 100, + }).status_code == 200 + assert seen["fingerprint"] == "fp_ssh_denied" and seen["status"] == "parsed" + assert seen["limit"] == 100 + + +def test_export_ocsf_nonparsed_status_is_skipped_not_nulled(client, queries, monkeypatch): + # The status filter narrows WITHIN parsed: status=unparsed matches nothing + # (non-parsed rows have no OCSF document), so the export is a 200 with an + # EMPTY stream — never a "null" line. What the SQL selects is pinned in + # test_fetch_export_rows_sql below; here the route contract: 200, empty. + seen = {} + + def fake(fingerprint=None, status=None, limit=None): + seen["status"] = status + return [] # what the parsed-only SQL selects for status=unparsed + + monkeypatch.setattr(queries, "fetch_export_rows", fake) + r = client.get("/api/export/ocsf", params={"status": "unparsed"}) + assert r.status_code == 200 + assert r.content == b"" + assert seen["status"] == "unparsed" # forwarded, applied inside the fetch + + +def test_export_ocsf_huge_limit_is_clamped_not_rejected(client, queries, monkeypatch): + # Bulk exports must not 422: no le= on the param; EXPORT_MAX is enforced + # inside fetch_export_rows (pinned below via the fake conn). + monkeypatch.setattr(queries, "fetch_export_rows", lambda *a, **k: []) + assert client.get("/api/export/ocsf", params={"limit": 10**12}).status_code == 200 + + +# ---------- GET /api/export/parquet ---------- + +def test_export_parquet_round_trips_rows(client, queries, monkeypatch): + import pyarrow as pa + + rows = [event_row(ocsf=ocsf_doc()), event_row(ocsf=ocsf_doc(severity_id=1), + rule_version=4)] + monkeypatch.setattr(queries, "fetch_export_rows", lambda *a, **k: rows) + r = client.get("/api/export/parquet") + assert r.status_code == 200 + table = _read_parquet(r.content) # the data-lake path: bytes -> table + assert table.num_rows == 2 + assert table.column_names == PARQUET_COLUMNS # EventRow order, flat schema + assert table.schema.field("rule_version").type == pa.int32() + # wire serialization applied BEFORE the table (UUID -> str, datetime -> + # ISO-8601): byte-identical to what the JSONL endpoint's _json_default puts out + assert table.column("event_id").to_pylist() == [str(row["event_id"]) for row in rows] + assert table.column("raw_id").to_pylist() == [str(row["raw_id"]) for row in rows] + assert table.column("parsed_at").to_pylist() == ["2026-09-19T12:00:00+00:00"] * 2 + assert table.column("rule_version").to_pylist() == [3, 4] + # ocsf column = one JSON string per row; parsing it back yields the doc + ocsf_strings = table.column("ocsf").to_pylist() + assert ocsf_strings == [json.dumps(row["ocsf"]) for row in rows] + assert [json.loads(s) for s in ocsf_strings] == [row["ocsf"] for row in rows] + + +def test_export_parquet_headers_and_empty_valid_file(client, queries, monkeypatch): + monkeypatch.setattr(queries, "fetch_export_rows", lambda *a, **k: []) + r = client.get("/api/export/parquet") + assert r.status_code == 200 # 200 always: empty export = empty file, not error + assert r.headers["content-type"].startswith("application/octet-stream") + assert r.headers["content-disposition"] == \ + 'attachment; filename="ulpf-events.parquet"' + table = _read_parquet(r.content) + assert table.num_rows == 0 + assert table.column_names == PARQUET_COLUMNS # schema present and stable + + +def test_export_parquet_forwards_fingerprint_parsed_default_and_cap( + client, queries, monkeypatch): + seen = {} + + def fake(fingerprint=None, status=None, limit=None): + seen.update(fingerprint=fingerprint, status=status, limit=limit) + return [] + + monkeypatch.setattr(queries, "fetch_export_rows", fake) + assert client.get("/api/export/parquet").status_code == 200 + assert seen == {"fingerprint": None, "status": None, "limit": queries.EXPORT_MAX} + + assert client.get("/api/export/parquet", params={ + "fingerprint": "fp_x", "limit": 7}).status_code == 200 + # frozen signature has NO status param — the fetch's parsed-only default rides + assert seen == {"fingerprint": "fp_x", "status": None, "limit": 7} + + # cap enforced as a clamp, never a 422 (same shared fetch as JSONL) + assert client.get("/api/export/parquet", params={"limit": 10**12}).status_code == 200 + + +# ---------- export fetch over a fake conn (real SQL + cap pinned) ---------- + +def test_fetch_export_rows_sql_pins_parsed_only_filters_order_and_cap( + queries, monkeypatch): + row = event_row() + conn = _FakeRowsConn(fetchall=[[row]]) + monkeypatch.setattr(queries, "_connect", lambda: conn) + out = queries.fetch_export_rows(fingerprint="fp_a", status=None, limit=10**9) + assert out == [row] + sql = conn.sql[0] + assert "superseded_by_event_id IS NULL" in sql # R9 current view + assert "AND status = 'parsed'" in sql # parsed-only literal (module vocabulary) + assert "AND (%s::text IS NULL OR status = %s)" in sql # narrows WITHIN parsed + assert "AND (%s::text IS NULL OR fingerprint_id = %s)" in sql + assert "ORDER BY parsed_at DESC LIMIT %s" in sql # newest first + # nonsense limit clamped to EXPORT_MAX; None statuses forwarded as-is + assert conn.params[0] == (None, None, "fp_a", "fp_a", queries.EXPORT_MAX) + + # the status param intersects: forwarded parameterized, never interpolated + conn2 = _FakeRowsConn(fetchall=[[row]]) + monkeypatch.setattr(queries, "_connect", lambda: conn2) + queries.fetch_export_rows(status="unparsed") + assert conn2.params[0][:2] == ("unparsed", "unparsed") + + +def test_clamp_export_limit(queries): + assert queries.EXPORT_MAX == 50_000 # separate from MAX_LIMIT (500) + assert queries.clamp_export_limit(123) == 123 + assert queries.clamp_export_limit(50_000) == 50_000 + assert queries.clamp_export_limit(10**12) == queries.EXPORT_MAX + assert queries.clamp_export_limit(0) == 1 + assert queries.clamp_export_limit(-5) == 1 diff --git a/services/gateway/tests/test_writes.py b/services/gateway/tests/test_writes.py index 3abb5ea..677e5d7 100644 --- a/services/gateway/tests/test_writes.py +++ b/services/gateway/tests/test_writes.py @@ -502,3 +502,64 @@ def test_reactivate_rowcount_zero_conflicts_without_audit(): writes.reactivate_rule(5, conn=conn) assert not _audit_rows(conn) assert "TXN-ROLLBACK" in conn.sql + + +# --- un-quarantine (M3 Task 7: human override of drift's field quarantine) ----- + + +def test_unquarantine_removes_field_audits_inside_txn_and_returns_post_state(): + conn = FakeConn(fetchone=[ + {"id": 5}, # existence: the fingerprint has rule rows + {"quarantined_fields": [], "id": 5, "version": 2}, # RETURNING post-state + ]) + result = writes.unquarantine_field("fp_a", "src_endpoint.ip", actor="amy", + reason="false positive", conn=conn) + + assert result == {"fingerprint_id": "fp_a", "field": "src_endpoint.ip", + "quarantined_fields": []} + assert conn.txns_opened == 1 # single transaction + assert conn.sql[0] == "TXN-START" and conn.sql[-1] == "TXN-COMMIT" + + # Existence pre-check: any rule row proves the fingerprint known (status + # deliberately unfiltered — inactive is a 409, not a 404). + assert "SELECT id FROM rules WHERE fingerprint_id = %s LIMIT 1" in conn.sql[1] + assert conn.params[1] == ("fp_a",) + + updates = _statements(conn, "UPDATE rules") + assert len(updates) == 1 + assert "quarantined_fields = array_remove(quarantined_fields, %s)" in updates[0][0] + assert "WHERE fingerprint_id = %s AND status = 'active'" in updates[0][0] + assert "(%s = ANY(quarantined_fields))" in updates[0][0] # TOCTOU predicate + assert updates[0][1] == ("src_endpoint.ip", "fp_a", "src_endpoint.ip") + + audits = _audit_rows(conn) + assert len(audits) == 1 + (actor, action, entity, _), detail = audits[0] + assert (actor, action, entity) == ("amy", "field_unquarantined", "fp_a") + assert detail == {"field": "src_endpoint.ip", "rule_id": 5, "version": 2, + "reason": "false positive"} + # audit paired INSIDE the mutation's transaction bracket + start, end = conn.sql.index("TXN-START"), conn.sql.index("TXN-COMMIT") + audit_idx = next(i for i, sql in enumerate(conn.sql) + if isinstance(sql, str) and sql.startswith("INSERT INTO audit_log")) + assert start < audit_idx < end + + +def test_unquarantine_unknown_fingerprint_not_found(): + conn = FakeConn(fetchone=[None]) + with pytest.raises(writes.NotFoundError): + writes.unquarantine_field("fp_none", "src_endpoint.ip", conn=conn) + assert not _statements(conn, "UPDATE rules") + assert not _audit_rows(conn) + + +def test_unquarantine_field_not_quarantined_or_inactive_conflicts(): + # The fingerprint exists (existence check passes) but the UPDATE matches + # zero rows: the rule is inactive, or the field was never quarantined / + # already un-quarantined — 409, no audit for a mutation that never landed. + conn = FakeConn(fetchone=[{"id": 5}], rowcounts=[0]) + with pytest.raises(writes.ConflictError): + writes.unquarantine_field("fp_a", "src_endpoint.ip", conn=conn) + assert _statements(conn, "UPDATE rules") # the UPDATE ran and lost the race + assert not _audit_rows(conn) + assert "TXN-ROLLBACK" in conn.sql diff --git a/services/gateway/writes.py b/services/gateway/writes.py index bce4cc0..d1b9a96 100644 --- a/services/gateway/writes.py +++ b/services/gateway/writes.py @@ -482,3 +482,59 @@ def _reactivate_once(conn, rule_id, actor, reason): "reason": reason, }, actor=actor) return {"rule_id": rule_id, "version": row["version"], "status": "active"} + + +# --- un-quarantine (M3 Task 7: human override of drift's field quarantine) ------ + +# Column-scoped exactly to migration 007's rules_role grant (quarantined_fields +# only). The predicate carries BOTH TOCTOU backstops: status='active' and the +# field actually being quarantined — so an inactive rule, a concurrent +# un-quarantine, or a never-quarantined field updates zero rows and can never +# still write the audit row. RETURNING supplies the post-state list for the +# response body plus the rule identity for the audit detail. +_UNQUARANTINE_SQL = ( + "UPDATE rules SET quarantined_fields = array_remove(quarantined_fields, %s) " + "WHERE fingerprint_id = %s AND status = 'active' " + "AND (%s = ANY(quarantined_fields)) " + "RETURNING quarantined_fields, id, version" +) + + +def unquarantine_field(fingerprint_id: str, field: str, *, actor: str = "anonymous", + reason: str | None = None, + conn: psycopg.Connection | None = None) -> dict: + """Remove one field from the ACTIVE rule's quarantined_fields — the human + override that re-enables a mapping drift quarantined (values were preserved + in unmapped, so nothing is lost). 404 when the fingerprint has no rule row + at all; 409 when the UPDATE matches nothing: the rule is inactive, or the + field is not quarantined (anymore). The field_unquarantined audit row is + paired INSIDE the same transaction as the mutation.""" + return _run(_unquarantine_once, conn, fingerprint_id, field, actor, reason) + + +def _unquarantine_once(conn, fingerprint_id, field, actor, reason): + with conn.transaction(), conn.cursor() as cur: + # Existence pre-check first (the deactivate_rule pattern) so an unknown + # fingerprint answers 404, distinct from the UPDATE's 409 cases. Any + # rule row proves the fingerprint known — status is deliberately NOT + # filtered here (inactive is a 409, not a 404). + cur.execute( + "SELECT id FROM rules WHERE fingerprint_id = %s LIMIT 1", + (fingerprint_id,), + ) + if cur.fetchone() is None: + raise NotFoundError(f"no rule for fingerprint {fingerprint_id!r}") + cur.execute(_UNQUARANTINE_SQL, (field, fingerprint_id, field)) + if cur.rowcount == 0: + raise ConflictError( + f"field {field!r} is not quarantined on the active rule for " + f"fingerprint {fingerprint_id!r} (or the rule is not active)") + row = cur.fetchone() + _audit(cur, "field_unquarantined", fingerprint_id, { + "field": field, + "rule_id": row["id"], + "version": row["version"], + "reason": reason, + }, actor=actor) + return {"fingerprint_id": fingerprint_id, "field": field, + "quarantined_fields": row["quarantined_fields"]} diff --git a/services/pipeline/rules.py b/services/pipeline/rules.py index 33b2473..1428661 100644 --- a/services/pipeline/rules.py +++ b/services/pipeline/rules.py @@ -3,11 +3,14 @@ # One SELECT per batch (the worker refreshes every consume batch, so a rule # activated mid-stream is picked up on the next batch — no restart, no push). # mappings arrives as JSONB (list of {source_field, ocsf_path} dicts) and is -# rebuilt into the typed Mapping model before parse() ever sees the rule. +# rebuilt into the typed Mapping model before parse() ever sees the rule; +# quarantined_fields arrives as TEXT[] (a Python list via psycopg) — the M3 +# drift role appends OCSF paths there and the worker filters those mappings. from __future__ import annotations from typing import TYPE_CHECKING +from pydantic import Field from ulpf_core.models import Mapping, Rule if TYPE_CHECKING: @@ -20,10 +23,13 @@ class ActiveRule(Rule): load_active_rules must return dict[str, Rule] per the plan contract, but normalized_events.rule_id wants the DB id — so the id rides on this Rule subclass (an ActiveRule IS a Rule; parse() and the type contract are - unaffected, and the worker reads ``rule.id`` for attribution). + unaffected, and the worker reads ``rule.id`` for attribution). M3 adds + the quarantine list the same way: model_copy in the worker's filter + preserves this class, so id and version ride along on filtered rules. """ id: int + quarantined_fields: list[str] = Field(default_factory=list) def load_active_rules(conn: psycopg.Connection) -> dict[str, Rule]: @@ -32,16 +38,19 @@ def load_active_rules(conn: psycopg.Connection) -> dict[str, Rule]: rules: dict[str, Rule] = {} with conn.cursor() as cur: cur.execute( - "SELECT id, fingerprint_id, version, pattern, mappings, provenance " + "SELECT id, fingerprint_id, version, pattern, mappings, " + "quarantined_fields, provenance " "FROM rules WHERE status = 'active'" ) - for rule_id, fingerprint, version, pattern, mappings, provenance in cur.fetchall(): + for (rule_id, fingerprint, version, pattern, mappings, quarantined, + provenance) in cur.fetchall(): rules[fingerprint] = ActiveRule( id=rule_id, fingerprint_id=fingerprint, version=version, pattern=pattern, mappings=[Mapping(**m) for m in mappings], + quarantined_fields=list(quarantined or []), provenance=provenance, ) return rules diff --git a/services/pipeline/tests/test_quarantine.py b/services/pipeline/tests/test_quarantine.py new file mode 100644 index 0000000..6d94e84 --- /dev/null +++ b/services/pipeline/tests/test_quarantine.py @@ -0,0 +1,182 @@ +# services/pipeline/tests/test_quarantine.py — R-M3-6 (spec §7.2): the worker +# filters mappings whose OCSF path the drift role quarantined (Task 5 appends +# to rules.quarantined_fields) BEFORE parse; the captured key then lands in +# doc['unmapped'] — "mapping disabled, values preserved". An empty quarantine +# list is the M1/M2 hot path and MUST stay byte-identical: pinned here +# against an explicit expected document and a no-quarantine baseline run. +# No Kafka, no real DB (test_worker house style: fakes + monkeypatched +# persistence; the loader pin lives in test_rules.py). +import json + +from ulpf_core.fingerprint import fingerprint_id +from ulpf_core.ids import event_id_for, raw_id_for +from ulpf_core.models import Mapping + +import pipeline.worker as worker_mod +from pipeline.rules import ActiveRule +from pipeline.worker import PipelineWorker + +# Real corpus line (simulator/data/raw_logs_syslog.txt) — both this and the +# nomatch line below fingerprint to the known-format id "syslog", so they hit +# the same active rule. +IPTABLES_LINE = ("Aug 27 14:32:07 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= " + "SRC=203.0.113.45 DST=192.168.1.10 LEN=60 PROTO=TCP " + "SPT=51422 DPT=22 FLAGS=SYN") +NOMATCH_SYSLOG_LINE = "Aug 27 14:35:00 fw01 sshd[1234]: Accepted publickey for root" + +# The seed's syslog pattern: extension KV scan, SRC/DST mapped (M2 shape). +PATTERN = r"^.*?IPTABLES-\w+:\s*(?P.*)$" + +# What the line parses to with NO quarantine — the explicit M2 regression +# pin. OUT= is never captured by the extension KV grammar (empty value with +# no terminator), so it is absent from unmapped: asserted as the code +# behaves. time is the worker's received_at fill (no time mapping). +EXPECTED_DOC = { + "class_uid": 4001, + "class_name": "Network Activity", + "activity_id": 99, + "severity_id": None, + "time": "2026-09-19T12:00:00+00:00", + "src_endpoint": {"ip": "203.0.113.45"}, + "dst_endpoint": {"ip": "192.168.1.10"}, + "action": None, + "message": None, + "metadata": {"product": "ULPF"}, + "unmapped": {"IN": "eth0", "LEN": 60, "PROTO": "TCP", + "SPT": 51422, "DPT": 22, "FLAGS": "SYN"}, +} + + +def syslog_rule(quarantined=None): + """ActiveRule for the syslog fingerprint; quarantined=None means the kwarg + is not passed at all — the exact M2 construction path.""" + kwargs: dict = { + "id": 11, + "fingerprint_id": fingerprint_id(IPTABLES_LINE), + "version": 2, + "pattern": PATTERN, + "mappings": [Mapping(source_field="SRC", ocsf_path="src_endpoint.ip"), + Mapping(source_field="DST", ocsf_path="dst_endpoint.ip")], + "provenance": "human", + } + if quarantined is not None: + kwargs["quarantined_fields"] = quarantined + return ActiveRule(**kwargs) + + +def envelope_bytes(raw): + return json.dumps({"source_id": "s", "transport": "udp", + "received_at": "2026-09-19T12:00:00+00:00", + "raw": raw}).encode() + + +class FakeMsg: + def __init__(self, partition, offset, value: bytes): + self._p, self._o, self._v = partition, offset, value + + def topic(self): + return "raw.logs" + + def partition(self): + return self._p + + def offset(self): + return self._o + + def error(self): + return None + + def value(self): + return self._v + + +class StubConn: + """Only rollback() is reachable: persistence is monkeypatched out and + every message's fingerprint has an active rule (no sample-count query).""" + + def rollback(self): + pass + + +class SilentProducer: + def produce(self, *args, **kwargs): + pass + + def flush(self, timeout=10.0): + return 0 + + +class CommitCounter: + def __init__(self): + self.commits = 0 + + def commit(self): + self.commits += 1 + + +def run_batch(monkeypatch, rule, lines): + """One process_batch of *lines* under *rule* (all lines share the rule's + fingerprint — asserted); returns the NormalizedRows persist_normalized + received, in message order.""" + assert {fingerprint_id(line) for line in lines} == {rule.fingerprint_id} + seen: dict = {} + monkeypatch.setattr(worker_mod, "persist_batch", lambda conn, rows, parts: None) + monkeypatch.setattr(worker_mod, "load_active_rules", + lambda conn: {rule.fingerprint_id: rule}) + monkeypatch.setattr(worker_mod, "persist_normalized", + lambda conn, events, samples=None: seen.update(events=events)) + msgs = [FakeMsg(0, i + 1, envelope_bytes(line)) for i, line in enumerate(lines)] + worker = PipelineWorker(CommitCounter(), SilentProducer(), StubConn()) + assert worker.process_batch(msgs) is True + return seen["events"] + + +def test_quarantined_mapping_skipped_value_in_unmapped(monkeypatch): + events = run_batch(monkeypatch, syslog_rule(["src_endpoint.ip"]), [IPTABLES_LINE]) + row = events[0] + assert row.status == "parsed" + doc = row.ocsf + # src_endpoint's ONLY mapping was the quarantined one: _new_parts starts + # it as {} and _build_document renders an empty endpoint as None — that + # is what the code produces (mapping disabled, spec §7.2). + assert doc["src_endpoint"] is None + assert doc["dst_endpoint"] == {"ip": "192.168.1.10"} # untouched mapping + # The value is preserved verbatim in unmapped (never dropped, §6.4/§7.2): + # everything else is exactly the no-quarantine document. + assert doc["unmapped"] == dict(EXPECTED_DOC["unmapped"], SRC="203.0.113.45") + assert {k: v for k, v in doc.items() if k not in ("src_endpoint", "unmapped")} == { + k: v for k, v in EXPECTED_DOC.items() if k not in ("src_endpoint", "unmapped")} + + +def test_empty_quarantine_identical_behavior(monkeypatch): + # REGRESSION PIN: quarantined_fields=[] must be byte-identical to the M2 + # no-quarantine path (the worker's falsy check short-circuits the + # model_copy — zero object churn): same document, pinned explicitly + # below, and identical event_id / rule_id / rule_version. + baseline = run_batch(monkeypatch, syslog_rule(), [IPTABLES_LINE])[0] + empty_q = run_batch(monkeypatch, syslog_rule([]), [IPTABLES_LINE])[0] + + assert empty_q.status == baseline.status == "parsed" + assert empty_q.ocsf == EXPECTED_DOC + assert empty_q.ocsf == baseline.ocsf + assert empty_q.event_id == baseline.event_id == event_id_for( + raw_id_for("raw.logs", 0, 1), 2) + assert (empty_q.rule_id, empty_q.rule_version) == \ + (baseline.rule_id, baseline.rule_version) == (11, 2) + + +def test_model_copy_preserves_id(monkeypatch): + # The filter's model_copy must preserve the ActiveRule subclass: rule.id + # attribution and rule.version ride along on the filtered rule for parsed + # AND parse_error rows alike — the MAX_LINE_CHARS cap and the parse-error + # path read the same loop-local rule the copy produced. + parsed, error = run_batch( + monkeypatch, syslog_rule(["src_endpoint.ip"]), + [IPTABLES_LINE, NOMATCH_SYSLOG_LINE]) + + assert parsed.status == "parsed" + assert parsed.rule_id == 11 and parsed.rule_version == 2 # id survived the copy + assert error.status == "parse_error" + assert error.rule_id == 11 and error.rule_version == 2 + assert error.event_id == event_id_for(raw_id_for("raw.logs", 0, 2), 2) + assert error.ocsf is None diff --git a/services/pipeline/tests/test_rules.py b/services/pipeline/tests/test_rules.py new file mode 100644 index 0000000..7984ff0 --- /dev/null +++ b/services/pipeline/tests/test_rules.py @@ -0,0 +1,68 @@ +# services/pipeline/tests/test_rules.py — active-rule cache loader pins +# (M3 R-M3-6): the per-batch SELECT must carry rules.quarantined_fields so +# the worker can filter quarantined mappings at apply time (Task 5's drift +# role appends OCSF paths there). Fake conn in the drift/onboarding house +# style — NEVER a live DB. +from ulpf_core.models import Mapping + +from pipeline.rules import ActiveRule, load_active_rules + + +class FakeConn: + """Fake psycopg connection: records SQL, replays scripted fetch results.""" + + def __init__(self, fetchall_results=()): + self.sql: list = [] + self._fetchall = list(fetchall_results) + + def cursor(self): + conn = self + + class Cur: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params=None): + conn.sql.append(sql) + + def fetchall(self): + return conn._fetchall.pop(0) + + return Cur() + + +def _rule_row(rule_id=11, fp="syslog", quarantined=("src_endpoint.ip",)): + return (rule_id, fp, 2, r"^.*?IPTABLES-\w+:\s*(?P.*)$", [ + {"source_field": "SRC", "ocsf_path": "src_endpoint.ip"}, + {"source_field": "DST", "ocsf_path": "dst_endpoint.ip"}], + list(quarantined) if quarantined is not None else None, "human") + + +def test_load_active_rules_selects_quarantined_fields_column(): + conn = FakeConn([[_rule_row()]]) + rules = load_active_rules(conn) + + # Column list pinned exactly: quarantined_fields must ride the one + # per-batch SELECT alongside mappings (TEXT[] -> Python list via psycopg). + assert conn.sql == [ + ("SELECT id, fingerprint_id, version, pattern, mappings, " + "quarantined_fields, provenance FROM rules WHERE status = 'active'") + ] + rule = rules["syslog"] + assert isinstance(rule, ActiveRule) + assert rule.id == 11 and rule.version == 2 + assert rule.mappings == [ + Mapping(source_field="SRC", ocsf_path="src_endpoint.ip"), + Mapping(source_field="DST", ocsf_path="dst_endpoint.ip")] + assert rule.quarantined_fields == ["src_endpoint.ip"] + + +def test_load_active_rules_null_quarantine_becomes_empty_list(): + # The column is NOT NULL DEFAULT '{}' (migration 001), but the loader is + # defensive exactly like drift.store's: NULL -> [] so the worker's + # falsy check (the M1/M2 hot path) still short-circuits. + conn = FakeConn([[_rule_row(quarantined=None)]]) + assert load_active_rules(conn)["syslog"].quarantined_fields == [] diff --git a/services/pipeline/worker.py b/services/pipeline/worker.py index 101cb4a..f638648 100644 --- a/services/pipeline/worker.py +++ b/services/pipeline/worker.py @@ -205,6 +205,21 @@ def process_batch(self, msgs) -> bool: rules = load_active_rules(self.conn) # refreshed every batch + # M3 quarantine (R-M3-6, spec §7.2 "mapping disabled, values + # preserved"): drop mappings whose OCSF path the drift role + # quarantined — parse() then leaves the captured key in unmapped. + # Applied once per quarantined rule per batch, not per row: one rule + # object is shared by every row of its fingerprint. model_copy + # preserves ActiveRule, so rule.id attribution and rule.version ride + # along (quarantine never bumps the version — no event_id churn); + # the falsy check short-circuits, so an empty quarantine means zero + # copies and the M1/M2 hot path is byte-identical. + for fp, rule in rules.items(): + if rule.quarantined_fields: + rules[fp] = rule.model_copy(update={ + "mappings": [m for m in rule.mappings + if m.ocsf_path not in set(rule.quarantined_fields)]}) + # Sample capture (R17): only fingerprints with NO active rule, capped # per fingerprint by what onboarding_samples already holds. One grouped # count query for the whole batch; known fingerprints query nothing. diff --git a/simulator/simulate.py b/simulator/simulate.py index 8bfcd09..d87ca63 100644 --- a/simulator/simulate.py +++ b/simulator/simulate.py @@ -20,6 +20,16 @@ # is idle — no send, no cursor advance — so the already-running corpora are # untouched while it waits. # +# Modes (T11): "firmware-drift" keeps the steady corpus set but degrades it on +# the wire: after --drift-after seconds, every syslog line handed to the sender +# passes through corrupt_syslog() with a phase that escalates from 0 to 1 over +# the remaining run budget (R-P3). Corrupted lines split exactly 1:2 (R-P9): +# SRC is renamed to SRCADDR (the collector's SRC parse reads null — null-rate +# spike) or its value becomes the invalid SRC=uplink-trust-0x4f (tier-1 +# violation) — the drift-engine demo's live-rule degradation on real traffic. +# Only line CONTENT mutates: per-corpus counts, the other corpora, and the +# stdout contract below are unchanged. +# # Stdout contract (parsed by the M1 smoke test): exactly one line per corpus, # sent= # then a final "total sent=". Everything else goes to stderr. Exit 0 unless @@ -31,6 +41,7 @@ import asyncio import inspect import os +import re import socket import sys import time @@ -50,6 +61,14 @@ HTTP_KEYS = frozenset({"json"}) CORPUS_KEYS = sorted(UDP_KEYS | HTTP_KEYS) NEWAPP_KEY = "newapp" # loaded only in --mode new-appliance (see load_corpora) +SYSLOG_KEY = "syslog" # the only corpus --mode firmware-drift mutates (T11) + +# firmware-drift corruption profile (T11; controller rulings R-P3 / R-P9). +DRIFT_HASH_MULT = 2654435761 # Knuth multiplicative hash: the seed IS the line index +DRIFT_HASH_MOD = 1000 +ESCALATION_SPAN_FLOOR_S = 1.0 # escalation window floor: never divide by zero +_SRC_FIELD_RE = re.compile(r"\bSRC=") +_SRC_VALUE_RE = re.compile(r"\bSRC=\S*") def transport_for(key: str) -> str: @@ -200,13 +219,59 @@ def default_senders() -> dict: return senders +def escalation_span(duration: float, drift_after: float) -> float: + """Seconds over which corruption ramps to 100% once drift starts (R-P3). + + Defaults to the remaining run budget (duration - drift_after), floored at + ESCALATION_SPAN_FLOOR_S so a drift that starts at (or after) the deadline + still escalates over a sane window instead of dividing by zero. + """ + return max(duration - drift_after, ESCALATION_SPAN_FLOOR_S) + + +def drift_phase(elapsed_since_drift: float, span: float) -> float: + """Corruption fraction in [0, 1]; monotonic non-decreasing in elapsed (R-P3).""" + return min(max(elapsed_since_drift / span, 0.0), 1.0) + + +def corrupt_syslog(line: str, index: int, phase: float) -> str: + """Mutate one syslog line per the firmware-drift profile (T11, R-P3/R-P9). + + R-P3 selection (seed = line index; no RNG state anywhere): the line is + corrupted iff (index * 2654435761 % 1000) / 1000 < phase — strictly, so + phase 0 corrupts nothing and phase 1 corrupts everything (2654435761 % 1000 + = 761 is coprime with 1000, making the selector a permutation of [0, 1): + phase p hits exactly p of any 1000 consecutive indices). R-P9 then splits + the corrupted lines exactly 1:2 by index mod 3: index % 3 == 0 renames the + field SRC -> SRCADDR, value kept (the collector's SRC parse reads null — + the null-rate spike); every other corrupted line gets its SRC value + replaced with the invalid token uplink-trust-0x4f (tier-1 violation). A + line with no SRC= field passes through unchanged. + """ + selector = (index * DRIFT_HASH_MULT % DRIFT_HASH_MOD) / DRIFT_HASH_MOD + if selector >= phase: # R-P3: strict <; a selector equal to phase is NOT hit + return line + if "SRC=" not in line: + return line # nothing to mutate; the line passes through byte-identical + if index % 3 == 0: + return _SRC_FIELD_RE.sub("SRCADDR=", line, count=1) + return _SRC_VALUE_RE.sub("SRC=uplink-trust-0x4f", line, count=1) + + async def run(corpora: list[Corpus], *, eps: float, duration: float, - loop: bool = False) -> tuple[dict, int]: + loop: bool = False, drift_after: float | None = None) -> tuple[dict, int]: """Send the corpora round-robin at eps total events/sec. A corpus with start_at > 0 stays idle (no send, no cursor advance) until that many seconds have elapsed, then joins the rotation in place. + drift_after (firmware-drift mode, T11): once that many seconds have + elapsed, syslog lines pass through corrupt_syslog() with a phase that + escalates from 0 to 1 over escalation_span(duration, drift_after); the + decision is made per send (send-start time), so a pass that straddles the + boundary sends clean lines before it and mutated lines after it. None + (steady / new-appliance) never mutates a line. + Returns (sent_per_corpus, error_count). A send counts only when the transport accepted it (UDP sendto returned; HTTP got a 202). """ @@ -220,6 +285,9 @@ async def run(corpora: list[Corpus], *, eps: float, duration: float, start = time.monotonic() deadline = start + max(duration, 0.0) cursors = [0] * n + if drift_after is not None: + drift_after = max(drift_after, 0.0) + span = escalation_span(duration, drift_after) pass_no = 0 while True: @@ -239,7 +307,13 @@ async def run(corpora: list[Corpus], *, eps: float, duration: float, continue cursors[i] = 0 line = corpus.lines[cursors[i]] + index = cursors[i] # the corpus line index is the R-P3 seed cursors[i] += 1 + if drift_after is not None and corpus.name == SYSLOG_KEY: + elapsed = time.monotonic() - start + if elapsed >= drift_after: # boundary respected mid-pass, per send + line = corrupt_syslog(line, index, + drift_phase(elapsed - drift_after, span)) try: sent[corpus.name] += await corpus.sender.send(line) except Exception as exc: # noqa: BLE001 - one failed send counts and moves on @@ -282,12 +356,19 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: help="directory holding raw_logs_*.txt (default: simulator/data)") p.add_argument("--loop", action="store_true", help="repeat the corpora until --duration elapses") - p.add_argument("--mode", choices=("steady", "new-appliance"), default="steady", + p.add_argument("--mode", choices=("steady", "new-appliance", "firmware-drift"), + default="steady", help="steady: golden corpora only (default); new-appliance: " - "also replay the unknown-format newapp corpus (T10)") + "also replay the unknown-format newapp corpus (T10); " + "firmware-drift: syslog lines mutate after --drift-after, " + "escalating to 100% by the end of the run (T11)") p.add_argument("--new-after", type=float, default=0.0, dest="new_after", help="new-appliance only: seconds before the newapp corpus " "joins the rotation (default 0; steady ignores this)") + p.add_argument("--drift-after", type=float, default=5.0, dest="drift_after", + help="firmware-drift only: seconds before syslog lines start " + "mutating (corruption then escalates over the remaining " + "run budget) (default 5; other modes ignore this)") return p.parse_args(argv) @@ -313,10 +394,12 @@ def main(argv: list[str] | None = None) -> int: for corpus in corpora: if corpus.name == NEWAPP_KEY: corpus.start_at = max(args.new_after, 0.0) + drift_after = max(args.drift_after, 0.0) if args.mode == "firmware-drift" else None async def drive(): try: - return await run(corpora, eps=args.eps, duration=args.duration, loop=args.loop) + return await run(corpora, eps=args.eps, duration=args.duration, + loop=args.loop, drift_after=drift_after) finally: await _close_all(senders) diff --git a/simulator/tests/test_drift.py b/simulator/tests/test_drift.py new file mode 100644 index 0000000..df75796 --- /dev/null +++ b/simulator/tests/test_drift.py @@ -0,0 +1,291 @@ +# simulator/tests/test_drift.py — firmware-drift mode (T11): deterministic +# escalating syslog corruption on the wire. R-P3 selection (seed = line index, +# strict <), R-P9 variant split (index % 3). Pure-function pins first, then +# run()/main() wiring via the fake senders from test_simulate (imported, not +# duplicated — that file must stay byte-identical per the M2 regression pins). +import math +import re + +import simulator.tests.test_simulate as ts +from simulator.simulate import ( + ESCALATION_SPAN_FLOOR_S, + _parse_args, + corrupt_syslog, + drift_phase, + escalation_span, + main, +) +from simulator.tests.test_simulate import Corpus, FakeHttp, FakeUdp + +# --- the R-P3 ruling restated verbatim (independent of the implementation) ------- + +def selector(index): + """R-P3: (line_index * 2654435761 % 1000) / 1000 — per-line value in [0, 1).""" + return (index * 2654435761 % 1000) / 1000 + + +def syslog_line(i): + return (f"Aug 27 14:32:0{i} fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= " + f"SRC=203.0.113.{i} DST=192.168.1.{i} LEN=60 PROTO=TCP SPT=5142{i} " + f"DPT=22 FLAGS=SYN") + + +def renamed(line): + """R-P9 variant A (index % 3 == 0): field SRC renamed to SRCADDR, value kept.""" + return line.replace("SRC=", "SRCADDR=", 1) + + +def poisoned(line): + """R-P9 variant B (other indices): the SRC value becomes an invalid token.""" + return re.sub(r"\bSRC=\S*", "SRC=uplink-trust-0x4f", line, count=1) + + +# --- corrupt_syslog: R-P3 selection + R-P9 variants ------------------------------- + +def test_p3_selection_covers_a_uniform_fraction_of_indices(): + # 2654435761 % 1000 == 761 is coprime with 1000, so the selector is a + # permutation of k/1000: phase p selects exactly p of any 1000 consecutive + # indices — the escalation is smooth and phase 1.0 selects EVERYTHING. + for phase in (0.1, 0.25, 0.5, 0.9, 1.0): + selected = sum(1 for i in range(1000) if selector(i) < phase) + assert selected == int(phase * 1000) + + +def test_phase_zero_leaves_lines_unchanged(): + for i in range(15): + assert corrupt_syslog(syslog_line(i), i, 0.0) == syslog_line(i) + + +def test_phase_one_corrupts_every_line_with_exact_variant_split(): + lines = [syslog_line(i) for i in range(15)] + outs = [corrupt_syslog(line, i, 1.0) for i, line in enumerate(lines)] + assert all(out != line for out, line in zip(outs, lines)) # every line hit + renames = 0 + for i, (line, out) in enumerate(zip(lines, outs)): + if i % 3 == 0: + renames += 1 + assert out == renamed(line) # SRC -> SRCADDR (null-rate spike) + else: + assert out == poisoned(line) # SRC=uplink-trust-0x4f (tier-1) + assert renames == 5 # exact 1/3 rename : 2/3 value-replace over 15 lines + + +def test_phase_one_variants_pinned_as_full_literals(): + # Independent of any helper: the two variants, spelled out byte for byte. + assert corrupt_syslog(syslog_line(0), 0, 1.0) == ( + "Aug 27 14:32:00 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= " + "SRCADDR=203.0.113.0 DST=192.168.1.0 LEN=60 PROTO=TCP SPT=51420 " + "DPT=22 FLAGS=SYN") + assert corrupt_syslog(syslog_line(1), 1, 1.0) == ( + "Aug 27 14:32:01 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= " + "SRC=uplink-trust-0x4f DST=192.168.1.1 LEN=60 PROTO=TCP SPT=51421 " + "DPT=22 FLAGS=SYN") + + +def test_line_without_src_passes_through_even_at_phase_one(): + bare = "Aug 27 14:32:07 fw01 sshd[1901]: Accepted publickey for root" + for i in range(6): + assert corrupt_syslog(bare, i, 1.0) == bare + + +def test_corrupt_syslog_is_deterministic(): + # same inputs -> same outputs, always: no RNG state anywhere. + for phase in (0.0, 0.044, 0.283, 0.5, 0.761, 1.0): + for i in range(30): + line = syslog_line(i % 15) + assert corrupt_syslog(line, i, phase) == corrupt_syslog(line, i, phase) + + +def test_p3_selection_is_strictly_less_than_phase(): + i = 4 # selector(4) == 0.044: the smallest positive selector below index 5 + line = syslog_line(i) + boundary = selector(i) # exactly the R-P3 value, computed the same way + assert corrupt_syslog(line, i, boundary) == line # equality -> NOT corrupted + just_above = math.nextafter(boundary, 1.0) + assert corrupt_syslog(line, i, just_above) == poisoned(line) # any epsilon above + + +# --- escalation math (R-P3): phase only grows; span never divides by zero -------- + +def test_drift_phase_is_monotonic_and_clamped(): + for span in (1.0, 2.5, 55.0): + phases = [drift_phase(t, span) for t in (0.0, 0.1, 0.3, span / 2, span, span + 5)] + assert phases == sorted(phases) # only grows with elapsed time + assert phases[0] == 0.0 + assert phases[-1] == 1.0 # clamped once the escalation span has elapsed + assert all(0.0 <= p <= 1.0 for p in phases) + assert drift_phase(-1.0, 1.0) == 0.0 # pre-drift elapsed clamps to clean + + +def test_escalation_span_defaults_to_remaining_budget(): + assert escalation_span(60.0, 5.0) == 55.0 # duration - drift_after + assert escalation_span(10.0, 0.0) == 10.0 + + +def test_escalation_span_floors_when_drift_consumes_the_budget(): + assert escalation_span(10.0, 10.0) == ESCALATION_SPAN_FLOOR_S # equal -> floor + assert escalation_span(3.0, 9.0) == ESCALATION_SPAN_FLOOR_S # past the budget + assert ESCALATION_SPAN_FLOOR_S > 0 + assert drift_phase(0.5, escalation_span(10.0, 10.0)) == 0.5 # no ZeroDivision + + +# --- run(): drift wiring (clean before drift_after; only syslog mutates) --------- + +def test_drift_clean_until_drift_after_and_byte_identical_to_steady(): + pristine = {"cef": ["c1", "c2"], "acmegw": ["a1"], + "json": ["j1"], "syslog": [syslog_line(i) for i in range(4)]} + drift_log, steady_log = [], [] + + def corpora_for(log): + return [Corpus(name=k, lines=list(v), + sender=FakeHttp(k, log, batch_size=100) if k == "json" + else FakeUdp(k, log)) + for k, v in pristine.items()] + + _sent_d, err_d = ts.do_run(corpora_for(drift_log), eps=100, duration=0.5, + drift_after=60.0) # drift never due within the run + _sent_s, err_s = ts.do_run(corpora_for(steady_log), eps=100, duration=0.5) + assert err_d == err_s == 0 + assert drift_log == steady_log # same wire content as steady, byte for byte + assert [l for n, l in drift_log if n == "syslog"] == pristine["syslog"] + + +def test_drift_corrupts_only_syslog_after_the_boundary(): + pristine = {"cef": [f"c{i}" for i in range(6)], + "acmegw": [f"a{i}" for i in range(6)], + "json": [f"j{i}" for i in range(6)], + "syslog": [syslog_line(i) for i in range(6)]} + log = [] + corpora = [Corpus(name=k, lines=list(v), + sender=FakeHttp(k, log, batch_size=100) if k == "json" + else FakeUdp(k, log)) + for k, v in pristine.items()] + sent, errors = ts.do_run(corpora, eps=30, duration=0.6, loop=True, drift_after=0.0) + assert errors == 0 + assert sent["syslog"] >= 2 # enough sends to see the escalation + # the other corpora stay byte-identical to their pristine files + for k in ("cef", "acmegw", "json"): + assert {l for n, l in log if n == k} <= set(pristine[k]) + # every mutated syslog line is exactly one of the two R-P9 variants + variants = {renamed(l) if i % 3 == 0 else poisoned(l) + for i, l in enumerate(pristine["syslog"])} + syslog_out = [l for n, l in log if n == "syslog"] + corrupted = [l for l in syslog_out if l not in pristine["syslog"]] + assert corrupted # drift did fire + assert set(corrupted) <= variants + assert renamed(syslog_line(0)) in corrupted # selector(0)=0.0 < any phase>0 + assert poisoned(syslog_line(4)) in corrupted # selector(4)=0.044, phase>0.1 + + +def test_drift_boundary_is_respected_mid_pass(): + # SlowUdp spends 0.05s per send, so one pass straddles drift_after=0.09: + # corruption is decided per send (send-start time), not per pass. + log = [] + slow = ts.SlowUdp("syslog", log, 0.05) + lines = [syslog_line(i) for i in range(6)] + corpora = [Corpus("cef", ["c1"], FakeUdp("cef", log)), + Corpus("syslog", lines, slow)] + sent, errors = ts.do_run(corpora, eps=100, duration=1.0, drift_after=0.09) + assert errors == 0 + assert sent == {"cef": 1, "syslog": 6} # counts unchanged: content only mutates + syslog_sends = [l for n, l in log if n == "syslog"] + assert len(syslog_sends) == 6 + # decided at ~0.00s/~0.05s: before the boundary -> pristine + assert syslog_sends[0] == lines[0] + assert syslog_sends[1] == lines[1] + # past the boundary but selector(i) > phase -> still pristine + assert syslog_sends[2] == lines[2] # selector(2)=0.522 + assert syslog_sends[3] == lines[3] # selector(3)=0.283 + assert syslog_sends[5] == lines[5] # selector(5)=0.805 + # decided at >=0.20s: phase >= 0.11 > selector(4)=0.044 -> P-9 value variant + assert syslog_sends[4] == poisoned(lines[4]) + + +# --- main(): --mode firmware-drift wiring + the M1 stdout contract ---------------- + +def wire(tmp_path, monkeypatch, contents): + ts.make_corpora(tmp_path, contents) + log = [] + + def senders(): + return {k: (FakeHttp(k, log, batch_size=100) if k == "json" else FakeUdp(k, log)) + for k in contents} + + monkeypatch.setattr("simulator.simulate.default_senders", senders) + return log + + +SIM_LINE = re.compile(r"^(\w+) sent=(\d+)$", re.MULTILINE) # the M1 smoke regex + + +def test_main_firmware_drift_stdout_contract(tmp_path, capsys, monkeypatch): + log = wire(tmp_path, monkeypatch, + {"cef": "c1\nc2\n", "json": "j1\n", + "syslog": "\n".join(syslog_line(i) for i in range(3)) + "\n"}) + rc = main(["--eps", "100", "--duration", "1", "--mode", "firmware-drift", + "--drift-after", "60", "--corpora", str(tmp_path)]) + assert rc == 0 + out = capsys.readouterr().out + assert out.strip().splitlines() == [ + "cef sent=2", "json sent=1", "syslog sent=3", "total sent=6"] + matches = SIM_LINE.findall(out) # still exactly one line per corpus + total + assert ("syslog", "3") in matches and ("total", "6") in matches + assert sum(1 for name, _ in matches if name == "syslog") == 1 + assert len(log) == 6 # counts unchanged: only line CONTENT may mutate + + +def test_main_firmware_drift_omits_newapp(tmp_path, capsys, monkeypatch): + wire(tmp_path, monkeypatch, {"cef": "c1\n", "newapp": "n1\n"}) + rc = main(["--eps", "100", "--duration", "1", "--mode", "firmware-drift", + "--corpora", str(tmp_path)]) + out = capsys.readouterr().out.strip().splitlines() + assert rc == 0 + assert out == ["cef sent=1", "total sent=1"] # steady corpus set: no newapp + + +def test_drift_after_flag_defaults_to_five_seconds(): + args = _parse_args(["--mode", "firmware-drift"]) + assert args.drift_after == 5.0 + args = _parse_args(["--mode", "firmware-drift", "--drift-after", "12.5"]) + assert args.drift_after == 12.5 + + +def test_main_firmware_drift_default_after_keeps_short_run_clean( + tmp_path, capsys, monkeypatch): + log = wire(tmp_path, monkeypatch, + {"syslog": "\n".join(syslog_line(i) for i in range(4)) + "\n"}) + rc = main(["--eps", "100", "--duration", "0.3", "--mode", "firmware-drift", + "--corpora", str(tmp_path)]) # default --drift-after 5 > run length + assert rc == 0 + assert [l for n, l in log] == [syslog_line(i) for i in range(4)] # pristine + + +def test_main_two_identical_drift_runs_produce_identical_bytes( + tmp_path, capsys, monkeypatch): + outs, logs = [], [] + for _ in range(2): + log = wire(tmp_path, monkeypatch, + {"cef": "c1\nc2\n", "json": "j1\n", + "syslog": "\n".join(syslog_line(i) for i in range(3)) + "\n"}) + rc = main(["--eps", "100", "--duration", "0.4", "--mode", "firmware-drift", + "--drift-after", "0", "--corpora", str(tmp_path)]) + assert rc == 0 + outs.append(capsys.readouterr().out) + logs.append(log) + assert outs[0] == outs[1] # stdout bytes identical across runs + assert logs[0] == logs[1] # wire content identical too + # index 0 (selector 0.0 < any positive phase) takes the rename variant; + # indices 1-2 (selectors 0.761 / 0.522) stay pristine at these tiny phases + assert [l for n, l in logs[0] if n == "syslog"] == [ + renamed(syslog_line(0)), syslog_line(1), syslog_line(2)] + + +def test_steady_modes_still_reject_no_drift_wiring(tmp_path, capsys, monkeypatch): + # steady/new-appliance never mutate a line even if --drift-after is passed: + # the flag is firmware-drift-only (mirrors --new-after being ignored). + log = wire(tmp_path, monkeypatch, + {"syslog": "\n".join(syslog_line(i) for i in range(3)) + "\n"}) + rc = main(["--eps", "100", "--duration", "0.3", "--drift-after", "0", + "--corpora", str(tmp_path)]) + assert rc == 0 + assert [l for n, l in log] == [syslog_line(i) for i in range(3)] # pristine diff --git a/web/src/App.css b/web/src/App.css index 75c4faa..d1bb62c 100644 --- a/web/src/App.css +++ b/web/src/App.css @@ -587,3 +587,132 @@ body { opacity: 0.5; cursor: not-allowed; } + +/* --- events browser (M3 Task 9) -------------------------------------------- + The table reuses .feed-table (borders, sticky header, row hover) and the + badge-* tokens; the additions below only cover the filter bar, which no + existing class expresses. */ + +.filters { + display: flex; + gap: 12px; + align-items: flex-end; + flex-wrap: wrap; + margin-bottom: 16px; +} + +.filters label { + display: flex; + flex-direction: column; + gap: 4px; + font-size: 12px; +} + +.filters select, +.filters input { + background: var(--bg); + border: 1px solid var(--border); + border-radius: 6px; + color: var(--text); + padding: 6px 8px; + font-size: 13px; +} + +.filters select:focus-visible, +.filters input:focus-visible { + outline: none; + border-color: var(--accent); +} + +.filters button { + background: var(--panel-2); + color: var(--text); + border: 1px solid var(--border); + border-radius: 6px; + padding: 6px 12px; + font-size: 13px; + cursor: pointer; +} + +.filters button:hover { + border-color: var(--accent); +} + +/* --- drift & health (M3 Task 10) -------------------------------------------- + The field-health table reuses .feed-table and the badge-* tokens; the feed + reuses .audit-feed / .audit-sep; the opportunity cards reuse .cards / .card. + The additions below only cover what no existing class expresses: severity + badge colors, non-pointer rows, the un-quarantine button, the section + headers, the merged-feed items and the opportunity card copy. */ + +/* severity ladder: none is quiet, minor warns, moderate is the quarantine + band (echoing badge-quarantined's purple), severe is the deactivate band */ +.badge-none { color: var(--muted); } +.badge-minor { color: var(--warn); } +.badge-moderate { color: var(--quar); } +.badge-severe { color: var(--err); } + +/* section headers inside the page (Field health / Enforcement feed / + Extension opportunities) */ +.drift-section h2 { + font-size: 13px; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--muted); + margin: 22px 0 8px; +} + +/* drift rows are read-only — no row pointer/hover-click affordance */ +.drift-table tbody tr { + cursor: default; +} + +/* the human override button on currently-quarantined field rows */ +.drift-table .unquarantine { + background: rgba(163, 113, 247, 0.14); + border: 1px solid var(--quar); + border-radius: 6px; + color: var(--quar); + padding: 3px 10px; + font-size: 12px; + cursor: pointer; + margin-left: 8px; +} + +.drift-table .unquarantine:hover { + border-color: var(--text); +} + +.drift-table .unquarantine:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +/* the per-row un-quarantine error banner spans the whole table row */ +.drift-error-row td { + cursor: default; + background: var(--panel-2); +} + +/* one merged latest-first feed: window alerts and audit actions alike */ +.drift-feed-item { + display: flex; + align-items: baseline; + gap: 2px; + flex-wrap: wrap; +} + +/* opportunity cards: the "new telemetry key observed" copy stays lowercase + (it reads as a finding, not a header) */ +.opportunity h3 { + margin: 0 0 6px; + font-size: 14px; + text-transform: none; + letter-spacing: 0; + color: var(--text); +} + +.opportunity p { + margin: 4px 0; + font-size: 13px; +} diff --git a/web/src/App.test.tsx b/web/src/App.test.tsx new file mode 100644 index 0000000..5c5c945 --- /dev/null +++ b/web/src/App.test.tsx @@ -0,0 +1,156 @@ +// @vitest-environment jsdom +// web/src/App.test.tsx — the five-tab shell tests (M3 Task 10; spec §12's +// five pages: Overview, Review Queue, Events, Rules, Drift & Health). The +// './api' module is mocked wholesale (house rule: tests never touch the +// network) with empty-resolved fetchers for every page the tree can mount; +// EventSource is stubbed because the default Overview tab subscribes to the +// SSE stream and jsdom ships no EventSource. +import { cleanup, fireEvent, render, screen } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import App from './App' +import { + getDriftAlerts, + getDriftMetrics, + getEvents, + getRules, + getStats, +} from './api' +import type { Stats } from './types' + +vi.mock('./api', () => ({ + getStats: vi.fn(), + getEvents: vi.fn(), + getRaw: vi.fn(), + getRules: vi.fn(), + getRuleHistory: vi.fn(), + getAudit: vi.fn(), + getDriftMetrics: vi.fn(), + getDriftAlerts: vi.fn(), + postJson: vi.fn(), +})) + +const mockGetStats = vi.mocked(getStats) +const mockGetEvents = vi.mocked(getEvents) +const mockGetRules = vi.mocked(getRules) +const mockGetDriftMetrics = vi.mocked(getDriftMetrics) +const mockGetDriftAlerts = vi.mocked(getDriftAlerts) + +/** Minimal EventSource double — the Overview tab only needs construction and + * clean close on switch-away; no events are delivered in these tests. */ +class FakeEventSource { + static readonly CONNECTING = 0 + static readonly OPEN = 1 + static readonly CLOSED = 2 + + url: string + readyState: number = FakeEventSource.OPEN + onmessage: ((ev: MessageEvent) => void) | null = null + onerror: ((ev: Event) => void) | null = null + + constructor(url: string) { + this.url = url + } + + close(): void { + this.readyState = FakeEventSource.CLOSED + } +} + +const emptyStats: Stats = { + raw_total: 0, + by_status: { parsed: 0, unparsed: 0, parse_error: 0, quarantined: 0 }, + by_fingerprint: [], + events_last_minute: 0, + dlq_total: null, +} + +beforeEach(() => { + vi.stubGlobal('EventSource', FakeEventSource as unknown as typeof EventSource) + mockGetStats.mockResolvedValue(emptyStats) + mockGetEvents.mockResolvedValue({ events: [] }) + mockGetRules.mockResolvedValue({ rules: [] }) + mockGetDriftMetrics.mockResolvedValue({ metrics: [] }) + mockGetDriftAlerts.mockResolvedValue({ alerts: [] }) +}) + +afterEach(() => { + cleanup() // vitest without globals does not auto-cleanup RTL renders + vi.unstubAllGlobals() + vi.clearAllMocks() +}) + +describe('App shell — five tabs (spec §12)', () => { + it('renders the five §12 tabs with Overview as the default page', () => { + render() + + const tabs = screen.getAllByRole('tab') + expect(tabs.map((t) => t.textContent)).toEqual([ + 'Overview', + 'Review Queue', + 'Events', + 'Rules', + 'Drift & Health', + ]) + // Overview mounts by default (its SSE feed + stats fetch fired) + expect( + screen.getByRole('heading', { name: 'ULPF Overview' }), + ).not.toBeNull() + expect(mockGetStats).toHaveBeenCalled() + // …and none of the other pages is mounted + expect(screen.queryByRole('heading', { name: 'Events' })).toBeNull() + expect(screen.queryByRole('heading', { name: 'Rules' })).toBeNull() + }) + + it('switches to each of the other four pages, one at a time', async () => { + render() + + fireEvent.click(screen.getByRole('tab', { name: 'Events' })) + expect( + await screen.findByRole('heading', { name: 'Events' }), + ).not.toBeNull() + expect(mockGetEvents).toHaveBeenCalledWith({ + status: undefined, + fingerprint: undefined, + limit: 50, + }) + expect( + screen.queryByRole('heading', { name: 'ULPF Overview' }), + ).toBeNull() + + fireEvent.click(screen.getByRole('tab', { name: 'Review Queue' })) + expect( + await screen.findByRole('heading', { name: 'Review Queue' }), + ).not.toBeNull() + expect(mockGetRules).toHaveBeenCalledWith({ status: 'pending_review' }) + expect(screen.queryByRole('heading', { name: 'Events' })).toBeNull() + + fireEvent.click(screen.getByRole('tab', { name: 'Rules' })) + expect( + await screen.findByRole('heading', { name: 'Rules' }), + ).not.toBeNull() + expect(mockGetRules).toHaveBeenLastCalledWith() + + fireEvent.click(screen.getByRole('tab', { name: 'Drift & Health' })) + expect( + await screen.findByRole('heading', { name: 'Drift & Health' }), + ).not.toBeNull() + expect(mockGetDriftMetrics).toHaveBeenCalled() + expect(mockGetDriftAlerts).toHaveBeenCalled() + expect(screen.queryByRole('heading', { name: 'Rules' })).toBeNull() + }) + + it('marks exactly the active tab aria-selected', () => { + render() + const tabs = screen.getAllByRole('tab') + expect(tabs[0]?.getAttribute('aria-selected')).toBe('true') + expect(tabs[4]?.getAttribute('aria-selected')).toBe('false') + + fireEvent.click(tabs[4]) + expect(tabs[4]?.getAttribute('aria-selected')).toBe('true') + expect(tabs[0]?.getAttribute('aria-selected')).toBe('false') + // the fifth page is Drift & Health + expect( + screen.getByRole('heading', { name: 'Drift & Health' }), + ).not.toBeNull() + }) +}) diff --git a/web/src/App.tsx b/web/src/App.tsx index 0c91aca..32a3e27 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,25 +1,32 @@ -// web/src/App.tsx — the three-tab M2 shell: Overview (the M1 dashboard), -// Review Queue (Task 8 human review loop) and the Rules registry + audit -// trail (Task 9). +// web/src/App.tsx — the five-tab M3 shell (spec §12's five pages, in spec +// order): Overview (the M1 dashboard), Review Queue (Task 8 human review +// loop), the Events browser (Task 9), the Rules registry + audit trail +// (Task 9) and Drift & Health (Task 10, the fifth page). // Tab state is plain useState — no router (air-gapped dashboard: system // fonts, zero external requests). Same-origin /api only (dev: Vite proxy, // prod: Caddy) — the gateway ships no CORS. import { useCallback, useEffect, useState } from 'react' -import { getRaw, getStats } from './api' -import type { EventRow, OcsfEndpoint, RawTrace, Stats } from './types' +import { getStats } from './api' +import type { EventRow, Stats } from './types' +import { endpointIp, formatTime } from './cells' import { useEventStream } from './useEventStream' import ReviewQueue from './ReviewQueue' +import Events from './Events' import Rules from './Rules' +import DriftHealth from './DriftHealth' +import TraceDrawer from './components/TraceDrawer' import './App.css' const STATS_REFRESH_MS = 5000 -type Tab = 'overview' | 'review' | 'rules' +type Tab = 'overview' | 'review' | 'events' | 'rules' | 'drift' const TABS: { id: Tab; label: string }[] = [ { id: 'overview', label: 'Overview' }, { id: 'review', label: 'Review Queue' }, + { id: 'events', label: 'Events' }, { id: 'rules', label: 'Rules' }, + { id: 'drift', label: 'Drift & Health' }, ] /** parsed / (parsed + unparsed + parse_error + quarantined); "—" before any @@ -31,14 +38,6 @@ function parsedPercent(byStatus: Stats['by_status']): string { return `${Math.round((byStatus.parsed / total) * 100)}%` } -function endpointIp(ep: OcsfEndpoint | null): string { - return ep?.ip ?? '—' -} - -function formatTime(iso: string): string { - return new Date(iso).toLocaleTimeString() -} - interface StatCard { label: string value: string | number @@ -62,88 +61,8 @@ function statCards(stats: Stats | null): StatCard[] { ] } -/** Side drawer: full OCSF document plus the raw line behind the event - * (fetched by event_id; the fetch is aborted via AbortController if the - * drawer closes first). */ -function TraceDrawer({ row, onClose }: { row: EventRow; onClose: () => void }) { - const [trace, setTrace] = useState(null) - const [error, setError] = useState(null) - - useEffect(() => { - const controller = new AbortController() - getRaw(row.event_id, controller.signal) - .then((t) => setTrace(t)) - .catch((err: unknown) => { - if (!controller.signal.aborted) { - setError(err instanceof Error ? err.message : String(err)) - } - }) - return () => controller.abort() - }, [row.event_id]) - - useEffect(() => { - const onKey = (ev: KeyboardEvent) => { - if (ev.key === 'Escape') onClose() - } - window.addEventListener('keydown', onKey) - return () => window.removeEventListener('keydown', onKey) - }, [onClose]) - - return ( - - ) -} - /** The M1 Overview page: 4 stat cards, the SSE live-feed table, and the - * raw-traceability drawer. Unchanged behavior. */ + * raw-traceability drawer (shared component, components/TraceDrawer). */ function Overview() { const [stats, setStats] = useState(null) const [statsError, setStatsError] = useState(null) @@ -257,7 +176,9 @@ export default function App() { {tab === 'overview' && } {tab === 'review' && } + {tab === 'events' && } {tab === 'rules' && } + {tab === 'drift' && } ) } diff --git a/web/src/DriftHealth.test.tsx b/web/src/DriftHealth.test.tsx new file mode 100644 index 0000000..134b97c --- /dev/null +++ b/web/src/DriftHealth.test.tsx @@ -0,0 +1,414 @@ +// @vitest-environment jsdom +// web/src/DriftHealth.test.tsx — the Drift & Health page tests (M3 Task 10, +// the fifth §12 page). The './api' module is mocked wholesale (house rule: +// tests never touch the network); this file's tree imports getDriftMetrics, +// getDriftAlerts, getRuleHistory (the page — the last is the P-8 un-quarantine +// gate source) and postJson (the un-quarantine button). +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import DriftHealth from './DriftHealth' +import { getDriftAlerts, getDriftMetrics, getRuleHistory, postJson } from './api' +import type { + AuditRow, + DriftAlertRow, + DriftMetricRow, + RuleHistoryResponse, + RuleRow, + UnquarantineResponse, +} from './types' + +vi.mock('./api', () => ({ + getDriftMetrics: vi.fn(), + getDriftAlerts: vi.fn(), + getRuleHistory: vi.fn(), + postJson: vi.fn(), +})) + +const mockGetDriftMetrics = vi.mocked(getDriftMetrics) +const mockGetDriftAlerts = vi.mocked(getDriftAlerts) +const mockGetRuleHistory = vi.mocked(getRuleHistory) +const mockPostJson = vi.mocked(postJson) + +/** A severe window row for fp-acme-fw / src_endpoint.ip — the firmware-drift + * demo shape: violation rate through the tier-1 ladder, ipv4-dominant. */ +function metricRow(overrides: Partial = {}): DriftMetricRow { + return { + fingerprint_id: 'fp-acme-fw', + rule_version: 3, + field: 'src_endpoint.ip', + window_start: '2026-09-22T10:00:00Z', + window_end: '2026-09-22T10:05:00Z', + events_count: 240, + null_rate: 0.42, + match_rate: null, + violation_rate: 0.61, + shape_dist: { ipv4: 90, int: 10 }, + severity: 'severe', + action_taken: 'rule_deactivated', + ...overrides, + } +} + +/** A window-kind alert row (the row IS the alert). */ +function windowAlert(overrides: Partial = {}): DriftAlertRow { + return { ...metricRow(overrides), kind: 'window' } +} + +/** An audit-kind alert row from the enforcement vocabulary. */ +function auditAlert(overrides: Partial = {}): DriftAlertRow { + return { + id: 41, + ts: '2026-09-22T10:06:00Z', + actor: 'drift', + action: 'field_quarantined', + entity: 'fp-acme-fw', + detail: { field: 'dst_endpoint.ip', rule_id: 3, version: 3 }, + kind: 'audit', + ...overrides, + } +} + +/** POST /api/rules/{fp}/unquarantine 200 response shape. */ +const unquarantineOk: UnquarantineResponse = { + fingerprint_id: 'fp-acme-fw', + field: 'dst_endpoint.ip', + quarantined_fields: [], +} + +/** One GET /api/rules/{fp} rule row (all 14 gateway _RULE_COLUMNS keys); the + * default is the fp-acme-fw ACTIVE v3 whose quarantined_fields gate the + * Un-quarantine button (P-8). */ +function historyRule(overrides: Partial = {}): RuleRow { + return { + id: 7, + fingerprint_id: 'fp-acme-fw', + version: 3, + pattern: '^.*?(?P.*)$', + mappings: [{ source_field: 'SRCADDR', ocsf_path: 'src_endpoint.ip' }], + provenance: 'slm', + confidence: 0.93, + status: 'active', + quarantined_fields: [], + created_by: 'onboarding', + created_at: '2026-09-19T09:00:00Z', + activated_at: '2026-09-19T09:05:00Z', + deactivated_at: null, + validation: null, + ...overrides, + } +} + +/** GET /api/rules/{fp} response for a fingerprint whose ACTIVE rule + * quarantines exactly `quarantined` (and nothing else). */ +function ruleHistory(quarantined: string[] = []): RuleHistoryResponse { + return { rules: [historyRule({ quarantined_fields: quarantined })], audit: [] } +} + +afterEach(() => { + cleanup() // vitest without globals does not auto-cleanup RTL renders + vi.clearAllMocks() +}) + +describe('Drift & Health — Field health', () => { + it('renders one row per (fp, field) metric: severity badge, rates, shape top class, action, topbar counts', async () => { + mockGetDriftMetrics.mockResolvedValue({ + metrics: [ + metricRow(), + metricRow({ + field: 'dst_endpoint.ip', + window_start: '2026-09-22T09:55:00Z', + window_end: '2026-09-22T10:00:00Z', + null_rate: 0, + violation_rate: 0.05, + shape_dist: {}, // empty histogram — shape top renders '—' + severity: 'moderate', + action_taken: 'field_quarantined', + }), + ], + }) + mockGetDriftAlerts.mockResolvedValue({ alerts: [] }) + mockGetRuleHistory.mockResolvedValue(ruleHistory()) + const { container } = render() + + // both feeds load on mount + await screen.findAllByText('fp-acme-fw') // one cell per row (same fp) + expect(mockGetDriftMetrics).toHaveBeenCalledWith() + expect(mockGetDriftAlerts).toHaveBeenCalledWith() + + // identity cells are mono; severity badges carry badge- + expect(screen.getAllByText('fp-acme-fw').length).toBeGreaterThanOrEqual(2) + expect(container.querySelector('.badge-severe')).not.toBeNull() + expect(container.querySelector('.badge-moderate')).not.toBeNull() + // rates render as rounded percentages ('—' where the window measures none) + expect(screen.getByText('42%')).not.toBeNull() // null_rate + expect(screen.getByText('61%')).not.toBeNull() // violation_rate + expect(screen.getByText('0%')).not.toBeNull() // second row null_rate + // shape top class = argmax of shape_dist; '—' when the histogram is empty + // (also '—' for the null match_rate — __rule__-only measurement) + expect(screen.getByText('ipv4')).not.toBeNull() + expect(screen.getAllByText('—').length).toBeGreaterThanOrEqual(2) + // action strings render verbatim (the string IS the label) + expect(screen.getByText('rule_deactivated')).not.toBeNull() + expect(screen.getByText('field_quarantined')).not.toBeNull() + // events count rides each row (both fixtures carry 240) + expect(screen.getAllByText('240')).toHaveLength(2) + // topbar counts both feeds + expect( + screen.getByText('2 field signal(s) · 0 alert(s)'), + ).not.toBeNull() + }) + + it('renders __rule__ sentinel rows with their match_rate', async () => { + mockGetDriftMetrics.mockResolvedValue({ + metrics: [ + metricRow({ + field: '__rule__', + null_rate: null, + match_rate: 0.93, + violation_rate: null, + shape_dist: null, + severity: 'minor', + action_taken: 'alert', + }), + ], + }) + mockGetDriftAlerts.mockResolvedValue({ alerts: [] }) + mockGetRuleHistory.mockResolvedValue(ruleHistory()) + render() + await screen.findByText('__rule__') + + expect(screen.getByText('93%')).not.toBeNull() + // null_rate / violation_rate / shape_dist are all null on a sentinel row + expect(screen.getAllByText('—')).toHaveLength(3) + }) + + it('shows the empty states when no windows have closed and no alerts exist', async () => { + mockGetDriftMetrics.mockResolvedValue({ metrics: [] }) + mockGetDriftAlerts.mockResolvedValue({ alerts: [] }) + render() + await screen.findByText(/no closed drift windows yet/i) + expect(screen.getByText(/no drift alerts/i)).not.toBeNull() + expect(screen.getByText(/no unmapped telemetry keys observed/i)).not.toBeNull() + }) +}) + +describe('Drift & Health — Enforcement feed', () => { + it('renders the ONE merged feed in endpoint order, discriminated by kind (no client re-sort)', async () => { + // Deliberately NOT time-sorted: the endpoint's latest-first order is the + // contract (P-4) and the page must render the array verbatim — a client + // re-sort would reorder these rows. + mockGetDriftMetrics.mockResolvedValue({ metrics: [metricRow()] }) + mockGetRuleHistory.mockResolvedValue(ruleHistory()) + mockGetDriftAlerts.mockResolvedValue({ + alerts: [ + windowAlert({ window_start: '2026-09-22T10:00:00Z', severity: 'severe' }), + auditAlert({ + id: 44, + ts: '2026-09-22T10:30:00Z', + actor: 'karthik', + action: 'field_unquarantined', + detail: { field: 'src_endpoint.ip', rule_id: 3, version: 3, reason: 'fixed' }, + }), + windowAlert({ + field: 'dst_endpoint.ip', + window_start: '2026-09-22T09:00:00Z', + severity: 'minor', + action_taken: 'alert', + shape_dist: {}, + }), + auditAlert({ + id: 40, + ts: '2026-09-22T09:10:00Z', + action: 'rule_deactivated', + detail: { rule_id: 3, version: 3, window_start: '2026-09-22T09:00:00Z' }, + }), + ], + }) + const { container } = render() + await screen.findByText(/alert\(s\)/) + + const items = container.querySelectorAll('.drift-feed li') + expect(items).toHaveLength(4) + // item order is the endpoint's array order, verbatim + expect(items[0]?.textContent).toContain('fp-acme-fw') + expect(items[0]?.textContent).toContain('src_endpoint.ip') + expect(items[0]?.textContent).toContain('severe') + // window rows carry no actor — the audit rows do + expect(items[0]?.textContent).not.toContain('karthik') + expect(items[1]?.textContent).toContain('karthik') + expect(items[1]?.textContent).toContain('field_unquarantined') + expect(items[1]?.textContent).toContain('src_endpoint.ip') + expect(items[2]?.textContent).toContain('dst_endpoint.ip') + expect(items[2]?.textContent).toContain('minor') + expect(items[2]?.textContent).toContain('alert') + // rule_deactivated audit rows have no field in detail — '—' + expect(items[3]?.textContent).toContain('drift') + expect(items[3]?.textContent).toContain('rule_deactivated') + expect(items[3]?.textContent).toContain('—') + // topbar counts the merged feed + expect( + screen.getByText('1 field signal(s) · 4 alert(s)'), + ).not.toBeNull() + }) +}) + +describe('Drift & Health — Un-quarantine', () => { + it("gates on the ACTIVE rule version's quarantined_fields (P-8), POSTs {field, actor:'ui'} and refetches feeds + histories", async () => { + const metrics = [ + metricRow({ field: 'src_endpoint.ip' }), + metricRow({ field: 'dst_endpoint.ip' }), + ] + mockGetDriftMetrics.mockResolvedValue({ metrics }) + mockGetDriftAlerts.mockResolvedValue({ alerts: [] }) // display-only now + // the ACTIVE rule quarantines exactly dst_endpoint.ip (newest first) + mockGetRuleHistory.mockResolvedValueOnce( + ruleHistory(['dst_endpoint.ip']), + ) + // post-click refetch: the active rule quarantines nothing anymore + mockGetRuleHistory.mockResolvedValueOnce(ruleHistory([])) + mockPostJson.mockResolvedValueOnce(unquarantineOk) + render() + // the field shows in its table row + await screen.findAllByText('dst_endpoint.ip') + + // exactly one button — on the dst row, not the src row + const buttons = screen.getAllByRole('button', { name: 'Un-quarantine' }) + expect(buttons).toHaveLength(1) + expect(buttons[0]?.closest('tr')?.textContent).toContain('dst_endpoint.ip') + // the gate asked for the (single) metrics fingerprint's rule history + expect(mockGetRuleHistory).toHaveBeenCalledWith('fp-acme-fw') + + fireEvent.click(buttons[0]) + await waitFor(() => { + expect(mockPostJson).toHaveBeenCalledWith( + '/api/rules/fp-acme-fw/unquarantine', + { field: 'dst_endpoint.ip', actor: 'ui' }, + ) + }) + // everything was refetched and the button left with the recomputed gate + await waitFor(() => { + expect(screen.queryByRole('button', { name: 'Un-quarantine' })).toBeNull() + }) + expect(mockGetDriftMetrics).toHaveBeenCalledTimes(2) + expect(mockGetDriftAlerts).toHaveBeenCalledTimes(2) + expect(mockGetRuleHistory).toHaveBeenCalledTimes(2) + }) + + it('fetches one rule history per DISTINCT metrics fingerprint', async () => { + mockGetDriftMetrics.mockResolvedValue({ + metrics: [ + metricRow({ field: 'src_endpoint.ip' }), + metricRow({ field: 'dst_endpoint.ip' }), // same fp — deduped + metricRow({ fingerprint_id: 'fp-acme-db', field: 'src_endpoint.ip' }), + ], + }) + mockGetDriftAlerts.mockResolvedValue({ alerts: [] }) + mockGetRuleHistory.mockResolvedValue(ruleHistory()) + render() + await screen.findAllByText('fp-acme-db') + + expect(mockGetRuleHistory.mock.calls).toEqual([ + ['fp-acme-fw'], + ['fp-acme-db'], + ]) + }) + + it('gates nothing when the newest version is inactive — the ACTIVE one decides', async () => { + mockGetDriftMetrics.mockResolvedValue({ + metrics: [metricRow({ field: 'src_endpoint.ip' })], + }) + mockGetDriftAlerts.mockResolvedValue({ alerts: [] }) + mockGetRuleHistory.mockResolvedValue({ + rules: [ + // newest (v4) deactivated but still carrying the field in its list; + // the active v3 quarantines nothing — the gate follows the ACTIVE one + historyRule({ version: 4, status: 'deactivated', quarantined_fields: ['src_endpoint.ip'] }), + historyRule({ version: 3 }), + ], + audit: [], + }) + render() + await screen.findAllByText('src_endpoint.ip') + + expect(screen.queryByRole('button', { name: 'Un-quarantine' })).toBeNull() + }) + + it('shows a row-level error banner when the POST fails (409: not quarantined)', async () => { + mockGetDriftMetrics.mockResolvedValue({ + metrics: [metricRow({ field: 'dst_endpoint.ip' })], + }) + mockGetDriftAlerts.mockResolvedValue({ alerts: [] }) + mockGetRuleHistory.mockResolvedValue(ruleHistory(['dst_endpoint.ip'])) + mockPostJson.mockRejectedValueOnce( + new Error( + 'POST /api/rules/fp-acme-fw/unquarantine failed: field ' + + "'dst_endpoint.ip' is not quarantined on the active rule", + ), + ) + render() + fireEvent.click(await screen.findByRole('button', { name: 'Un-quarantine' })) + + const banner = await screen.findByRole('alert') + expect(banner.textContent).toMatch(/un-quarantine failed/) + expect(banner.textContent).toMatch(/not quarantined/) + }) +}) + +describe('Drift & Health — Extension opportunities', () => { + it('renders one distinct card per unmapped.* metric row, titled "new telemetry key observed"', async () => { + mockGetDriftMetrics.mockResolvedValue({ + metrics: [ + metricRow({ + field: 'unmapped.SRCADDR', + null_rate: 0.98, + violation_rate: null, + shape_dist: { free_text: 5 }, + severity: 'minor', + action_taken: 'alert', + events_count: 50, + }), + metricRow({ + field: 'unmapped.fw_build', + null_rate: 0, + violation_rate: null, + shape_dist: {}, + severity: 'none', + action_taken: null, + events_count: 50, + }), + metricRow(), // a mapped field — no card + ], + }) + mockGetDriftAlerts.mockResolvedValue({ alerts: [] }) + mockGetRuleHistory.mockResolvedValue(ruleHistory()) + render() + // the key shows in its table row AND its opportunity card + await screen.findAllByText('unmapped.SRCADDR') + + const titles = screen.getAllByText('new telemetry key observed') + expect(titles).toHaveLength(2) + // each unmapped key shows in its table row AND its card + expect(screen.getAllByText('unmapped.fw_build')).toHaveLength(2) + // the mapped field stays in the field-health table but gets no card copy + expect(screen.getAllByText('src_endpoint.ip').length).toBe(1) + }) +}) + +describe('Drift & Health — load failure', () => { + it('shows an error banner when the feeds fail, and Retry recovers', async () => { + mockGetDriftMetrics.mockRejectedValueOnce( + new Error('GET /api/drift/metrics failed: 500 Internal Server Error'), + ) + mockGetDriftAlerts.mockResolvedValue({ alerts: [] }) + render() + const banner = await screen.findByRole('alert') + expect(banner.textContent).toMatch(/drift data unavailable/) + expect(banner.textContent).toMatch(/500/) + + mockGetDriftMetrics.mockResolvedValueOnce({ metrics: [metricRow()] }) + mockGetRuleHistory.mockResolvedValue(ruleHistory()) + fireEvent.click(screen.getByRole('button', { name: 'Retry' })) + expect(await screen.findByText('fp-acme-fw')).not.toBeNull() + expect(screen.queryByRole('alert')).toBeNull() + }) +}) diff --git a/web/src/DriftHealth.tsx b/web/src/DriftHealth.tsx new file mode 100644 index 0000000..afb7e15 --- /dev/null +++ b/web/src/DriftHealth.tsx @@ -0,0 +1,358 @@ +// web/src/DriftHealth.tsx — the Drift & Health page (M3 Task 10, the fifth +// and final §12 page): field health across the latest closed drift windows, +// the merged enforcement/alert feed, and unmapped.* extension opportunities. +// +// Two fetches fire together on mount and after every action (one effect, one +// alive flag, Promise.all — one shared Retry banner when either fails). The +// alerts endpoint (ruling P-4) already merges window rows and audit actions +// into ONE latest-first list — this page renders that array verbatim, +// discriminated only by the `kind` field, and never re-merges or re-sorts +// client-side. +// +// Un-quarantine gating (ruling P-8): once the metrics land, one +// getRuleHistory(fp) per DISTINCT fingerprint they name supplies the ACTIVE +// version's quarantined_fields — the authoritative quarantine state (the +// gateway now exposes the column on every rule row; migration 001's +// rules_one_active partial unique index guarantees at most one active +// version per fingerprint, and the un-quarantine POST itself 409s unless the +// rule is active, so a fingerprint with no active version gates nothing). +// The histories load with the feeds or fail the page as a unit. +// +// Air-gap rules: same-origin /api only, system fonts, zero external requests. +import { useCallback, useEffect, useState } from 'react' +import { getDriftAlerts, getDriftMetrics, getRuleHistory, postJson } from './api' +import type { + AuditRow, + DriftAlertRow, + DriftMetricRow, + RuleHistoryResponse, +} from './types' +import { formatTime } from './cells' + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err) +} + +/** 0..1 rate -> rounded percentage; '—' where the window measures no rate. */ +function pctOrNull(rate: number | null): string { + return rate === null ? '—' : `${Math.round(rate * 100)}%` +} + +/** "Shape top class" = the argmax key of shape_dist; '—' when the histogram + * is empty or null (no observed values / __rule__ rows). */ +function shapeTop(dist: Record | null): string { + if (dist === null) return '—' + let best: string | null = null + let bestCount = -1 + for (const [cls, count] of Object.entries(dist)) { + if (count > bestCount) { + best = cls + bestCount = count + } + } + return best ?? '—' +} + +/** detail.field of an audit row, when the action carried one + * (rule_deactivated details name the rule, not a field). */ +function auditField(row: AuditRow): string | undefined { + if (row.detail === null) return undefined + const detail = row.detail as { field?: unknown } + return typeof detail.field === 'string' ? detail.field : undefined +} + +/** Currently-quarantined fields per fingerprint, from each fingerprint's + * rule history (ruling P-8): the ACTIVE version's quarantined_fields — not + * the newest version's, which may be a deactivated row still carrying the + * list. Fingerprints with no active version gate nothing. */ +function quarantinedByFingerprint( + histories: RuleHistoryResponse[], +): Map> { + const quarantined = new Map>() + for (const { rules } of histories) { + const active = rules.find((rule) => rule.status === 'active') + if (active !== undefined) { + quarantined.set(active.fingerprint_id, new Set(active.quarantined_fields)) + } + } + return quarantined +} + +/** Stable React key for a merged-feed row (window rows key on the + * deterministic drift_windows key; audit rows on their audit id). */ +function alertKey(row: DriftAlertRow): string { + return row.kind === 'window' + ? `window:${row.fingerprint_id}:${row.rule_version}:${row.field}:${row.window_start}` + : `audit:${row.id}` +} + +/** One Field-health row: the latest closed window for a (fingerprint, field). + * The Un-quarantine button (human override, actor "ui") shows only while the + * field sits in the active rule's quarantined_fields (P-8); its POST + * refetches the feeds and histories so the gate, the badges and the + * enforcement feed all move together. */ +function FieldHealthRow({ + row, + quarantined, + refresh, +}: { + row: DriftMetricRow + quarantined: boolean + refresh: () => void +}) { + const [busy, setBusy] = useState(false) + const [actionError, setActionError] = useState(null) + + const unquarantine = () => { + setBusy(true) + setActionError(null) + postJson(`/api/rules/${encodeURIComponent(row.fingerprint_id)}/unquarantine`, { + field: row.field, + actor: 'ui', + }) + .then(() => refresh()) // refetch — the gate recomputes, the button leaves + .catch((err: unknown) => setActionError(errorMessage(err))) + .finally(() => setBusy(false)) + } + + return ( + <> + + {row.fingerprint_id} + {row.field} + + {row.severity} + + {pctOrNull(row.null_rate)} + {pctOrNull(row.match_rate)} + {pctOrNull(row.violation_rate)} + {shapeTop(row.shape_dist)} + {row.events_count} + {formatTime(row.window_end)} + + {row.action_taken ?? '—'} + {quarantined && ( + + )} + + + {actionError !== null && ( + + +

+ un-quarantine failed: {actionError} +

+ + + )} + + ) +} + +/** The Drift & Health tab: field health, the merged enforcement feed, and + * unmapped.* extension opportunities. */ +export default function DriftHealth() { + const [metrics, setMetrics] = useState(null) + const [alerts, setAlerts] = useState(null) + const [quarantined, setQuarantined] = useState> | null>( + null, + ) + const [error, setError] = useState(null) + const [reloadTick, setReloadTick] = useState(0) + + useEffect(() => { + let alive = true + // Both feeds as a unit; the rule histories follow the metrics (their + // fingerprints come from them) and the un-quarantine gate (P-8) loads + // with the rest or fails the page — one shared Retry banner. + Promise.all([getDriftMetrics(), getDriftAlerts()]) + .then(([metricsRes, alertsRes]) => { + const fingerprints = [ + ...new Set(metricsRes.metrics.map((row) => row.fingerprint_id)), + ] + return Promise.all(fingerprints.map((fp) => getRuleHistory(fp))).then( + (histories) => [metricsRes, alertsRes, histories] as const, + ) + }) + .then(([metricsRes, alertsRes, histories]) => { + if (!alive) return + setMetrics(metricsRes.metrics) + setAlerts(alertsRes.alerts) + setQuarantined(quarantinedByFingerprint(histories)) + setError(null) + }) + .catch((err: unknown) => { + if (!alive) return + setError(errorMessage(err)) + }) + return () => { + alive = false + } + }, [reloadTick]) + + const refresh = useCallback(() => setReloadTick((n) => n + 1), []) + + const opportunities = + metrics === null ? null : metrics.filter((r) => r.field.startsWith('unmapped.')) + + return ( +
+
+

Drift & Health

+ + {metrics === null ? '…' : metrics.length} field signal(s) ·{' '} + {alerts === null ? '…' : alerts.length} alert(s) + +
+ + {error !== null && ( +

+ drift data unavailable: {error}{' '} + +

+ )} + + {metrics === null && alerts === null && error === null && ( +

Loading drift data…

+ )} + +
+

Field health

+ {metrics === null && error === null && ( +

Loading windows…

+ )} + {metrics !== null && metrics.length === 0 && ( +

+ No closed drift windows yet — field health appears once the drift + loop has closed a window. +

+ )} + {metrics !== null && metrics.length > 0 && ( +
+ + + + + + + + + + + + + + + + + {metrics.map((row) => ( + + ))} + +
FingerprintFieldSeverityNullMatchViolationShape topEventsWindow endAction
+
+ )} +
+ +
+

Enforcement feed

+ {alerts === null && error === null && ( +

Loading alerts…

+ )} + {alerts !== null && alerts.length === 0 && ( +

+ No drift alerts or enforcement actions yet — windows below the + severity thresholds and quiet enforcement show nothing here. +

+ )} + {alerts !== null && alerts.length > 0 && ( +
    + {alerts.map((row) => ( +
  • + {row.kind === 'window' ? ( + <> + {row.window_start} + · + {row.fingerprint_id} + · + {row.field} + · + + {row.severity} + + · + {row.action_taken ?? '—'} + + ) : ( + <> + {row.ts} + · + {row.actor} + · + {row.action} + · + {row.entity} + · + {auditField(row) ?? '—'} + + )} +
    • + ))} +
+ )} +
+ +
+

Extension opportunities

+ {opportunities === null && error === null && ( +

Loading opportunities…

+ )} + {opportunities !== null && opportunities.length === 0 && ( +

No unmapped telemetry keys observed.

+ )} + {opportunities !== null && opportunities.length > 0 && ( +
+ {opportunities.map((row) => ( +
+

new telemetry key observed

+

{row.field}

+

+ {row.fingerprint_id} ·{' '} + {row.events_count} events · window ended{' '} + {formatTime(row.window_end)} +

+

+ + {row.severity} + {' '} + {row.action_taken ?? '—'} +

+
+ ))} +
+ )} +
+
+ ) +} diff --git a/web/src/Events.test.tsx b/web/src/Events.test.tsx new file mode 100644 index 0000000..9e20cc1 --- /dev/null +++ b/web/src/Events.test.tsx @@ -0,0 +1,199 @@ +// @vitest-environment jsdom +// web/src/Events.test.tsx — the Events browser tests (M3 Task 9). The +// './api' module is mocked wholesale (house rule: tests never touch the +// network); this file's tree imports getEvents (Events) and getRaw (the +// shared TraceDrawer), so those are the exports the mock carries. +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import Events from './Events' +import { getEvents, getRaw } from './api' +import type { EventRow, RawTrace } from './types' + +vi.mock('./api', () => ({ + getEvents: vi.fn(), + getRaw: vi.fn(), +})) + +const mockGetEvents = vi.mocked(getEvents) +const mockGetRaw = vi.mocked(getRaw) + +/** A parsed row for fp-acme-fw with both endpoints mapped. */ +function eventRow(overrides: Partial = {}): EventRow { + return { + event_id: '3f2bd0a6-1b1e-4c5a-9d3e-a1b2c3d4e5f6', + raw_id: '9a8b7c6d-5e4f-4a3b-8c2d-0e9f8a7b6c5d', + fingerprint_id: 'fp-acme-fw', + rule_version: 3, + status: 'parsed', + parsed_at: '2026-09-21T18:04:05Z', + ocsf: { + class_uid: 4001, + class_name: 'Network Activity', + activity_id: 0, + severity_id: null, + time: null, + src_endpoint: { ip: '10.0.0.1', port: 445 }, + dst_endpoint: { ip: '10.0.0.2', port: 445 }, + action: 'DROP', + message: null, + metadata: { product: 'ULPF' }, + unmapped: {}, + }, + ...overrides, + } +} + +/** The raw line behind the parsed row (GET /api/events/{id}/raw shape). */ +const rawTrace: RawTrace = { + raw_id: '9a8b7c6d-5e4f-4a3b-8c2d-0e9f8a7b6c5d', + received_at: '2026-09-21T18:04:04Z', + source_id: 'acme-fw-01', + transport: 'file', + content_hash: 'sha256:abc123', + raw_text: 'Sep 21 18:04:05 fw1 kernel: DROP tcp 10.0.0.1:445 -> 10.0.0.2:445', +} + +/** One parsed + one unparsed row: the second exercises ocsf-null rendering + * (— endpoints, — rule version). */ +const twoRows: EventRow[] = [ + eventRow(), + eventRow({ + event_id: 'c81d4d2e-7f6a-4b0c-9e5d-4f3e2d1c0b0a', + raw_id: '01234567-89ab-4cde-8f01-234567890abc', + fingerprint_id: 'fp-br-dns', + rule_version: null, + status: 'unparsed', + parsed_at: '2026-09-21T18:04:07Z', + ocsf: null, + }), +] + +afterEach(() => { + cleanup() // vitest without globals does not auto-cleanup RTL renders +}) + +describe('Events browser', () => { + it('renders rows from the current-view fetch: badges, mono fingerprints, rule v, src → dst', async () => { + mockGetEvents.mockResolvedValue({ events: twoRows }) + const { container } = render() + + // mount fetch: no status/fingerprint filter, default limit 50 + await screen.findByText('fp-acme-fw') + expect(mockGetEvents).toHaveBeenCalledWith({ + status: undefined, + fingerprint: undefined, + limit: 50, + }) + + // both fingerprints render in mono cells + expect(screen.getByText('fp-acme-fw').className).toBe('mono') + expect(screen.getByText('fp-br-dns').className).toBe('mono') + // status badges carry the badge- classes + expect(container.querySelector('.badge-parsed')).not.toBeNull() + expect(container.querySelector('.badge-unparsed')).not.toBeNull() + // parsed row: rule version + mapped endpoints; unparsed row: — / — → — + expect(screen.getByText('3')).not.toBeNull() + expect(screen.getByText('10.0.0.1 → 10.0.0.2')).not.toBeNull() + expect(screen.getAllByText('—').length).toBeGreaterThanOrEqual(1) + expect(screen.getByText('— → —')).not.toBeNull() + // topbar counts the current view + expect(screen.getByText('2 event(s) in the current view')).not.toBeNull() + }) + + it('forwards filter changes to getEvents: status, fingerprint, limit', async () => { + mockGetEvents.mockResolvedValue({ events: twoRows }) + render() + await screen.findByText('fp-acme-fw') + + fireEvent.change(screen.getByLabelText('status'), { + target: { value: 'parsed' }, + }) + await waitFor(() => { + expect(mockGetEvents).toHaveBeenLastCalledWith({ + status: 'parsed', + fingerprint: undefined, + limit: 50, + }) + }) + + fireEvent.change(screen.getByLabelText('fingerprint'), { + target: { value: 'fp-acme-fw' }, + }) + await waitFor(() => { + expect(mockGetEvents).toHaveBeenLastCalledWith({ + status: 'parsed', + fingerprint: 'fp-acme-fw', + limit: 50, + }) + }) + + fireEvent.change(screen.getByLabelText('limit'), { + target: { value: '100' }, + }) + await waitFor(() => { + expect(mockGetEvents).toHaveBeenLastCalledWith({ + status: 'parsed', + fingerprint: 'fp-acme-fw', + limit: 100, + }) + }) + }) + + it('clicking a row opens the TraceDrawer: getRaw by event id, raw line shown, Close clears', async () => { + mockGetEvents.mockResolvedValue({ events: [eventRow()] }) + mockGetRaw.mockResolvedValue(rawTrace) + render() + fireEvent.click(await screen.findByText('fp-acme-fw')) + + expect(mockGetRaw).toHaveBeenCalledWith( + '3f2bd0a6-1b1e-4c5a-9d3e-a1b2c3d4e5f6', + expect.anything(), // the AbortSignal the drawer cancels on close + ) + expect( + screen.getByRole('heading', { name: 'Traceability' }), + ).not.toBeNull() + // the raw line and the normalized document both render + expect( + await screen.findByText( + 'Sep 21 18:04:05 fw1 kernel: DROP tcp 10.0.0.1:445 -> 10.0.0.2:445', + ), + ).not.toBeNull() + expect(screen.getByText(/"src_endpoint"/)).not.toBeNull() + + fireEvent.click(screen.getByRole('button', { name: 'Close (Esc)' })) + expect( + screen.queryByRole('heading', { name: 'Traceability' }), + ).toBeNull() + }) + + it('Enter on a row also opens the drawer (keyboard activation)', async () => { + mockGetEvents.mockResolvedValue({ events: [eventRow()] }) + mockGetRaw.mockResolvedValue(rawTrace) + render() + fireEvent.keyDown(await screen.findByText('fp-acme-fw'), { key: 'Enter' }) + + expect( + await screen.findByRole('heading', { name: 'Traceability' }), + ).not.toBeNull() + }) + + it('shows the empty state when no events match', async () => { + mockGetEvents.mockResolvedValue({ events: [] }) + render() + await screen.findByText(/no events match/i) + }) + + it('shows an error banner when the load fails, and Retry recovers', async () => { + mockGetEvents.mockRejectedValueOnce( + new Error('GET /api/events failed: 500 Internal Server Error'), + ) + render() + await screen.findByRole('alert') + expect(screen.getByRole('alert').textContent).toMatch(/500/) + + mockGetEvents.mockResolvedValueOnce({ events: [eventRow()] }) + fireEvent.click(screen.getByRole('button', { name: 'Retry' })) + expect(await screen.findByText('fp-acme-fw')).not.toBeNull() + expect(screen.queryByRole('alert')).toBeNull() + }) +}) diff --git a/web/src/Events.tsx b/web/src/Events.tsx new file mode 100644 index 0000000..f0bd4be --- /dev/null +++ b/web/src/Events.tsx @@ -0,0 +1,161 @@ +// web/src/Events.tsx — the Events browser (M3 Task 9, the fourth §12 page): +// a filterable current-view table (GET /api/events) with the raw↔normalized +// drill-down drawer shared with Overview (components/TraceDrawer). +// +// Filters refetch live: status / fingerprint / limit are useEffect deps, so +// every change re-fires the fetch (empty-string status and fingerprint are +// forwarded as undefined — no filter). The Reload button and the error +// banner's Retry bump the same reloadTick the M2 pages use. Rows reuse the +// Overview interaction pattern: click or Enter opens the drawer, keyed by +// event_id so switching rows remounts it. +// +// Air-gap rules: same-origin /api only, system fonts, zero external requests. +import { useCallback, useEffect, useState } from 'react' +import { getEvents } from './api' +import type { EventRow } from './types' +import { endpointIp, formatTime } from './cells' +import TraceDrawer from './components/TraceDrawer' + +/** The Events tab: filter bar + current-view table + traceability drawer. */ +export default function Events() { + const [status, setStatus] = useState('') + const [fingerprint, setFingerprint] = useState('') + const [limit, setLimit] = useState(50) + const [events, setEvents] = useState(null) + const [error, setError] = useState(null) + const [reloadTick, setReloadTick] = useState(0) + const [selected, setSelected] = useState(null) + + useEffect(() => { + let alive = true + getEvents({ + status: status || undefined, + fingerprint: fingerprint || undefined, + limit, + }) + .then((res) => { + if (!alive) return + setEvents(res.events) + setError(null) + }) + .catch((err: unknown) => { + if (!alive) return + setError(err instanceof Error ? err.message : String(err)) + }) + return () => { + alive = false + } + }, [status, fingerprint, limit, reloadTick]) + + const refresh = useCallback(() => setReloadTick((n) => n + 1), []) + + return ( +
+
+

Events

+ + {events === null ? '…' : events.length} event(s) in the current view + +
+ +
+ + + + +
+ + {error !== null && ( +

+ events unavailable: {error}{' '} + +

+ )} + + {events === null && error === null && ( +

Loading events…

+ )} + + {events !== null && events.length === 0 && ( +

No events match the current filters.

+ )} + + {events !== null && events.length > 0 && ( +
+ + + + + + + + + + + + {events.map((row) => ( + setSelected(row)} tabIndex={0} + onKeyDown={(ev) => { if (ev.key === 'Enter') setSelected(row) }}> + + + + + + + ))} + +
TimeFingerprintStatusRule vSource → Destination
{formatTime(row.parsed_at)}{row.fingerprint_id} + {row.status} + {row.rule_version ?? '—'} + {endpointIp(row.ocsf?.src_endpoint ?? null)} →{' '} + {endpointIp(row.ocsf?.dst_endpoint ?? null)} +
+
+ )} + + {selected !== null && ( + setSelected(null)} + /> + )} +
+ ) +} diff --git a/web/src/ReviewQueue.test.tsx b/web/src/ReviewQueue.test.tsx index 0b13ffb..027c109 100644 --- a/web/src/ReviewQueue.test.tsx +++ b/web/src/ReviewQueue.test.tsx @@ -81,6 +81,7 @@ function pendingRule(overrides: Partial = {}): RuleRow { provenance: 'slm', confidence: 0.87, status: 'pending_review', + quarantined_fields: [], // P-8 additive key — empty on this fixture created_by: 'onboarding', created_at: '2026-09-19T09:05:00Z', activated_at: null, diff --git a/web/src/Rules.test.tsx b/web/src/Rules.test.tsx index 9731af0..f26e8fb 100644 --- a/web/src/Rules.test.tsx +++ b/web/src/Rules.test.tsx @@ -33,6 +33,7 @@ function ruleRow(overrides: Partial = {}): RuleRow { provenance: 'slm', confidence: 0.87, status: 'active', + quarantined_fields: [], // P-8 additive key — empty on this fixture created_by: 'onboarding', created_at: '2026-09-18T09:00:00Z', activated_at: '2026-09-19T10:00:00Z', diff --git a/web/src/api.ts b/web/src/api.ts index e1e9bf0..797eb36 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -5,9 +5,14 @@ // validation failures) so the Review Queue can render the gate's verdict. // // The Overview page is fed by the SSE stream; getRules/getSamples serve the -// M2 review surfaces (Tasks 8/9), mirroring the gateway endpoint shapes. +// M2 review surfaces (Tasks 8/9) and getDriftMetrics/getDriftAlerts the M3 +// Drift & Health page (Task 10), mirroring the gateway endpoint shapes. The +// un-quarantine POST rides the exported postJson directly (Rules-deactivate +// style) rather than a dedicated wrapper. import type { AuditResponse, + DriftAlertsResponse, + DriftMetricsResponse, EventsResponse, RawTrace, RuleHistoryResponse, @@ -128,3 +133,18 @@ export function getSamples( `/api/onboarding/samples${qs}`, ) } + +// --- M3 drift surfaces (Task 7 endpoints; consumed by Task 10) --------------- + +/** GET /api/drift/metrics — the latest closed window per (fingerprint_id, + * field); the page's Field-health table renders rows verbatim. */ +export function getDriftMetrics(): Promise { + return getJson('/api/drift/metrics') +} + +/** GET /api/drift/alerts — ONE latest-first merged list (window rows ∪ audit + * rows, discriminated by `kind`) that the page renders verbatim without + * re-merging or re-sorting (ruling P-4). */ +export function getDriftAlerts(): Promise { + return getJson('/api/drift/alerts') +} diff --git a/web/src/cells.ts b/web/src/cells.ts new file mode 100644 index 0000000..49a159e --- /dev/null +++ b/web/src/cells.ts @@ -0,0 +1,12 @@ +// web/src/cells.ts — tiny pure cell formatters shared by more than one page +// (Overview in App.tsx, the Events browser). Kept here rather than copy-pasted +// per page so the fifth §12 page can reuse them too. +import type { OcsfEndpoint } from './types' + +export function endpointIp(ep: OcsfEndpoint | null): string { + return ep?.ip ?? '—' +} + +export function formatTime(iso: string): string { + return new Date(iso).toLocaleTimeString() +} diff --git a/web/src/components/TraceDrawer.tsx b/web/src/components/TraceDrawer.tsx new file mode 100644 index 0000000..6a7d4fe --- /dev/null +++ b/web/src/components/TraceDrawer.tsx @@ -0,0 +1,89 @@ +// web/src/components/TraceDrawer.tsx — the shared raw↔normalized traceability +// drawer, extracted VERBATIM from App.tsx (M3 Task 9) so the Overview feed +// and the Events browser both drill into the raw line behind an event. +// Behavior, props and markup are unchanged — only the location moved. +import { useEffect, useState } from 'react' +import { getRaw } from '../api' +import type { EventRow, RawTrace } from '../types' + +/** Side drawer: full OCSF document plus the raw line behind the event + * (fetched by event_id; the fetch is aborted via AbortController if the + * drawer closes first). */ +function TraceDrawer({ row, onClose }: { row: EventRow; onClose: () => void }) { + const [trace, setTrace] = useState(null) + const [error, setError] = useState(null) + + useEffect(() => { + const controller = new AbortController() + getRaw(row.event_id, controller.signal) + .then((t) => setTrace(t)) + .catch((err: unknown) => { + if (!controller.signal.aborted) { + setError(err instanceof Error ? err.message : String(err)) + } + }) + return () => controller.abort() + }, [row.event_id]) + + useEffect(() => { + const onKey = (ev: KeyboardEvent) => { + if (ev.key === 'Escape') onClose() + } + window.addEventListener('keydown', onKey) + return () => window.removeEventListener('keydown', onKey) + }, [onClose]) + + return ( + + ) +} + +export default TraceDrawer diff --git a/web/src/types.ts b/web/src/types.ts index c75ccbd..b999608 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -114,9 +114,12 @@ export type RuleStatus = /** rules.provenance CHECK vocabulary. "slm-edited" marks approve-with-edits. */ export type RuleProvenance = 'slm' | 'slm-edited' | 'human' -/** GET /api/rules row — EXACTLY these 13 keys (gateway _RULE_COLUMNS). +/** GET /api/rules row — EXACTLY these 14 keys (gateway _RULE_COLUMNS). * `id` doubles as the candidate id in the approve/reject URLs; UUID-ish ids - * are strings, timestamps ISO-8601 strings. */ + * are strings, timestamps ISO-8601 strings. `quarantined_fields` mirrors + * rules.quarantined_fields TEXT[] (migration 001; additive P-8) — the fields + * drift enforcement disabled on this version; the Drift & Health page gates + * un-quarantine on the ACTIVE version's list. */ export interface RuleRow { id: number fingerprint_id: string @@ -126,6 +129,7 @@ export interface RuleRow { provenance: RuleProvenance confidence: number | null status: RuleStatus + quarantined_fields: string[] created_by: string created_at: string activated_at: string | null @@ -207,3 +211,72 @@ export interface SamplesStatus { export interface SamplesListResponse { samples: SamplesStatus[] } + +// --- M3 drift surfaces (Task 7 shapes, mirrored key for key; Task 10) --------- + +/** drift_windows.severity CHECK vocabulary (migration 007 / SEVERITIES). */ +export type DriftSeverity = 'none' | 'minor' | 'moderate' | 'severe' + +/** drift_windows.action_taken CHECK vocabulary; null when the window's + * severity warranted no action ("none", or an already-quarantined field — + * the idempotent skip records no action). */ +export type DriftAction = 'alert' | 'field_quarantined' | 'rule_deactivated' + +/** One drift_windows row — GET /api/drift/metrics rows carry EXACTLY these + * 12 keys (gateway _DRIFT_WINDOW_COLUMNS): the latest closed window per + * (fingerprint_id, field). `field` is an OCSF path, 'unmapped.', or + * the '__rule__' sentinel; rates are 0..1 REALs, null where the window does + * not measure that rate for the field (match_rate is __rule__-only; + * violation_rate is null on unmapped rows); shape_dist is the JSONB + * shape-class histogram over non-null scalar values ({} when the field + * observed no values, null on __rule__ rows); timestamps ISO-8601 strings. */ +export interface DriftMetricRow { + fingerprint_id: string + rule_version: number + field: string + window_start: string + window_end: string + events_count: number + null_rate: number | null + match_rate: number | null + violation_rate: number | null + shape_dist: Record | null + severity: DriftSeverity + action_taken: DriftAction | null +} + +/** GET /api/drift/metrics response. */ +export interface DriftMetricsResponse { + metrics: DriftMetricRow[] +} + +/** One GET /api/drift/alerts row — ruling P-4's kind-discriminated union: + * ONE latest-first merged list the endpoint already ordered (the page + * renders it verbatim and never re-merges or re-sorts client-side). + * "window" rows carry the drift_windows columns (minor/moderate/severe — + * the row IS the alert); "audit" rows carry the audit_log shape with action + * field_quarantined / field_unquarantined / rule_deactivated (any actor). */ +export type DriftAlertRow = + | (DriftMetricRow & { kind: 'window' }) + | (AuditRow & { kind: 'audit' }) + +/** GET /api/drift/alerts response. */ +export interface DriftAlertsResponse { + alerts: DriftAlertRow[] +} + +/** POST /api/rules/{fp}/unquarantine body — the human override that removes + * one field from the active rule's quarantined_fields (audited, paired in + * the same transaction as the mutation). */ +export interface UnquarantineBody { + field: string + actor?: string + reason?: string +} + +/** POST /api/rules/{fp}/unquarantine 200 response — the post-state list. */ +export interface UnquarantineResponse { + fingerprint_id: string + field: string + quarantined_fields: string[] +}