diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1f788e..10932e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -# M1 CI (Task 12). Unit tests only — no container services; the live E2E gate +# M1+M2 CI. Unit tests only — no container services; the live E2E gate # is `python scripts/smoke.py`, run locally against the compose stack. name: ci @@ -18,8 +18,9 @@ jobs: pip install -r services/collector/requirements.txt \ -r services/pipeline/requirements.txt \ -r services/gateway/requirements.txt \ + -r services/onboarding/requirements.txt \ -r simulator/requirements.txt - pip install pytest ruff==0.16.8 + pip install pytest~=8.4 ruff==0.16.8 - name: Lint run: ruff check . - name: Unit tests (libs + services + simulator) @@ -36,7 +37,7 @@ jobs: with: node-version: 22 - name: Install - run: npm ci || npm i + run: npm ci - name: Test run: npm run test --if-present - name: Build diff --git a/.gitignore b/.gitignore index ae12b44..3a5c76e 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,6 @@ ENV/ # Env / secrets .env -deploy/.env *.env.local # OS / editor cruft @@ -42,14 +41,10 @@ Thumbs.db .ollama #zip files *.zip -testdata/registered_rules.json #references references/ # MVP build artifacts node_modules/ web/dist/ -.venv/ -*.egg-info/ -__pycache__/ -.pytest_cache/ \ No newline at end of file +.pytest_cache/ diff --git a/README.md b/README.md index 6109919..35fd90b 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,77 @@ # ULPF — Universal Log Pre-processing Framework -SIH 2026 PS 26156 MVP. Status: M1 walking skeleton complete — the milestone -exit is `python scripts/smoke.py` (all 5 checks PASS). +SIH 2026 PS 26156 MVP. Status: M2 onboarding loop complete — the milestone +exit is `python scripts/smoke.py` (all 8 checks PASS). -- `services/` — collector, pipeline, gateway (+ onboarding, drift in M2/M3) +- `services/` — collector, pipeline, gateway, onboarding (drift in M3) - `libs/ulpf-core` — fingerprinting, parsing, IDs, hash chain (single implementation) - `libs/ocsf-schema` — curated OCSF subset schema + validator - `web/` — React dashboard - `simulator/` — multi-vendor traffic player - `deploy/` — docker-compose, migrations -- `scripts/smoke.py` — M1 acceptance gate (live E2E incl. forced replay) +- `scripts/smoke.py` — M1+M2 acceptance gate (live E2E incl. forced replay + the onboarding hero loop) - `legacy-demo/` — the original hackathon demo, frozen. See its README. -## Quickstart (M1) +## Collector ingest + +Syslog UDP + TCP on 5514, HTTP `POST /v1/ingest` on 8080 (≤1000 lines/request). +Oversized input is handled asymmetrically by design: UDP drops just the one +oversized datagram (each datagram is an independent event), while TCP closes +the connection — a byte stream has no line boundary to resynchronize to. + +## Quickstart + +```bash +cd deploy && docker compose --profile slm-4b up -d --build +``` + +Brings up the M1 pipeline plus the M2 onboarding loop and the default SLM +tier: `qwen3:4b` behind the `ollama-4b` sidecar (runs on 8GB boxes; the model +is pulled by the sidecar on first start — a one-time multi-GB download). +16GB boxes may upgrade: `--profile slm-8b` plus `ULPF_OLLAMA_URL=http://ollama-8b:11434` +and `ULPF_OLLAMA_MODEL=qwen3:8b` in `deploy/.env` (see `.env.example`). The +ollama sidecars publish **no host port** — they are reachable only inside the +compose network (the onboarding service is their only client). + +Hero loop (unknown appliance → candidate rule): play the unknown `newapp` +corpus at the collector, then watch the Review Queue while onboarding splits +samples, asks the SLM, validates, and stores the candidate. On the default +`qwen3:4b` tier the SLM may store no rule on modest hardware — in that case +the fingerprint stays backlogged until a candidate lands (author one via the +dashboard or `POST /api/rules/manual`); see docs/m2-slm-sanity.md for the +measured tier numbers. The full lifecycle — grants, version minting, backlog +re-parse — is recorded in docs/m2-rule-lifecycle.md: + +```bash +docker compose --profile sim run --rm simulator --eps 25 --duration 20 --mode new-appliance +open http://localhost:3000 # Review Queue: the newapp fingerprint lands as pending_review when a candidate stores +# sustained live traffic for the dashboard (`run` needs the sim profile re-activated): +docker compose --profile sim run --rm simulator --eps 20 --duration 300 --loop +``` + +Acceptance gate (starts the stack itself — bring the SLM tier profile so the +onboarding sidecar is up): ```bash -cd deploy && docker compose --profile sim up -d --build -python scripts/smoke.py # end-to-end acceptance incl. forced replay -open http://localhost:3000 # Overview: live feed + raw traceability -# sustained live traffic for the dashboard (the --profile sim one-shot above -# exits after its single ~56-line pass): -docker compose run --rm simulator --eps 20 --duration 300 --loop +python scripts/smoke.py # end-to-end acceptance incl. forced replay + hero loop ``` -The smoke runs the simulator over the golden corpora, then verifies: lossless -ingest (raw delta == sent), the parsed/unparsed split per fingerprint, raw -traceability (API + SQL join with recomputed sha256), forced replay -idempotency (Kafka consumer-group delete + pipeline restart: raw events -unchanged, raw_batches grown), and an independent recomputation of every -raw_batches merkle root plus full per-partition chain linkage. +The smoke runs the simulator over the golden corpora, then verifies: +lossless ingest (raw delta == sent), the parsed/unparsed split per +fingerprint, raw traceability (API + SQL join with recomputed sha256), +forced replay idempotency (Kafka consumer-group delete + pipeline restart: +raw events unchanged, raw_batches grown), an independent recomputation of +every raw_batches merkle root plus full per-partition chain linkage (A–E), +then the M2 onboarding loop: the newapp hero loop through approve → backlog +re-parse (SLM candidate if one stores within `ULPF_SMOKE_SLM_WAIT_S` +(default 300s), a locally pre-validated hand-authored candidate otherwise), +the deterministic zenwall manual rule (G), and a replay-stability check with +active rules, samples included (H). -Smoke host deps (Python 3.13 tested): `psycopg[binary]` only. No confluent-kafka +Smoke host deps (Python 3.13 tested): `psycopg[binary]` plus the two libs +importable — `pip install -e libs/ulpf-core -e libs/ocsf-schema` (the smoke +computes fingerprints and pre-validates hand-authored rules with the same +`ulpf_core` gate the services run). No confluent-kafka on the host: Kafka interaction goes through `docker compose exec redpanda rpk`, and the simulator runs in its own container (the services' `confluent-kafka==2.5.0` pin is in-image only, py3.12). The smoke is diff --git a/bench/fingerprint_fragmentation.py b/bench/fingerprint_fragmentation.py index 57d4ce1..6768dee 100755 --- a/bench/fingerprint_fragmentation.py +++ b/bench/fingerprint_fragmentation.py @@ -24,12 +24,11 @@ from __future__ import annotations import random -import re import sys -from collections.abc import Callable from pathlib import Path -from ulpf_core.fingerprint import _classify, fingerprint_id, shape_hash +from ulpf_core.fingerprint import fingerprint_id, shape_hash +from ulpf_core.mutations import mutate_line ROOT = Path(__file__).resolve().parents[1] GOLDEN = ROOT / "libs" / "ulpf-core" / "tests" / "data" / "golden" @@ -38,98 +37,10 @@ SEED = 20260919 MUTATIONS = 100 -# --- mutation machinery ------------------------------------------------------ -# Timestamps mutate first and park behind \x00 sentinels so later numeric -# passes cannot chew freshly generated values into shape-breaking forms. -# IP:port mutates before the IP pass so the ":port" tail does not lose the -# digit its lookbehind needs. - -_ISO_TS_RE = re.compile( - r"(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2}):(\d{2}))?([+-]\d{2}:?\d{2}|Z)?" -) -_SYSLOG_TS_RE = re.compile( - r"\b((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec))\s+(\d{1,2})\s+(\d{2}):(\d{2}):(\d{2})\b" -) -_PORT_RE = re.compile(r"(?<=\d):\d{1,5}\b") # port tail of IP:port -_IP_RE = re.compile(r"\d{1,3}(?:\.\d{1,3}){3}") -_KV_NUM_RE = re.compile(r"(?<==)\d{1,5}\b") # key=12345 values -_JSON_NUM_RE = re.compile(r'(?<=":)\d{1,5}\b') # "key":12345 values -_HOSTNAME_RE = re.compile(r"\b[a-z][a-z0-9-]*\d{1,4}\b") # fw01, vpn-gw01, eth0 -_SENTINEL_RE = re.compile(r"\x00A(\d+)\x00") -_MONTH_RE = re.compile(r"^(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)$") -_MARKUP_CHARS = set('{}"<>') - - -def _rand_iso_ts(match: re.Match[str], rng: random.Random) -> str: - text = f"{rng.randint(2020, 2030):04d}-{rng.randint(1, 12):02d}-{rng.randint(1, 28):02d}" - if match.group(4): - text += f"T{rng.randint(0, 23):02d}:{rng.randint(0, 59):02d}:{rng.randint(0, 59):02d}" - return text + (match.group(7) or "") - - -def _rand_syslog_ts(match: re.Match[str], rng: random.Random) -> str: - # Keep the month word so the mutant still detects as syslog. - return ( - f"{match.group(1)} {rng.randint(1, 28)} " - f"{rng.randint(0, 23):02d}:{rng.randint(0, 59):02d}:{rng.randint(0, 59):02d}" - ) - - -def _rand_ip(rng: random.Random) -> str: - return ".".join(str(rng.randint(1, 254)) for _ in range(4)) - - -def mutate_line(line: str, rng: random.Random) -> str: - """Return a same-format mutant of *line* with randomized values.""" - stash: list[str] = [] - - def _swap(make: Callable[[re.Match[str]], str]) -> Callable[[re.Match[str]], str]: - def _sub(match: re.Match[str]) -> str: - stash.append(make(match)) - return f"\x00A{len(stash) - 1}\x00" - - return _sub - - line = _ISO_TS_RE.sub(_swap(lambda m: _rand_iso_ts(m, rng)), line) - line = _SYSLOG_TS_RE.sub(_swap(lambda m: _rand_syslog_ts(m, rng)), line) - line = _PORT_RE.sub(_swap(lambda m: f":{rng.randint(1, 65535)}"), line) - line = _IP_RE.sub(_swap(lambda m: _rand_ip(rng)), line) - line = _KV_NUM_RE.sub(_swap(lambda m: str(rng.randint(1, 65535))), line) - line = _JSON_NUM_RE.sub(_swap(lambda m: str(rng.randint(1, 65535))), line) - line = _HOSTNAME_RE.sub(_swap(lambda m: f"h{rng.randint(0, 9999):04d}"), line) - - # Free-text runs (message bodies) get a random word count. Structural - # tokens (markup, sentinels, syslog month, anything classified) are kept. - if " " in line: - kept: list[str] = [] - run: list[str] = [] - - def _flush() -> None: - if run: - kept.extend(rng.choice(WORDS) for _ in range(rng.randint(1, 6))) - run.clear() - - for token in line.split(): - if ( - _SENTINEL_RE.fullmatch(token) - or _MONTH_RE.fullmatch(token) - or _MARKUP_CHARS.intersection(token) - or _classify(token) != "W" - ): - _flush() - kept.append(token) - else: - run.append(token) - _flush() - line = " ".join(kept) - - return _SENTINEL_RE.sub(lambda m: stash[int(m.group(1))], line) - - -WORDS = [ - "alpha", "bravo", "delta", "echo", "fox", "golf", - "hotel", "india", "juliet", "kilo", "lima", "mike", -] +# Mutation machinery (regexes, rand helpers, WORDS, mutate_line) lives in +# ulpf_core.mutations since M2 — single definition shared with the candidate +# probe. The bench passes vary_text=True to keep its historical behavior +# (free-text message word counts are mutated too). def run_bench(mutations: int = MUTATIONS, seed: int = SEED) -> dict: @@ -164,7 +75,7 @@ def run_bench(mutations: int = MUTATIONS, seed: int = SEED) -> dict: for _ in range(mutations): base = rng.choice(lines) fp = fingerprint_id(base) - mutant = mutate_line(base, rng) + mutant = mutate_line(base, rng, vary_text=True) mutant_hashes.add(shape_hash(mutant)) ok = shape_hash(mutant) == canonical[fp] if not fp.startswith("auto_"): diff --git a/bench/onboarding_eval/sanity_eval.py b/bench/onboarding_eval/sanity_eval.py new file mode 100755 index 0000000..ddab764 --- /dev/null +++ b/bench/onboarding_eval/sanity_eval.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +"""sanity_eval.py — offline SLM sanity numbers for the onboarding candidate loop. + +For each corpus (acmegw, zenwall, newapp): split the lines 5/15 via +onboarding's assign_roles (the exact production logic), then --runs times +per corpus run generate_candidate + validate_candidate against a LIVE Ollama +endpoint (the same OllamaClient the onboarding service uses). Tally per +corpus: valid-JSON rate, compiles rate, held-out match rate, mean attempts, +wall-clock per rule; print a markdown table and optionally --write the +docs/m2-slm-sanity.md page (measured numbers only — model tag, host CPU, +date, run count in the header). + +Metric definitions (also emitted into the written doc): + valid_json — SLM responses from which extract_json recovered an object + carrying both "pattern" and "field_mappings" / all responses. + compiles — of those, patterns that re.compile() (the __JSON__ sentinel + counts as compiling). + held_out — mean validate_candidate held_out_match_rate over the runs + that produced a rule (a run whose retries were exhausted + contributes no rate and is counted in failures). + attempts — mean chat() calls per run (1.0 = first response usable). + s/rule — total run wall-clock (generation + validation, failed runs + included) divided by rules stored. + +Stdlib + onboarding imports only at module load; `ollama` stays lazy inside +OllamaClient.chat, so this imports (and the unit tests run) without it. No +DB, no pytest against the live endpoint — a manual bench tool. + +Run from the repo root: + python bench/onboarding_eval/sanity_eval.py --runs 3 \ + --model qwen3:4b --url http://ollama-4b:11434 --write docs/m2-slm-sanity.md +""" +from __future__ import annotations + +import argparse +import json +import platform +import re +import sys +import threading +import time +from datetime import datetime +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +# The onboarding package lives under services/ (pytest does the same insertion +# for the unit tests; the bench script must also work standalone). +sys.path.insert(0, str(REPO_ROOT / "services")) + +from onboarding.generate import GenerationError, extract_json, generate_candidate +from onboarding.samples import assign_roles +from ulpf_core.parsing import JSON_SENTINEL +from ulpf_core.validation import validate_candidate + +CORPUS_NAMES = ("acmegw", "zenwall", "newapp") +HEADERS = ["corpus", "runs", "rules", "valid_json", "compiles", + "held_out_match", "mean_attempts", "s/rule"] + + +# ---------- formatting (pure; unit-tested) ---------- + +def pct(value: float | None) -> str: + return "n/a" if value is None else f"{value * 100:.1f}%" + + +def num(value: float | None) -> str: + return "n/a" if value is None else f"{value:.1f}" + + +def format_table(headers: list[str], rows: list[list[str]]) -> str: + """Markdown table with columns padded to the widest cell per column.""" + widths = [len(h) for h in headers] + for row in rows: + for i, cell in enumerate(row): + widths[i] = max(widths[i], len(cell)) + + def line(cells: list[str]) -> str: + return "| " + " | ".join(c.ljust(widths[i]) for i, c in enumerate(cells)) + " |" + + out = [line(headers), "| " + " | ".join("-" * w for w in widths) + " |"] + out.extend(line(row) for row in rows) + return "\n".join(out) + + +# ---------- corpus resolution ---------- + +def find_corpora(names: list[str], dirs: list[Path]) -> dict[str, Path | None]: + """First raw_logs_.txt found across `dirs` (in order); None if a + corpus is missing everywhere — reported, not fatal.""" + found: dict[str, Path | None] = {} + for name in names: + path = next((d / f"raw_logs_{name}.txt" for d in dirs + if (d / f"raw_logs_{name}.txt").is_file()), None) + found[name] = path + return found + + +def load_lines(path: Path) -> list[str]: + return [line for line in path.read_text(encoding="utf-8").splitlines() + if line.strip()] + + +# ---------- the harness ---------- + +class RecordingClient: + """Wraps an SLMClient; records every response + its chat latency so the + tally sees per-attempt truth (generate_candidate only surfaces the rule).""" + + def __init__(self, inner): + self.inner = inner + self.calls: list[tuple[str, float]] = [] + + def chat(self, prompt: str, temperature: float) -> str: + start = time.monotonic() + text = self.inner.chat(prompt, temperature) + self.calls.append((text, time.monotonic() - start)) + return text + + +class ChatTimeout(Exception): + """A single chat() call exceeded the --chat-timeout bound. Rides the + production retry ladder like any transport failure, so a degenerate + answer becomes honest failed-run rows instead of an overnight hang.""" + + +class TimeoutClient: + """Opt-in per-chat wall-clock bound (the "no overnight failure loops" + mechanism). Runs the inner chat on a daemon thread: on expiry the eval + proceeds immediately and the abandoned call can never block interpreter + exit. Default (no bound) stays production-faithful.""" + + def __init__(self, inner, timeout_s: float): + self.inner = inner + self.timeout_s = timeout_s + + def chat(self, prompt: str, temperature: float) -> str: + outcome: dict = {} + + def run(): + try: + outcome["text"] = self.inner.chat(prompt, temperature) + except Exception as exc: # noqa: BLE001 - relayed verbatim below + outcome["exc"] = exc + + worker = threading.Thread(target=run, daemon=True) + worker.start() + worker.join(self.timeout_s) + if worker.is_alive(): + raise ChatTimeout(f"no response within {self.timeout_s:.0f}s") + if "exc" in outcome: + raise outcome["exc"] + return outcome["text"] + + +def _response_shape_ok(text: str) -> bool: + """valid_json metric: extract_json recovered an object with the two + required keys (deeper breakage — bad mapping shapes — counts as valid + JSON but is caught downstream by generate_candidate's retries).""" + try: + parsed = extract_json(text) + except (ValueError, json.JSONDecodeError): + return False + return isinstance(parsed, dict) and "pattern" in parsed and "field_mappings" in parsed + + +def _pattern_compiles(pattern: str) -> bool: + if pattern == JSON_SENTINEL: + return True + try: + re.compile(pattern) + return True + except re.error: + return False + + +def evaluate_corpus(corpus: str, lines: list[str], *, client_factory, + runs: int, prompt_size: int = 5, held_out_size: int = 15) -> dict: + """One corpus through the production path, `runs` times (fresh SLM calls + each run; the deterministic split is derived once).""" + rows = [{"id": i, "raw_text": line} for i, line in enumerate(lines)] + split = assign_roles(rows, prompt_size, held_out_size) + prompt_lines = [row["raw_text"] for row in split["prompt"]] + held_out_lines = [row["raw_text"] for row in split["held_out"]] + + responses = valid_json = compiles = 0 + rules_stored = 0 + attempt_counts: list[int] = [] + match_rates: list[float] = [] + wall = 0.0 + failures: list[str] = [] + + for _ in range(runs): + client = RecordingClient(client_factory()) + start = time.monotonic() + try: + rule = generate_candidate("sanity-eval", prompt_lines, client) + except GenerationError as exc: + wall += time.monotonic() - start # failed run: generation only + failures.append(f"{len(client.calls)} attempts: {exc}") + else: + report = validate_candidate(rule, prompt_lines, held_out_lines) + # Window stays open through validation: s/rule is documented as + # "generation + validation, failed runs included". + wall += time.monotonic() - start + rules_stored += 1 + match_rates.append(report.held_out_match_rate) + + attempt_counts.append(len(client.calls)) + for text, _ in client.calls: + responses += 1 + if _response_shape_ok(text): + valid_json += 1 + if _pattern_compiles(extract_json(text)["pattern"]): + compiles += 1 + + return { + "corpus": corpus, + "runs": runs, + "rules_stored": rules_stored, + "responses": responses, + "valid_json": valid_json, + "compiles": compiles, + "held_out_match_rate": (sum(match_rates) / len(match_rates) + if match_rates else None), + "mean_attempts": (sum(attempt_counts) / len(attempt_counts) + if attempt_counts else None), + "sec_per_rule": wall / rules_stored if rules_stored else None, + "prompt_n": len(prompt_lines), + "held_out_n": len(held_out_lines), + "failures": failures, + } + + +def _row(result: dict) -> list[str]: + responses = result["responses"] or None + valid_json = result["valid_json"] or None + return [ + result["corpus"], + str(result["runs"]), + str(result["rules_stored"]), + pct(result["valid_json"] / responses if responses else None), + # Documented denominator: "of those" — the valid-JSON responses. + pct(result["compiles"] / valid_json if valid_json else None), + pct(result["held_out_match_rate"]), + num(result["mean_attempts"]), + num(result["sec_per_rule"]), + ] + + +def build_doc(results: list[dict], *, model: str, host_cpu: str, + date_str: str, runs: int, chat_timeout: float | None = None) -> str: + """docs/m2-slm-sanity.md body: measured header notes, metric definitions, + the numbers table, and per-corpus failure footnotes (only when a run's + retries were exhausted).""" + lines = [ + "# M2 SLM sanity numbers — onboarding candidate loop", + "", + "Measured numbers only (bench/onboarding_eval/sanity_eval.py against a", + "live Ollama endpoint; the split + generation + validation are the", + "production code paths).", + "", + f"- model: {model}", + f"- host CPU: {host_cpu}", + f"- date: {date_str}", + f"- runs per corpus: {runs}", + ] + if chat_timeout is not None: + lines.append( + f"- per-chat timeout: {int(chat_timeout)}s (eval-side bound; the " + "production loop is unbounded)") + lines += [ + "", + "Metric definitions: `valid_json` = SLM responses from which", + "`extract_json` recovered an object with `pattern` + `field_mappings`;", + "`compiles` = of those, patterns that compile (`__JSON__` sentinel counts);", + "`held_out_match` = mean `validate_candidate` held-out match rate over runs", + "that produced a rule; `mean_attempts` = mean chat() calls per run;", + "`s/rule` = run wall-clock per stored rule (failed runs included in the", + "numerator). Split: 5 prompt / up to 15 held-out per corpus.", + "", + format_table(HEADERS, [_row(r) for r in results]), + ] + for result in results: + if result.get("failures"): + lines += ["", f"{result['corpus']} failures:"] + lines += [f"- {failure}" for failure in result["failures"]] + lines.append("") + return "\n".join(lines) + + +# ---------- CLI ---------- + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="sanity_eval.py", + description="Offline SLM sanity numbers for the onboarding candidate loop.") + parser.add_argument("--corpora", action="append", default=None, metavar="DIR", + help="directory holding raw_logs_*.txt (repeatable; default: " + "the repo's golden + simulator/data locations)") + parser.add_argument("--runs", type=int, default=3, + help="generation runs per corpus (default 3)") + parser.add_argument("--model", default="qwen3:4b", + help="SLM tag (default qwen3:4b)") + parser.add_argument("--url", default="http://localhost:11434", + help="Ollama base URL (default http://localhost:11434; the " + "compose sidecar is http://ollama-4b:11434 from the " + "compose network)") + parser.add_argument("--write", default=None, metavar="PATH", + help="also write the full markdown doc (e.g. docs/m2-slm-sanity.md)") + parser.add_argument("--host-cpu", default=None, dest="host_cpu", + help="host CPU description for the doc header " + "(default: platform.processor())") + parser.add_argument("--chat-timeout", type=float, default=None, + dest="chat_timeout", metavar="SECONDS", + help="bound each chat() call (eval-side; the production " + "loop is unbounded). A timed-out call retries like " + "any transport failure, so a degenerate answer " + "becomes honest failed-run rows instead of an " + "overnight hang.") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + dirs = ([Path(d) for d in args.corpora] if args.corpora else + [REPO_ROOT / "libs/ulpf-core/tests/data/golden", REPO_ROOT / "simulator/data"]) + found = find_corpora(list(CORPUS_NAMES), dirs) + + missing = [name for name, path in found.items() if path is None] + for name in missing: + print(f"warning: raw_logs_{name}.txt not found in " + f"{[str(d) for d in dirs]}; skipping", file=sys.stderr) + if not missing: + print(f"corpora: {', '.join(f'{n}={found[n]}' for n in CORPUS_NAMES)}", + file=sys.stderr) + + from onboarding.generate import OllamaClient # deferred: SLM tier only + + def client_factory(): + client = OllamaClient(args.url, args.model) + if args.chat_timeout is not None: + client = TimeoutClient(client, args.chat_timeout) + return client + + results = [] + for name in CORPUS_NAMES: + if found[name] is None: + continue + print(f"evaluating {name} ({found[name]}) ...", file=sys.stderr) + results.append(evaluate_corpus(name, load_lines(found[name]), + client_factory=client_factory, runs=args.runs)) + if not results: + print("error: no corpora evaluated", file=sys.stderr) + return 2 + + table = format_table(HEADERS, [_row(r) for r in results]) + print(table) + + if args.write: + doc = build_doc(results, model=args.model, + host_cpu=args.host_cpu or platform.processor() or "unknown", + date_str=datetime.now().astimezone().date().isoformat(), + runs=args.runs, chat_timeout=args.chat_timeout) + out = Path(args.write) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(doc, encoding="utf-8") + print(f"wrote {out}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/onboarding_eval/test_sanity_eval.py b/bench/onboarding_eval/test_sanity_eval.py new file mode 100644 index 0000000..3bf98d7 --- /dev/null +++ b/bench/onboarding_eval/test_sanity_eval.py @@ -0,0 +1,241 @@ +# bench/onboarding_eval/test_sanity_eval.py — unit tests for the sanity-eval's +# PURE layer: table formatting, rate cells, corpus-path resolution, doc header, +# and the per-corpus aggregation driven by a canned (offline) SLM client. The +# harness itself is a manual tool (no pytest run against a live Ollama). +import time + +import pytest +import sanity_eval +from onboarding.generate import GenerationError + +# ---------- format_table ---------- + +def test_format_table_aligns_columns(): + table = sanity_eval.format_table( + ["corpus", "runs", "held_out_match"], + [["acmegw", "3", "80.0%"], ["zenwall", "3", "100.0%"]], + ) + lines = table.splitlines() + assert lines[0] == "| corpus | runs | held_out_match |" + assert lines[1] == "| ------- | ---- | -------------- |" + assert lines[2] == "| acmegw | 3 | 80.0% |" + assert lines[3] == "| zenwall | 3 | 100.0% |" + + +def test_format_table_empty_rows_still_renders_header(): + assert sanity_eval.format_table(["a", "bb"], []) == "| a | bb |\n| - | -- |" + + +# ---------- cell helpers ---------- + +def test_pct(): + assert sanity_eval.pct(1.0) == "100.0%" + assert sanity_eval.pct(0.855) == "85.5%" + assert sanity_eval.pct(0.0) == "0.0%" + assert sanity_eval.pct(None) == "n/a" # no responses observed (all runs failed) + + +def test_num(): + assert sanity_eval.num(2.0) == "2.0" + assert sanity_eval.num(1.333333) == "1.3" + assert sanity_eval.num(None) == "n/a" + + +# ---------- row assembly (rate denominators per the documented metrics) ---------- + +def test_row_compiles_rate_denominator_is_valid_json(): + # Documented metric: compiles is "of those" — the valid-JSON responses — + # not of all responses. 1 compiles out of 1 valid JSON (3 total responses) + # is a 100% compiles rate, even though valid_json is 33.3%. + result = {"corpus": "fake", "runs": 1, "rules_stored": 1, "responses": 3, + "valid_json": 1, "compiles": 1, "held_out_match_rate": 1.0, + "mean_attempts": 3.0, "sec_per_rule": 1.5} + row = sanity_eval._row(result) + assert row[3] == "33.3%" # valid_json / responses + assert row[4] == "100.0%" # compiles / valid_json + + +def test_row_compiles_rate_na_when_no_valid_json(): + # Zero-division guard: with no valid-JSON responses the compiles rate has + # no denominator and renders like any other missing value. + result = {"corpus": "fake", "runs": 1, "rules_stored": 0, "responses": 2, + "valid_json": 0, "compiles": 0, "held_out_match_rate": None, + "mean_attempts": 2.0, "sec_per_rule": None} + row = sanity_eval._row(result) + assert row[3] == "0.0%" + assert row[4] == "n/a" + + +# ---------- corpus resolution ---------- + +def test_find_corpora_prefers_first_dir_with_the_file(tmp_path): + golden = tmp_path / "golden" + sim = tmp_path / "sim" + golden.mkdir() + sim.mkdir() + (golden / "raw_logs_acmegw.txt").write_text("x\n", encoding="utf-8") + (sim / "raw_logs_newapp.txt").write_text("y\n", encoding="utf-8") + + found = sanity_eval.find_corpora(["acmegw", "zenwall", "newapp"], [golden, sim]) + assert found["acmegw"] == golden / "raw_logs_acmegw.txt" + assert found["newapp"] == sim / "raw_logs_newapp.txt" + assert found["zenwall"] is None # missing corpus stays visible, not fatal + + +# ---------- aggregation over runs (offline fake SLM) ---------- + +class FakeClient: + """Canned SLMClient: pops one prepared response per chat() call.""" + + def __init__(self, *responses): + self.responses = list(responses) + + def chat(self, prompt, temperature): + return self.responses.pop(0) + + +GOOD_RULE = """{ + "pattern": "^.*? (?Psrc=.*)$", + "field_mappings": [ + {"source_field": "src", "ocsf_path": "src_endpoint.ip"}, + {"source_field": "dst", "ocsf_path": "dst_endpoint.ip"} + ] +}""" + +LINES = [ + f"host=h{i:02d} src=10.0.0.{i} dst=10.0.1.{i} " + f"sport={1000 + i} dport={80 + i} action=allow msg=flow ok" + for i in range(20) +] + + +def test_evaluate_corpus_success_run_metrics(): + # One clean attempt per run: response is valid JSON whose pattern compiles. + factory = lambda: FakeClient(GOOD_RULE) + result = sanity_eval.evaluate_corpus( + "fake", LINES, client_factory=factory, runs=2) + assert result["corpus"] == "fake" + assert result["runs"] == 2 + assert result["rules_stored"] == 2 + assert result["responses"] == 2 + assert result["valid_json"] == 2 + assert result["compiles"] == 2 + assert result["mean_attempts"] == 1.0 + assert result["held_out_match_rate"] == 1.0 + assert result["sec_per_rule"] >= 0.0 + + +def test_evaluate_corpus_counts_failed_attempts(): + bad = "not json at all" + factory = lambda: FakeClient(bad, bad, GOOD_RULE) + result = sanity_eval.evaluate_corpus( + "fake", LINES, client_factory=factory, runs=1) + assert result["responses"] == 3 + assert result["valid_json"] == 1 + assert result["mean_attempts"] == 3.0 + assert result["rules_stored"] == 1 + + +def test_sec_per_rule_includes_validation_time(monkeypatch): + # Documented metric: s/rule is "generation + validation, failed runs + # included" — the timing window must stay open through validate_candidate, + # not close the moment generate_candidate returns. The fake gate sleeps + # 50ms per run; with the window closed early sec_per_rule would be ~0. + import types + + def slow_validate(rule, prompt_lines, held_out_lines): + time.sleep(0.05) + return types.SimpleNamespace(held_out_match_rate=1.0) + + monkeypatch.setattr(sanity_eval, "validate_candidate", slow_validate) + result = sanity_eval.evaluate_corpus( + "fake", LINES, client_factory=lambda: FakeClient(GOOD_RULE), runs=1) + assert result["rules_stored"] == 1 + assert result["sec_per_rule"] >= 0.05 + + +# ---------- doc assembly ---------- + +def test_build_doc_carries_header_notes_and_table(): + results = [{ + "corpus": "fake", "runs": 2, "rules_stored": 2, "responses": 2, + "valid_json": 2, "compiles": 2, "mean_attempts": 1.0, + "held_out_match_rate": 1.0, "sec_per_rule": 0.5, + }] + doc = sanity_eval.build_doc( + results, model="fake-model", host_cpu="test-cpu", + date_str="2026-09-20", runs=2) + assert "- model: fake-model" in doc + assert "- host CPU: test-cpu" in doc + assert "- date: 2026-09-20" in doc + assert "- runs per corpus: 2" in doc + assert "| corpus" in doc and "| fake " in doc + + +def test_build_doc_notes_chat_timeout_only_when_set(): + results = [{ + "corpus": "fake", "runs": 1, "rules_stored": 1, "responses": 1, + "valid_json": 1, "compiles": 1, "mean_attempts": 1.0, + "held_out_match_rate": 1.0, "sec_per_rule": 0.5, + }] + doc = sanity_eval.build_doc(results, model="m", host_cpu="c", + date_str="2026-09-21", runs=1, + chat_timeout=1200.0) + assert "- per-chat timeout: 1200s" in doc + plain = sanity_eval.build_doc(results, model="m", host_cpu="c", + date_str="2026-09-21", runs=1) + assert "timeout" not in plain + + +# ---------- --chat-timeout bound (eval-side; the production loop is unbounded) ---------- + +class SlowClient: + """chat() that sleeps past any sane test bound — stands in for a + degenerate SLM answer that would otherwise run for hours.""" + + def __init__(self, delay, result="ok"): + self.delay = delay + self.result = result + + def chat(self, prompt, temperature): + time.sleep(self.delay) + return self.result + + +def test_timeout_client_returns_fast_answers_unchanged(): + client = sanity_eval.TimeoutClient(FakeClient(GOOD_RULE), timeout_s=5.0) + assert client.chat("p", 0.1) == GOOD_RULE + + +def test_timeout_client_bounds_a_runaway_chat(): + client = sanity_eval.TimeoutClient(SlowClient(1.0), timeout_s=0.05) + start = time.monotonic() + with pytest.raises(sanity_eval.ChatTimeout): + client.chat("p", 0.1) + assert time.monotonic() - start < 0.5 # bounded well below the inner 1.0s + + +def test_timeout_client_relays_transport_errors(): + class Boom: + def chat(self, prompt, temperature): + raise ConnectionError("down") + + client = sanity_eval.TimeoutClient(Boom(), timeout_s=5.0) + with pytest.raises(ConnectionError): + client.chat("p", 0.1) + + +def test_chat_timeout_rides_the_production_retry_ladder(): + # ChatTimeout must be an ordinary chat() failure for generate_candidate: + # retried at escalating temperature, then GenerationError -> the eval + # records an honest failed-run row instead of hanging overnight. + calls = [] + + class AlwaysSlow: + def chat(self, prompt, temperature): + calls.append(temperature) + raise sanity_eval.ChatTimeout("no response within 1s") + + with pytest.raises(GenerationError): + sanity_eval.generate_candidate("fp", LINES, AlwaysSlow()) + assert [round(t, 1) for t in calls] == [0.1, 0.3, 0.5] diff --git a/deploy/.env.example b/deploy/.env.example index 73b5553..c652beb 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -1,14 +1,19 @@ # deploy/.env.example — copy to deploy/.env and adjust as needed. # docker compose reads deploy/.env automatically; the real file is gitignored, -# so never commit it. The values below are the M1 development defaults that -# docker-compose.yml also hardcodes for the in-network services. - -# Pipeline service — least-privilege role: INSERT+SELECT on raw_events, -# raw_batches, normalized_events, onboarding_samples; SELECT on rules. -PIPELINE_DATABASE_URL=postgresql://pipeline_role:pipeline_dev@postgres:5432/ulpf - -# Gateway service — read-only role: SELECT on all ulpf tables. -GATEWAY_DATABASE_URL=postgresql://gateway_role:gateway_dev@postgres:5432/ulpf +# so never commit it. # Kafka bootstrap address for the streaming services (inside the compose network). KAFKA_BOOTSTRAP=redpanda:9092 + +# Ollama endpoint used by the onboarding services for SLM calls. +# Default (8GB boxes): the 4b sidecar, enabled by `--profile slm-4b`. +ULPF_OLLAMA_URL=http://ollama-4b:11434 + +# SLM tier tag pulled by the sidecar at startup. Default qwen3:4b; 16GB boxes +# may upgrade via `--profile slm-8b` + ULPF_OLLAMA_URL=http://ollama-8b:11434 +# + ULPF_OLLAMA_MODEL=qwen3:8b. The ollama sidecars publish NO host port — +# they are reachable only inside the compose network (onboarding-internal). +ULPF_OLLAMA_MODEL=qwen3:4b + +# Per-service database DSNs (pipeline, gateway, admin) are compose-internal +# constants defined in docker-compose.yml — M2 adds no user-facing DB knobs. diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 9509a5e..dc7bae6 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -54,6 +54,9 @@ services: build: { context: .., dockerfile: services/gateway/Dockerfile } environment: DATABASE_URL: postgresql://gateway_role:gateway_dev@postgres:5432/ulpf + # M2 write path (Task 7): lifecycle mutations as the least-privilege + # rules_role; reads keep the read-only gateway_role DSN above. + WRITE_DATABASE_URL: postgresql://rules_role:rules_dev@postgres:5432/ulpf ports: ["8000:8000"] depends_on: { migrate: { condition: service_completed_successfully } } web: @@ -65,4 +68,37 @@ services: build: { context: .., dockerfile: simulator/Dockerfile } environment: { COLLECTOR_UDP: collector:5514, COLLECTOR_HTTP: http://collector:8080 } depends_on: [ collector ] -volumes: { pgdata: {} } + # M2 onboarding (cold path): polls onboarding_samples, splits 5/15, asks the + # SLM for a candidate rule, validates it, stores it as pending_review. The + # SLM tier is picked by profile: --profile slm-4b (default, 8GB boxes) or + # --profile slm-8b (16GB) + ULPF_OLLAMA_URL/ULPF_OLLAMA_MODEL in deploy/.env + # pointing at ollama-8b. Fail-closed: with no SLM reachable the loop logs + # and retries — the hot path is never touched. + onboarding: + build: { context: .., dockerfile: services/onboarding/Dockerfile } + environment: + DATABASE_URL: postgresql://onboarding_role:onboarding_dev@postgres:5432/ulpf + OLLAMA_URL: ${ULPF_OLLAMA_URL:-http://ollama-4b:11434} + OLLAMA_MODEL: ${ULPF_OLLAMA_MODEL:-qwen3:4b} + restart: unless-stopped + depends_on: { migrate: { condition: service_completed_successfully } } + # Ollama sidecars. No host port by design: only the onboarding service talks + # to them, inside the compose network. The entrypoint serves, then blocks on + # `ollama pull` until the tier's model is local, then waits on serve. + ollama-4b: + image: ollama/ollama:0.34.2 + profiles: [ slm-4b ] + entrypoint: [ "/bin/sh", "/entrypoint.sh" ] + environment: { MODEL: "qwen3:4b" } + volumes: + - ollama4:/root/.ollama + - ./ollama-entrypoint.sh:/entrypoint.sh:ro + ollama-8b: + image: ollama/ollama:0.34.2 + profiles: [ slm-8b ] + entrypoint: [ "/bin/sh", "/entrypoint.sh" ] + environment: { MODEL: "qwen3:8b" } + volumes: + - ollama8:/root/.ollama + - ./ollama-entrypoint.sh:/entrypoint.sh:ro +volumes: { pgdata: {}, ollama4: {}, ollama8: {} } diff --git a/deploy/migrate.py b/deploy/migrate.py index 56b1fae..3697125 100755 --- a/deploy/migrate.py +++ b/deploy/migrate.py @@ -52,12 +52,16 @@ def main() -> int: for path in pending: print(f"applying {path.name}") - with conn.transaction(): - cur.execute(path.read_text(encoding="utf-8")) - cur.execute( - "INSERT INTO schema_migrations (name) VALUES (%s)", - (path.name,), - ) + try: + with conn.transaction(): + cur.execute(path.read_text(encoding="utf-8")) + cur.execute( + "INSERT INTO schema_migrations (name) VALUES (%s)", + (path.name,), + ) + except Exception as exc: # noqa: BLE001 - any per-file failure (SQL, unreadable file) is reported, not a traceback + print(f"migration {path.name} failed: {exc}", file=sys.stderr) + return 1 print(f"applied {path.name}") return 0 diff --git a/deploy/migrations/003_m2_lifecycle.sql b/deploy/migrations/003_m2_lifecycle.sql new file mode 100644 index 0000000..bd83a14 --- /dev/null +++ b/deploy/migrations/003_m2_lifecycle.sql @@ -0,0 +1,51 @@ +-- deploy/migrations/003_m2_lifecycle.sql — M2 rule lifecycle (R11/R17 rulings). +-- The grant matrix IS the design: content columns of `rules` are UPDATE-denied +-- for every role; the only mutable cell in the event store is +-- normalized_events.superseded_by_event_id, held by onboarding_role. +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='rules_role') THEN + CREATE ROLE rules_role LOGIN PASSWORD 'rules_dev'; END IF; + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='onboarding_role') THEN + CREATE ROLE onboarding_role LOGIN PASSWORD 'onboarding_dev'; END IF; +END $$; + +ALTER TABLE rules ADD CONSTRAINT rules_version_pos CHECK (version >= 1); +ALTER TABLE rules ADD COLUMN validation JSONB; -- written once at candidate creation +CREATE UNIQUE INDEX rules_one_pending ON rules (fingerprint_id) WHERE status = 'pending_review'; + +-- Sample split becomes durable, auditable evidence (anti-leakage fix, spec §6.2). +ALTER TABLE onboarding_samples + ADD CONSTRAINT onboarding_samples_role_chk CHECK (role IN ('prompt','held_out','unused')); +-- Replay idempotency for sample capture: one raw line -> at most one sample. +CREATE UNIQUE INDEX onboarding_samples_raw_id ON onboarding_samples (raw_id); + +-- Reparse-sweep query support (find current rows older than the active version). +CREATE INDEX ne_current_fp_idx ON normalized_events (fingerprint_id) + INCLUDE (rule_version) WHERE superseded_by_event_id IS NULL; + +-- Generation-attempt bookkeeping (operational state, not event data). +CREATE TABLE onboarding_attempts ( + fingerprint_id TEXT PRIMARY KEY, + samples_seen INT NOT NULL, + attempted_at TIMESTAMPTZ NOT NULL DEFAULT now(), + error TEXT +); + +-- gateway write path (approve/reject/manual/deactivate/reactivate) +GRANT SELECT, INSERT ON rules TO rules_role; +GRANT UPDATE (status, activated_at, deactivated_at) ON rules TO rules_role; +GRANT INSERT ON audit_log TO rules_role; +GRANT SELECT ON onboarding_samples TO rules_role; + +-- onboarding service (candidate generation + backlog re-parse, R11) +GRANT SELECT, INSERT ON rules TO onboarding_role; +GRANT INSERT ON audit_log TO onboarding_role; +GRANT SELECT, UPDATE (role) ON onboarding_samples TO onboarding_role; +GRANT SELECT ON raw_events TO onboarding_role; +GRANT INSERT, SELECT, UPDATE (superseded_by_event_id) ON normalized_events TO onboarding_role; +GRANT SELECT, INSERT, UPDATE, DELETE ON onboarding_attempts TO onboarding_role; + +-- T5 carry-forward: gateway should not read migration metadata, and future +-- tables (M3 drift_windows/baseline_profiles) must inherit gateway SELECT. +REVOKE SELECT ON schema_migrations FROM gateway_role; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO gateway_role; diff --git a/deploy/migrations/004_sample_capture_grants.sql b/deploy/migrations/004_sample_capture_grants.sql new file mode 100644 index 0000000..6acd78f --- /dev/null +++ b/deploy/migrations/004_sample_capture_grants.sql @@ -0,0 +1,5 @@ +-- deploy/migrations/004_sample_capture_grants.sql — pipeline sample capture +-- (R17, Task 4) inserts into onboarding_samples, whose BIGSERIAL id needs +-- sequence USAGE; migration 001 granted the TABLE only, which stayed latent +-- until the first sample insert ran (caught by the Task 4 live check). +GRANT USAGE, SELECT ON SEQUENCE onboarding_samples_id_seq TO pipeline_role; diff --git a/deploy/migrations/005_audit_sequence_grants.sql b/deploy/migrations/005_audit_sequence_grants.sql new file mode 100644 index 0000000..8d67c29 --- /dev/null +++ b/deploy/migrations/005_audit_sequence_grants.sql @@ -0,0 +1,8 @@ +-- deploy/migrations/005_audit_sequence_grants.sql — onboarding_role and +-- rules_role INSERT audit_log rows, whose BIGSERIAL id needs sequence USAGE; +-- migration 003 granted the TABLE only, which stayed latent until the first +-- live audit write ran (caught by the Task 11 live verify: the onboarding +-- loop failed with "permission denied for sequence audit_log_id_seq" — the +-- same bug class 004 fixed for pipeline_role/onboarding_samples_id_seq). +GRANT USAGE, SELECT ON SEQUENCE audit_log_id_seq TO onboarding_role; +GRANT USAGE, SELECT ON SEQUENCE audit_log_id_seq TO rules_role; diff --git a/deploy/migrations/006_rules_sequence_grants.sql b/deploy/migrations/006_rules_sequence_grants.sql new file mode 100644 index 0000000..ffc3623 --- /dev/null +++ b/deploy/migrations/006_rules_sequence_grants.sql @@ -0,0 +1,11 @@ +-- deploy/migrations/006_rules_sequence_grants.sql — rules_role and +-- onboarding_role INSERT into rules, whose BIGSERIAL id needs sequence USAGE; +-- migration 003 granted the TABLE only, which stayed latent until the first +-- live candidate insert ran (caught by the Task 12 smoke: F's manual +-- fallback POST /api/rules/manual failed with "permission denied for +-- sequence rules_id_seq" — the same bug class 004 fixed for +-- pipeline_role/onboarding_samples_id_seq and 005 for the audit sequence). +-- rules_role needs it for manual candidates; onboarding_role for SLM +-- candidates (migration 003 grants both roles INSERT ON rules). +GRANT USAGE, SELECT ON SEQUENCE rules_id_seq TO rules_role; +GRANT USAGE, SELECT ON SEQUENCE rules_id_seq TO onboarding_role; diff --git a/deploy/ollama-entrypoint.sh b/deploy/ollama-entrypoint.sh new file mode 100755 index 0000000..3ee2b56 --- /dev/null +++ b/deploy/ollama-entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# deploy/ollama-entrypoint.sh — serve, pull $MODEL until local, wait on serve. +# PID1 is this shell: `wait` keeps the backgrounded `ollama serve` as the +# long-running job after the pull finishes. Exec bit set both ways for NTFS: +# chmod +x deploy/ollama-entrypoint.sh +# git update-index --chmod=+x deploy/ollama-entrypoint.sh +set -e +ollama serve & +sleep 2 +ollama pull "$MODEL" +wait diff --git a/docs/m2-rule-lifecycle.md b/docs/m2-rule-lifecycle.md new file mode 100644 index 0000000..681cae8 --- /dev/null +++ b/docs/m2-rule-lifecycle.md @@ -0,0 +1,106 @@ +# M2 rule lifecycle — ruling record + +How an unknown-appliance fingerprint becomes an active parsing rule, and why +the pieces look the way they do. Migrating decisions from the M2 plan +brainstorm (2026-09-19) are recorded here because they bind later milestones. + +## R11 — the grant matrix is the design + +`deploy/migrations/003_m2_lifecycle.sql`: + +```sql +-- gateway write path (approve/reject/manual/deactivate/reactivate) +GRANT SELECT, INSERT ON rules TO rules_role; +GRANT UPDATE (status, activated_at, deactivated_at) ON rules TO rules_role; +GRANT INSERT ON audit_log TO rules_role; +GRANT SELECT ON onboarding_samples TO rules_role; + +-- onboarding service (candidate generation + backlog re-parse, R11) +GRANT SELECT, INSERT ON rules TO onboarding_role; +GRANT INSERT ON audit_log TO onboarding_role; +GRANT SELECT, UPDATE (role) ON onboarding_samples TO onboarding_role; +GRANT SELECT ON raw_events TO onboarding_role; +GRANT INSERT, SELECT, UPDATE (superseded_by_event_id) ON normalized_events TO onboarding_role; +GRANT SELECT, INSERT, UPDATE, DELETE ON onboarding_attempts TO onboarding_role; +``` + +Completion note (migration `006_rules_sequence_grants.sql`): 003 granted the +TABLES but not the BIGSERIAL backing sequence, so the first live rules INSERT +— the Task 12 smoke's manual candidate — failed with "permission denied for +sequence rules_id_seq" (same bug class 004/005 fixed for the sample and audit +sequences; latent because unit tests mock the DB and no earlier live run ever +stored a rule). 006 grants `USAGE, SELECT ON SEQUENCE rules_id_seq` to +`rules_role` and `onboarding_role`; the sequence audit against +information_schema now covers every INSERT-granted table's sequence. + +Content columns of `rules` are UPDATE-denied for every role — a candidate is +immutable once stored. The event store has exactly ONE mutable cell: +`normalized_events.superseded_by_event_id`, held by `onboarding_role` +(column-scoped; `services/onboarding/reparse.py` issues exactly that +`UPDATE ... SET superseded_by_event_id = %s`). Everything else is +append-only. + +Rejected alternatives: **supersede-via-append** (mark old rows dead by +inserting tombstone rows) kills the `ne_current_idx` partial index, taxes +every hot-path query, and fights spec §4's current-view semantics; a +**migration-role batch** (run the sweep as a superuser from a migration) +opens a runtime superuser path. The sweep instead runs in the onboarding +service under its least-privilege role, batch by batch, each batch one +transaction (insert new-version rows + supersede old ones together), so a +crash never leaves a duplicated or orphaned backlog row and a re-run +converges: whatever stayed stale is re-found, swept rows are a no-op. + +## R17 — the topic feed carries all statuses + +The pipeline produces an envelope +`{event_id, raw_id, fingerprint_id, rule_version, status, ocsf?}` for EVERY +event — parsed, unparsed, and parse_error — keyed by `fingerprint_id`. +Two ledger notes for M3 drift: + +- The topic is **at-least-once** and envelope produce precedes the persist + transaction's commit, so consumers see duplicates. They are dedupable by + construction: `event_id = uuid5(NAMESPACE, "{raw_id}:{rule_version}")` is + deterministic, so dedup key = `(raw_id, rule_version)`. +- The **reparse sweep writes DB rows only — it emits no envelopes.** M3's + drift detector must read the DB current view + (`superseded_by_event_id IS NULL`, served by `ne_current_idx`), not the + topic, or it would miss backlog re-parses. + +The topic contract changed in M2 deliberately: no external consumer existed, +and M1's frozen contract was the HTTP API, not the topic. DLQ stays +pipeline-owned (hot path only); re-parse parse_errors are recorded as rows, +not DLQ messages. + +## Version minting (X-b′) — versions are not gap-free + +Candidates are created `pending_review` with `version = MAX(version)+1` for +the fingerprint. **Plain approve** flips the candidate row in place (status + +timestamps only — content columns are grant-denied). **Approve-with-edits** +INSERTs a fresh row at `version = MAX+1` with the edited content (provenance +`slm-edited`/`human`) and flips the draft candidate to `rejected` with audit +detail `consumed_by_edit`. Rejected candidates therefore burn version +numbers: gaps are expected and accepted. One pending candidate per fingerprint +(partial unique index). Every activation goes through one human-approved code +path. + +## Audit vocabulary (closed set) + +`action` ∈ {`samples_split`, `candidate_created`, `candidate_failed`, +`rule_approved`, `rule_rejected`, `rule_deactivated`, `rule_reactivated`, +`reparse_complete`}; entity = the fingerprint_id; detail JSONB carries +ids/actor/reason/counts. `actor` is a request field (MVP has no auth). + +## Deferred, with numbers + +- **Template miner (spec §9): NOT built.** M1's fragmentation bench + (docs/m1-fragmentation.md) measured 100% mutation stability and zero + fragmentation across all 5 golden corpora / 7 fingerprint groups (incl. + `auto_9d3d5688` acmegw, `auto_8274266a` zenwall) — "only if the numbers + warrant it" is answered no. +- **Baseline profiles (spec §6.2 step 7): M3.** Drift machinery needs + tables that do not exist yet; migration 003's `ALTER DEFAULT PRIVILEGES` + grants gateway SELECT on future tables so they inherit access when added. +- **SLM tier quality** is a recorded limitation of the default qwen3:4b + tier on 8GB hosts (0 rules stored; docs/m2-slm-sanity.md). The human + approval step and deterministic validation gate bound model quality: + a weaker model costs retries and latency, never correctness. diff --git a/docs/m2-slm-sanity.md b/docs/m2-slm-sanity.md new file mode 100644 index 0000000..c196436 --- /dev/null +++ b/docs/m2-slm-sanity.md @@ -0,0 +1,54 @@ +# M2 SLM sanity numbers — onboarding candidate loop + +> **FLAG (ruling 6): qwen3:4b stored ZERO rules on this host — held-out match +> is 0% on all three corpora, far below the 80% sanity threshold.** The table +> shows why this tier is not failing at JSON (valid_json 100%, compiles 100%): +> every response is well-formed and compiles, and every run still exhausts the +> 3-attempt ladder at "0.0 confidence. It failed to match the samples." The +> failure is rule generalization, not structure. Consistent live evidence from +> the same day: 9 bounded candidate-loop generations across two +> retry_new_samples cycles (acmegw + newapp), every one 0.0 held-out +> confidence; `GET /api/rules?status=pending_review` stayed empty. This is a +> RECORDED LIMITATION of the M2 default tier on an 8GB host (4096 ctx, CPU +> decode ~1-3 t/s), not a pipeline defect — the fail-closed loop, retry gate, +> attempt ledger, and audit trail all behaved as designed. Default tier P-1 +> (qwen3:4b) stands per ruling: flipping would ship a default unverifiable on +> the target box; the formal tier/harness question is assigned to M4. + +Measured numbers only (bench/onboarding_eval/sanity_eval.py against a +live Ollama endpoint; the split + generation + validation are the +production code paths). + +- model: qwen3:4b +- host CPU: Intel Core i5-10210U (4C/8T; Docker Desktop WSL2 VM, 3.74 GB) +- date: 2026-09-21 +- runs per corpus: 1 +- per-chat timeout: 1200s (eval-side bound; the production loop is unbounded) + +Metric definitions: `valid_json` = SLM responses from which +`extract_json` recovered an object with `pattern` + `field_mappings`; +`compiles` = of those, patterns that compile (`__JSON__` sentinel counts); +`held_out_match` = mean `validate_candidate` held-out match rate over runs +that produced a rule; `mean_attempts` = mean chat() calls per run; +`s/rule` = run wall-clock per stored rule (failed runs included in the +numerator). Split: 5 prompt / up to 15 held-out per corpus. + +| corpus | runs | rules | valid_json | compiles | held_out_match | mean_attempts | s/rule | +| ------- | ---- | ----- | ---------- | -------- | -------------- | ------------- | ------ | +| acmegw | 1 | 0 | 100.0% | 100.0% | n/a | 3.0 | n/a | +| zenwall | 1 | 0 | 100.0% | 100.0% | n/a | 3.0 | n/a | +| newapp | 1 | 0 | 100.0% | 100.0% | n/a | 3.0 | n/a | + +Note: s/rule values in this recorded run exclude validation time — the +tool's timing window closed before validation ran (corrected after the run; +validation is sub-second regex work over ≤15 lines, and future runs include +it). + +acmegw failures: +- 3 attempts: Failed to generate a valid rule for fingerprint 'sanity-eval' after 3 attempts. Last error: Generated rule yielded 0.0 confidence. It failed to match the samples. Try a simpler regex. + +zenwall failures: +- 3 attempts: Failed to generate a valid rule for fingerprint 'sanity-eval' after 3 attempts. Last error: Generated rule yielded 0.0 confidence. It failed to match the samples. Try a simpler regex. + +newapp failures: +- 3 attempts: Failed to generate a valid rule for fingerprint 'sanity-eval' after 3 attempts. Last error: Generated rule yielded 0.0 confidence. It failed to match the samples. Try a simpler regex. diff --git a/legacy-demo/README.md b/legacy-demo/README.md new file mode 100644 index 0000000..50abd30 --- /dev/null +++ b/legacy-demo/README.md @@ -0,0 +1,9 @@ +# legacy-demo (frozen pre-MVP demo) + +This directory is the frozen pre-MVP demo: services talk in-process (no +Kafka or Postgres), rules persist to SQLite, and onboarding auto-approves +every generated rule. It is kept for reference only and is not maintained — +see the root README and `docs/` for the real system. + +Do not run it on untrusted input: the KV scans here have no timeouts +(M1 ledger, Task 4 note). diff --git a/libs/ulpf-core/tests/test_models.py b/libs/ulpf-core/tests/test_models.py index 8c252e0..1aa6a85 100644 --- a/libs/ulpf-core/tests/test_models.py +++ b/libs/ulpf-core/tests/test_models.py @@ -13,3 +13,27 @@ def test_envelope_rejects_empty_and_oversized(): RawEnvelope(source_id="s", transport="http", received_at="t", raw="") with pytest.raises(ValidationError): RawEnvelope(source_id="s", transport="http", received_at="t", raw="x" * 70000) + + +def test_envelope_format_hint_capped_at_64(): + ok = RawEnvelope(source_id="s", transport="http", received_at="t", + raw="x", format_hint="f" * 64) + assert ok.format_hint == "f" * 64 + with pytest.raises(ValidationError): + RawEnvelope(source_id="s", transport="http", received_at="t", + raw="x", format_hint="f" * 65) + + +def test_event_envelope_contract_shape(): + # R17 topic contract (normalized.events): M3's drift service imports this + # model, so the exact field set and defaults are frozen here. + from ulpf_core.models import EventEnvelope + + env = EventEnvelope(event_id="e", raw_id="r", fingerprint_id="auto_x", + rule_version=2, status="parsed", ocsf={"severity_id": 5}) + assert env.model_dump() == {"event_id": "e", "raw_id": "r", + "fingerprint_id": "auto_x", "rule_version": 2, + "status": "parsed", "ocsf": {"severity_id": 5}} + unparsed = EventEnvelope(event_id="e", raw_id="r", fingerprint_id="f", + rule_version=0, status="unparsed") + assert unparsed.ocsf is None diff --git a/libs/ulpf-core/tests/test_validation.py b/libs/ulpf-core/tests/test_validation.py new file mode 100644 index 0000000..d0efe80 --- /dev/null +++ b/libs/ulpf-core/tests/test_validation.py @@ -0,0 +1,162 @@ +# libs/ulpf-core/tests/test_validation.py — the candidate validation gate, all with concrete data: +import dataclasses +import json +import random + +import pytest +from ulpf_core import validation +from ulpf_core.fingerprint import shape_hash +from ulpf_core.models import Mapping, Rule +from ulpf_core.validation import ( + CandidateReport, + find_hardcoded_literals, + mutate_line, + report_to_json, + validate_candidate, +) + +ACME = [ + "ACMEGW fw01 2026-09-19T10:00:03Z DROP IN=eth0 OUT= SRC=198.51.100.4 DST=203.0.113.9 SPT=52114 DPT=8443", + "ACMEGW fw02 2026-09-19T10:00:11Z ACCEPT IN=eth1 OUT= SRC=198.51.100.9 DST=203.0.113.4 SPT=49300 DPT=443", +] + +# The brief's kv_rule pinned literal DROP, but its own ACME held-out line is an +# ACCEPT event and test_valid_rule_passes_all_checks expects +# held_out_match_rate == 1.0 — so the pattern accepts both actions (the +# fixture's DROP/ACCEPT variety is clearly deliberate). +def kv_rule(pattern=r"^.*?(?:DROP|ACCEPT)\s+(?P.*)$"): + return Rule(fingerprint_id="auto_x", version=1, pattern=pattern, provenance="human", + mappings=[Mapping(source_field=k, ocsf_path=p) for k, p in [ + ("SRC", "src_endpoint.ip"), ("DST", "dst_endpoint.ip"), + ("SPT", "src_endpoint.port"), ("DPT", "dst_endpoint.port")]]) + +def json_rule(): + return Rule(fingerprint_id="json_x", version=1, pattern="__JSON__", provenance="human", + mappings=[Mapping(source_field=s, ocsf_path=o) for s, o in [ + ("src_ip", "src_endpoint.ip"), ("src_port", "src_endpoint.port"), + ("dest_ip", "dst_endpoint.ip"), ("dest_port", "dst_endpoint.port"), + ("timestamp", "time"), ("alert.severity", "severity_id"), + ("event_type", "action"), ("alert.signature", "message")]]) + +def jline(src="198.51.100.4", dst="203.0.113.9", sev=3): + return json.dumps({"timestamp": "2026-09-19T10:00:03Z", "event_type": "alert", + "src_ip": src, "src_port": 52114, "dest_ip": dst, "dest_port": 8443, + "alert": {"severity": sev, "signature": "suspicious dns query"}}, + separators=(",", ":")) + +def test_valid_rule_passes_all_checks(): + report = validate_candidate(kv_rule(), ACME[:1], ACME[1:]) + assert report.passed and all(report.checks.values()) + assert report.held_out_match_rate == 1.0 + assert len(report.previews) == 1 and report.previews[0]["status"] == "parsed" + +def test_pattern_missing_line_fails_match_all(): + bad = kv_rule(pattern=r"^NOMATCH.*$") + report = validate_candidate(bad, ACME[:1], ACME[1:]) + assert report.checks["samples_parse"] is False or report.checks["held_out_match_all"] is False + assert report.passed is False + +def test_hardcoded_hostname_caught_by_probe_and_literal_check(): + hardcoded = kv_rule(pattern=r"^ACMEGW fw01 2026-09-19T10:00:03Z DROP\s+(?P.*)$") + assert find_hardcoded_literals(hardcoded.pattern, ACME[:1]) # ts + hostname literals + report = validate_candidate(hardcoded, ACME[:1], ACME[1:]) + assert report.passed is False # held-out line has different ts/host -> no match + +def test_orphan_mapping_fails(): + rule = kv_rule() + rule = Rule(**{**rule.model_dump(), + "mappings": rule.mappings + [Mapping(source_field="NOPE", ocsf_path="action")]}) + report = validate_candidate(rule, ACME[:1], ACME[1:]) + assert report.checks["no_orphan_mappings"] is False + +def test_mutate_line_changes_values_not_shape(): + rng = random.Random(7) + m = mutate_line(ACME[0], rng) + assert m != ACME[0] and "SRC=" in m and "DST=" in m + +def test_no_samples_degrades_gracefully(): + report = validate_candidate(kv_rule(), [], []) + assert report.passed is True and any("no samples" in n for n in report.notes) + +def test_report_to_json_shape(): + j = report_to_json(validate_candidate(kv_rule(), ACME[:1], ACME[1:])) + assert set(j) == {"passed", "checks", "held_out_match_rate", "notes", "previews", + "prompt_count", "held_out_count"} + +# --- additional cases -------------------------------------------------------- + +def test_port_out_of_range_fails_port_fields_valid(): # mapped value AFTER parse + line = ("ACMEGW fw01 2026-09-19T10:00:03Z DROP IN=eth0 OUT= " + "SRC=198.51.100.4 DST=203.0.113.9 SPT=70000 DPT=443") + report = validate_candidate(kv_rule(), ACME[:1], [line]) + assert report.checks["port_fields_valid"] is False and report.passed is False + +def test_non_ip_fails_ip_fields_valid(): + line = ("ACMEGW fw01 2026-09-19T10:00:03Z DROP IN=eth0 OUT= " + "SRC=banana DST=203.0.113.9 SPT=52114 DPT=443") + report = validate_candidate(kv_rule(), ACME[:1], [line]) + assert report.checks["ip_fields_valid"] is False and report.passed is False + +def test_json_sentinel_rule_validates(): # __JSON__ via the JSON path + report = validate_candidate(json_rule(), [jline()], + [jline(src="198.51.100.8", dst="203.0.113.3", sev=5)]) + assert report.passed and all(report.checks.values()) + assert report.held_out_match_rate == 1.0 + +def test_adversarial_probe_20_mutants_per_prompt_line(): + assert validation.PROBE_MUTANTS_PER_LINE == 20 + brittle = kv_rule(r"^ACMEGW \S+ 2026-09-19T10:00:03Z (?:DROP|ACCEPT)\s+(?P.*)$") + report = validate_candidate(brittle, ACME[:1], ACME[1:]) + assert report.checks["adversarial_probe"] is False and report.passed is False + +def test_validate_rule_output_failures_surface_as_caps_and_allowlist(): + oversized = kv_rule(pattern="^(?:a|b)" * 400 + r"(?P.*)$") # > 1000 chars + report = validate_candidate(oversized, ACME[:1], ACME[1:]) + assert report.checks["caps_and_allowlist"] is False and report.passed is False + assert any("cap" in n for n in report.notes) + base = kv_rule() + evil = Rule(**{**base.model_dump(), + "mappings": base.mappings + + [Mapping(source_field="SRC", ocsf_path="evil.path")]}) + report = validate_candidate(evil, ACME[:1], ACME[1:]) + assert report.checks["caps_and_allowlist"] is False and any("allow" in n for n in report.notes) + +def test_severity_out_of_range_fails_value_sanity(): # severity must be int 0-6 or None + report = validate_candidate(json_rule(), [], [jline(sev=9)]) + assert report.checks["samples_parse"] is False and report.passed is False + assert any("severity" in n for n in report.notes) + +def test_mutate_line_preserves_shape_hash(): # probe premise: shape must not move + rng = random.Random(7) + m = mutate_line(ACME[0], rng) + assert shape_hash(m) == shape_hash(ACME[0]) and "DROP" in m + +def test_previews_capped_at_five_and_counts_recorded(): + held = [f"ACMEGW fw0{i} 2026-09-19T10:00:{i:02d}Z DROP IN=eth0 OUT= " + f"SRC=198.51.100.{i} DST=203.0.113.{i} SPT=5000{i} DPT=44{i}" for i in range(7)] + report = validate_candidate(kv_rule(), ACME[:1], held) + assert len(report.previews) == 5 + assert all(set(p) == {"line", "status", "ocsf"} for p in report.previews) + assert report.held_out_match_rate == 1.0 + j = report_to_json(report) + assert j["prompt_count"] == 1 and j["held_out_count"] == 7 + +def test_hardcoded_literals_common_values_not_flagged(): + # Lines differ in host/IP/port but share the timestamp: the shared ts is + # common to every line and must not count as hardcoded, unlike fw01 below. + same_ts = [ + ("ACMEGW fw01 2026-09-19T10:00:03Z DROP IN=eth0 OUT= " + "SRC=198.51.100.4 DST=203.0.113.9 SPT=52114 DPT=8443"), + ("ACMEGW fw09 2026-09-19T10:00:03Z DROP IN=eth0 OUT= " + "SRC=198.51.100.8 DST=203.0.113.3 SPT=52115 DPT=8444"), + ] + generic = kv_rule() + assert find_hardcoded_literals(generic.pattern, same_ts) == [] + pinned = r"^ACMEGW fw01 \S+ (?:DROP|ACCEPT)\s+(?P.*)$" + assert find_hardcoded_literals(pinned, same_ts) == ["fw01"] + +def test_candidate_report_is_frozen_dataclass(): + report = validate_candidate(kv_rule(), ACME[:1], ACME[1:]) + assert isinstance(report, CandidateReport) + with pytest.raises(dataclasses.FrozenInstanceError): + report.passed = False diff --git a/libs/ulpf-core/ulpf_core/models.py b/libs/ulpf-core/ulpf_core/models.py index ee25be7..f67a8b5 100644 --- a/libs/ulpf-core/ulpf_core/models.py +++ b/libs/ulpf-core/ulpf_core/models.py @@ -6,7 +6,20 @@ class RawEnvelope(BaseModel): transport: str # "syslog-udp" | "syslog-tcp" | "http" received_at: str # ISO8601 UTC, set by collector raw: str = Field(min_length=1, max_length=65536) # line cap, spec §10 - format_hint: str | None = None + format_hint: str | None = Field(default=None, max_length=64) + +class EventEnvelope(BaseModel): + """R17 topic contract (normalized.events, key=fingerprint_id): ONE envelope + per event — parsed, unparsed or parse_error — with the parsed OCSF document + present only for status='parsed'. M3's drift service imports this model, so + field names and types are a frozen cross-service contract.""" + + event_id: str + raw_id: str + fingerprint_id: str + rule_version: int + status: str # "parsed" | "unparsed" | "parse_error" + ocsf: dict | None = None class Mapping(BaseModel): source_field: str diff --git a/libs/ulpf-core/ulpf_core/mutations.py b/libs/ulpf-core/ulpf_core/mutations.py new file mode 100644 index 0000000..ef96814 --- /dev/null +++ b/libs/ulpf-core/ulpf_core/mutations.py @@ -0,0 +1,117 @@ +"""Value mutation for the candidate probe and the fragmentation bench (spec §9, §6.2). + +Single definition of the mutation machinery, ported verbatim from +bench/fingerprint_fragmentation.py (M1) — which now imports this module +instead of carrying its own copy. One switch differs from the bench's +historical entry point: ``vary_text``. The bench additionally replaces runs +of free-text words to prove message wording never changes shape; the +candidate validation probe (ulpf_core.validation) must NOT vary free words, +because a rule pattern may legitimately pin one (e.g. an iptables ``DROP`` +keyword), so it mutates values only. + +Timestamps mutate first and park behind \\x00 sentinels so later numeric +passes cannot chew freshly generated values into shape-breaking forms. +IP:port mutates before the IP pass so the ":port" tail does not lose the +digit its lookbehind needs. +""" + +from __future__ import annotations + +import random +import re +from collections.abc import Callable + +from ulpf_core.fingerprint import _classify + +_ISO_TS_RE = re.compile( + r"(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2}):(\d{2}))?([+-]\d{2}:?\d{2}|Z)?" +) +_SYSLOG_TS_RE = re.compile( + r"\b((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec))" + r"\s+(\d{1,2})\s+(\d{2}):(\d{2}):(\d{2})\b" +) +_PORT_RE = re.compile(r"(?<=\d):\d{1,5}\b") # port tail of IP:port +_IP_RE = re.compile(r"\d{1,3}(?:\.\d{1,3}){3}") +_KV_NUM_RE = re.compile(r"(?<==)\d{1,5}\b") # key=12345 values +_JSON_NUM_RE = re.compile(r'(?<=":)\d{1,5}\b') # "key":12345 values +_HOSTNAME_RE = re.compile(r"\b[a-z][a-z0-9-]*\d{1,4}\b") # fw01, vpn-gw01, eth0 +_SENTINEL_RE = re.compile(r"\x00A(\d+)\x00") +_MONTH_RE = re.compile(r"^(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)$") +_MARKUP_CHARS = set('{}"<>') + +WORDS = [ + "alpha", "bravo", "delta", "echo", "fox", "golf", + "hotel", "india", "juliet", "kilo", "lima", "mike", +] + + +def _rand_iso_ts(match: re.Match[str], rng: random.Random) -> str: + text = f"{rng.randint(2020, 2030):04d}-{rng.randint(1, 12):02d}-{rng.randint(1, 28):02d}" + if match.group(4): + text += f"T{rng.randint(0, 23):02d}:{rng.randint(0, 59):02d}:{rng.randint(0, 59):02d}" + return text + (match.group(7) or "") + + +def _rand_syslog_ts(match: re.Match[str], rng: random.Random) -> str: + # Keep the month word so the mutant still detects as syslog. + return ( + f"{match.group(1)} {rng.randint(1, 28)} " + f"{rng.randint(0, 23):02d}:{rng.randint(0, 59):02d}:{rng.randint(0, 59):02d}" + ) + + +def _rand_ip(rng: random.Random) -> str: + return ".".join(str(rng.randint(1, 254)) for _ in range(4)) + + +def mutate_line(line: str, rng: random.Random, *, vary_text: bool = False) -> str: + """Return a same-shape mutant of *line*: replaces an IPv4, a port, a + timestamp, a hostname token (``h\\d+``/``fw\\d+`` style) or a run of + digits — values change, shape must not. + + With ``vary_text=True`` (the bench) the free-text pass also randomizes + runs of free-text words; the probe (default) leaves free words alone. + """ + stash: list[str] = [] + + def _swap(make: Callable[[re.Match[str]], str]) -> Callable[[re.Match[str]], str]: + def _sub(match: re.Match[str]) -> str: + stash.append(make(match)) + return f"\x00A{len(stash) - 1}\x00" + + return _sub + + line = _ISO_TS_RE.sub(_swap(lambda m: _rand_iso_ts(m, rng)), line) + line = _SYSLOG_TS_RE.sub(_swap(lambda m: _rand_syslog_ts(m, rng)), line) + line = _PORT_RE.sub(_swap(lambda m: f":{rng.randint(1, 65535)}"), line) + line = _IP_RE.sub(_swap(lambda m: _rand_ip(rng)), line) + line = _KV_NUM_RE.sub(_swap(lambda m: str(rng.randint(1, 65535))), line) + line = _JSON_NUM_RE.sub(_swap(lambda m: str(rng.randint(1, 65535))), line) + line = _HOSTNAME_RE.sub(_swap(lambda m: f"h{rng.randint(0, 9999):04d}"), line) + + # Free-text runs (message bodies) get a random word count. Structural + # tokens (markup, sentinels, syslog month, anything classified) are kept. + if vary_text and " " in line: + kept: list[str] = [] + run: list[str] = [] + + def _flush() -> None: + if run: + kept.extend(rng.choice(WORDS) for _ in range(rng.randint(1, 6))) + run.clear() + + for token in line.split(): + if ( + _SENTINEL_RE.fullmatch(token) + or _MONTH_RE.fullmatch(token) + or _MARKUP_CHARS.intersection(token) + or _classify(token) != "W" + ): + _flush() + kept.append(token) + else: + run.append(token) + _flush() + line = " ".join(kept) + + return _SENTINEL_RE.sub(lambda m: stash[int(m.group(1))], line) diff --git a/libs/ulpf-core/ulpf_core/validation.py b/libs/ulpf-core/ulpf_core/validation.py new file mode 100644 index 0000000..f526dd0 --- /dev/null +++ b/libs/ulpf-core/ulpf_core/validation.py @@ -0,0 +1,235 @@ +"""Deterministic candidate validation gate (spec §6.2 step 4, §6.3). + +Shared by the onboarding service (SLM candidates) and the gateway (manual +authoring + approve-with-edits revalidation). Everything here is pure: rules +and lines in, CandidateReport out; no DB, no model calls. +""" +import random +import re +from dataclasses import dataclass, field + +import regex + +# Probe mutation ops (IPv4/port/timestamp/hostname/digit-run, values-only). +# Single definition ported from bench/fingerprint_fragmentation.py lives in +# ulpf_core.mutations; re-exported here as the M2 interface name (Tasks 5/7 +# import ulpf_core.validation.mutate_line). Bench passes vary_text=True there. +from ulpf_core.mutations import mutate_line +from ulpf_core.parsing import JSON_SENTINEL, parse, validate_rule_output + +CHECK_KEYS = ("caps_and_allowlist", "samples_parse", "held_out_match_all", + "ip_fields_valid", "port_fields_valid", "no_orphan_mappings", + "adversarial_probe", "no_hardcoded_literals") + +# Adversarial probe: value mutants per prompt line, all from one fixed seed so +# validate_candidate is deterministic (tests must be stable across runs). +PROBE_MUTANTS_PER_LINE = 20 +PROBE_SEED = 20260919 +PREVIEW_CAP = 5 + + +@dataclass(frozen=True) +class CandidateReport: + passed: bool + checks: dict[str, bool] + held_out_match_rate: float + notes: list[str] = field(default_factory=list) + previews: list[dict] = field(default_factory=list) + # Line counts so report_to_json can emit truthful prompt_count / + # held_out_count; trailing defaults keep the five-field contract above. + prompt_count: int = 0 + held_out_count: int = 0 + + +# Mapped-value sanity shapes (checked AFTER parse). +_IP_RE = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") +# Literal-extraction scans for the hardcoding check: timestamps, IPs, +# hostnames, then >=3-digit numbers outside already-claimed spans. +_TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}[T ][0-9:.]+Z?") +_IP_SCAN_RE = re.compile(r"\d{1,3}(?:\.\d{1,3}){3}") +_HOSTNAME_SCAN_RE = re.compile(r"\b[a-z][a-z0-9-]*\d{1,4}\b") # fw01, vpn-gw01, eth0 +_NUM_SCAN_RE = re.compile(r"\d{3,}") + + +def validate_candidate(rule, prompt_lines, held_out_lines): + """All CHECK_KEYS True == candidate may be stored for review. With no + samples, sample-dependent checks are skipped-and-noted (check key stays + True, a 'no samples' note is added) — manual authoring for an unseen + format must not be blocked for lack of samples.""" + samples = prompt_lines + held_out_lines + # 1. caps/compile/allow-list/output-schema via the M1 gate + base = validate_rule_output(rule, samples) if samples else [] + checks = {"caps_and_allowlist": not base} + notes = list(base) + if not samples: + notes.append("no samples provided; sample-dependent checks (parse, value " + "sanity, orphan, probe, hardcoding) were skipped") + + # 2. per-line parse + value sanity (prompt + held-out for sanity; match-rate + # counted on held-out ONLY) + docs: list[dict] = [] + samples_parse = ip_fields_valid = port_fields_valid = True + held_out_matched = 0 + previews: list[dict] = [] + for role, lines in (("prompt", prompt_lines), ("held-out", held_out_lines)): + for i, line in enumerate(lines): + doc, err = parse(line, rule) + if err is not None: + samples_parse = False + notes.append(f"{role} line {i}: {err}") + if role == "held-out" and len(previews) < PREVIEW_CAP: + previews.append({"line": line, "status": "error", "ocsf": None}) + continue + docs.append(doc) + if role == "held-out": + held_out_matched += 1 + if len(previews) < PREVIEW_CAP: + previews.append({"line": line, "status": "parsed", "ocsf": doc}) + for key, note in _value_sanity(doc, role, i): + notes.append(note) + if key == "ip_fields_valid": + ip_fields_valid = False + elif key == "port_fields_valid": + port_fields_valid = False + else: + samples_parse = False + checks["samples_parse"] = samples_parse + checks["held_out_match_all"] = held_out_matched == len(held_out_lines) + checks["ip_fields_valid"] = ip_fields_valid + checks["port_fields_valid"] = port_fields_valid + + # 3. orphans across all produced fields (sample-dependent: with no samples + # nothing can be observed, so skip-and-keep-True like the rest) + if samples: + seen = _observed_fields(rule, docs) + orphans = [m.source_field for m in rule.mappings if m.source_field not in seen] + checks["no_orphan_mappings"] = not orphans + if orphans: + notes.append(f"orphan mappings (source field never observed): {orphans}") + else: + checks["no_orphan_mappings"] = True + + # 4. adversarial probe: N mutants per prompt line must still parse + probe_ok = True + if prompt_lines: + rng = random.Random(PROBE_SEED) + for i, line in enumerate(prompt_lines): + for _ in range(PROBE_MUTANTS_PER_LINE): + doc, err = parse(mutate_line(line, rng), rule) + if err is not None: + probe_ok = False + notes.append( + f"adversarial probe: prompt line {i} mutant failed to parse: {err}" + ) + break + checks["adversarial_probe"] = probe_ok + + # 5. hardcoding: literals that vary across prompt lines must not appear + # verbatim in the pattern + offending = find_hardcoded_literals(rule.pattern, prompt_lines) + checks["no_hardcoded_literals"] = not offending + if offending: + notes.append(f"hardcoded literals (vary across prompt lines): {offending}") + + rate = held_out_matched / len(held_out_lines) if held_out_lines else 1.0 + return CandidateReport(passed=all(checks.values()), checks=checks, + held_out_match_rate=rate, notes=notes, previews=previews, + prompt_count=len(prompt_lines), held_out_count=len(held_out_lines)) + + +def report_to_json(report): + """The rules.validation JSONB shape (exact key set; Tasks 5/7 store and + serve this verbatim).""" + return { + "passed": report.passed, + "checks": dict(report.checks), + "held_out_match_rate": report.held_out_match_rate, + "notes": list(report.notes), + "previews": [dict(p) for p in report.previews], + "prompt_count": report.prompt_count, + "held_out_count": report.held_out_count, + } + + +def find_hardcoded_literals(pattern, prompt_lines): + """Literals that DISTINGUISH one prompt line from another (timestamps, + IPs, hostnames, >=3-digit numbers) yet appear verbatim in `pattern`. + Token-diff approach: extract candidate literals per line with the scan + regexes; a literal present in the pattern AND not common to every line is + hardcoded. With fewer than two prompt lines nothing is provably common — + a single sample gives no evidence of variation, so any of its literals + baked into the pattern counts. Returns the sorted offending literals.""" + per_line = [_extract_literals(line) for line in prompt_lines] + common = set.intersection(*per_line) if len(per_line) >= 2 else set() + return sorted({lit for lits in per_line for lit in lits + if lit not in common and lit in pattern}) + + +def _extract_literals(line: str) -> set[str]: + """Timestamps, IPs and hostnames, plus >=3-digit runs that are not part + of one of those (a 100 inside an IP is not a standalone literal).""" + lits: set[str] = set() + claimed: list[tuple[int, int]] = [] + for rx in (_TS_RE, _IP_SCAN_RE, _HOSTNAME_SCAN_RE): + for m in rx.finditer(line): + lits.add(m.group()) + claimed.append(m.span()) + for m in _NUM_SCAN_RE.finditer(line): + if any(max(s, m.start()) < min(e, m.end()) for s, e in claimed): + continue + lits.add(m.group()) + return lits + + +def _value_sanity(doc: dict, role: str, i: int): + """Yield (CHECK_KEY, note) per mapped value failing sanity. IP values + must match _IP_RE, ports must be ints in 0-65535 (a port that survived + _as_int as a string is a failure), severity_id an int in 0-6 or None.""" + for path in ("src_endpoint.ip", "dst_endpoint.ip"): + v = _doc_get(doc, path) + if v is not None and not (isinstance(v, str) and _IP_RE.match(v)): + yield "ip_fields_valid", f"{role} line {i}: {path} value {v!r} is not an IPv4 address" + for path in ("src_endpoint.port", "dst_endpoint.port"): + v = _doc_get(doc, path) + if v is not None and not _is_int_in(v, 0, 65535): + yield ("port_fields_valid", + f"{role} line {i}: {path} value {v!r} is not a port in 0-65535") + sev = doc.get("severity_id") + if sev is not None and not _is_int_in(sev, 0, 6): + yield "samples_parse", f"{role} line {i}: severity_id {sev!r} is not an int in 0-6" + + +def _is_int_in(v, lo: int, hi: int) -> bool: + # bool is an int subclass — True/False are neither ports nor severities. + return isinstance(v, int) and not isinstance(v, bool) and lo <= v <= hi + + +def _doc_get(doc: dict, dotted_path: str): + """Walk a document by dotted OCSF path ('src_endpoint.ip').""" + current = doc + for part in dotted_path.split("."): + if not isinstance(current, dict) or part not in current: + return None + current = current[part] + return current + + +def _observed_fields(rule, docs) -> set[str]: + """Every source-field name the samples actually produced: named groups of + the pattern, keys seen in parsed docs' `unmapped`, and mapped source + fields (they left a non-None value at their OCSF target — a mapped + extension-blob key never shows up in `unmapped`). Ports the + fields_seen_anywhere logic of legacy-demo/ingestion/validate_rule.py, + adapted to parse() outputs.""" + seen: set[str] = set() + if rule.pattern != JSON_SENTINEL: + try: + seen |= set(regex.compile(rule.pattern).groupindex) + except regex.error: + pass # broken patterns are already reported by caps_and_allowlist + for doc in docs: + seen |= set((doc.get("unmapped") or {}).keys()) + resolved = {m.ocsf_path for m in rule.mappings + if any(_doc_get(d, m.ocsf_path) is not None for d in docs)} + seen |= {m.source_field for m in rule.mappings if m.ocsf_path in resolved} + return seen diff --git a/pyproject.toml b/pyproject.toml index 80cfa3f..d12a5a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.pytest.ini_options] -testpaths = ["libs", "services", "simulator"] -pythonpath = [".", "services/collector", "services/pipeline", "services/gateway", "simulator"] +testpaths = ["libs", "services", "simulator", "bench"] +pythonpath = [".", "services/collector", "services/pipeline", "services/gateway", "simulator", "services/onboarding"] [tool.ruff] line-length = 100 diff --git a/scripts/smoke.py b/scripts/smoke.py index 7610126..2039ae1 100755 --- a/scripts/smoke.py +++ b/scripts/smoke.py @@ -1,20 +1,35 @@ #!/usr/bin/env python3 -"""ULPF M1 acceptance smoke (Task 12, spec section 15) — the milestone exit gate. +"""ULPF acceptance smoke (M1 Task 12 + M2 Task 12, spec section 15) — the +milestone exit gate. End-to-end against the live compose stack: drives the simulator over the golden corpora, then verifies lossless ingest, the parsed/unparsed split, raw traceability, forced replay idempotency (consumer-group delete + pipeline restart), and independently re-verifies EVERY raw_batches merkle root plus the -per-partition hash chain linkage. - -Host requirements (Python 3.13 tested): psycopg[binary] only. Everything else -is stdlib: urllib for HTTP, subprocess for docker compose; all Kafka -interaction is `docker compose exec redpanda rpk` (no confluent-kafka -host-side). +per-partition hash chain linkage (M1 asserts A-E). The M2 asserts then drive +the onboarding loop end to end: F posts the unknown newapp corpus and takes a +pending_review candidate for its fingerprint through approve -> backlog +re-parse (SLM candidate when the tier produces one, hand-authored manual +fallback otherwise — expected on the default qwen3:4b tier, see +docs/m2-slm-sanity.md), G does the same deterministically for the zenwall +corpus via /api/rules/manual, and H re-runs the replay dance with active +rules, proving raw/normalized/stats/sample-count idempotency including the +onboarding sample dedup. + +Host requirements (Python 3.13 tested): psycopg[binary] plus the two libs +importable (pip install -e libs/ulpf-core -e libs/ocsf-schema — the smoke +computes fingerprints and validates the hand-authored rules host-side with the +same ulpf_core gate the services run). Everything else is stdlib: urllib for +HTTP, subprocess for docker compose; all Kafka interaction is `docker compose +exec redpanda rpk` (no confluent-kafka host-side). Docker commands run from deploy/; the DB is NOT reset — every count assertion -is delta-based (snapshot before, diff after), so re-runs accumulate. +is delta-based (snapshot before, diff after), so re-runs accumulate (the M2 +asserts snapshot per run too: approved rules from earlier runs persist, later +runs mint the next version and assert deltas). Usage: python scripts/smoke.py (from the repo root) + ULPF_SMOKE_SLM_WAIT_S=300 (seconds to wait for an SLM candidate + before the hand-authored fallback) Exit: 0 all checks PASS, 1 any FAIL. """ @@ -22,6 +37,7 @@ import hashlib import json +import os import random import re import subprocess @@ -38,6 +54,15 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "services")) from pipeline.hashchain import GENESIS_PREV_HASH, merkle_root +# ulpf_core is needed host-side for Asserts F/G: fingerprint_id() names the +# newapp/zenwall fingerprints exactly as the pipeline mints them, and +# validate_candidate() fast-fails the hand-authored fallback rules through the +# SAME deterministic gate the gateway runs server-side. +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "libs" / "ulpf-core")) +from ulpf_core.fingerprint import fingerprint_id +from ulpf_core.models import Mapping, Rule +from ulpf_core.validation import validate_candidate + ROOT = Path(__file__).resolve().parents[1] DEPLOY = ROOT / "deploy" DSN = "postgresql://postgres:postgres@localhost:5433/ulpf" # admin, host port (5433: compose publish; 5432 often owned by a native Postgres) @@ -56,6 +81,44 @@ LAG_TIMEOUT_S = 180 POLL_INTERVAL_S = 2.0 +# M2 (Asserts F/G): R-T12-pre-a — the SLM candidate wait is env-tunable; on +# the default qwen3:4b tier over an 8GB host no candidate ever stores (0.0 +# held-out confidence, docs/m2-slm-sanity.md), so the wait expires and the +# hand-authored manual fallback fires. 300s gives a faster tier room to land. +SLM_WAIT_TIMEOUT_S = int(os.environ.get("ULPF_SMOKE_SLM_WAIT_S", "300")) +SAMPLES_WAIT_TIMEOUT_S = 60 # onboarding sample capture for a posted corpus +REPARSE_TIMEOUT_S = 120 # approve -> reparse_complete audit row (R-T12-pre-d) +AUDIT_PAGE = 200 # /api/audit page size (>= any fingerprint's M2-run trail) + +COLLECTOR = "http://localhost:8080" +NEWAPP_SOURCE = "newapp01" +ZENWALL_SOURCE = "zenwall01" + +NEWAPP_CORPUS = ROOT / "simulator" / "data" / "raw_logs_newapp.txt" +ZENWALL_CORPUS = ROOT / "simulator" / "data" / "raw_logs_zenwall.txt" +ZENWALL_GOLDEN = ROOT / "libs" / "ulpf-core" / "tests" / "data" / "golden" / "raw_logs_zenwall.txt" + +# Hand-authored rules (R-T12-pre-a: the newapp fallback is DISTINCT from G's +# generic pattern). Each targets the corpus's real shape; both pass the full +# validate_candidate gate against their corpus (checked host-side before the +# candidate is ever POSTed — same fast-fail discipline as the gateway). +# newapp: anchor the appliance envelope, map the KV extension keys per the +# §6.4 allow-list (ts/src/srcport/dst/dstport/action/msg). +NEWAPP_RULE = ( + r"^\S+ flowgate (?P.*)$", + (("ts", "time"), ("src", "src_endpoint.ip"), ("srcport", "src_endpoint.port"), + ("dst", "dst_endpoint.ip"), ("dstport", "dst_endpoint.port"), + ("action", "action"), ("msg", "message")), +) +# zenwall: the generic extension-blob pattern (the AcmeGW story) — the whole +# line is one KV scan. The file's ONLY machine-readable KV key is `rule=` +# (src=/dst= sit behind `;` separators the KV grammar cannot split), so the +# only allow-listed mapping is rule -> message. +ZENWALL_RULE = ( + r"^.*?(?P.*)$", + (("rule", "message"),), +) + _SIM_LINE = re.compile(r"^(\w+) sent=(\d+)$", re.MULTILINE) _LAG_LINE = re.compile(r"^TOTAL-LAG\s+(\S+)\s*$", re.MULTILINE) @@ -85,6 +148,11 @@ def http_get(url: str, timeout: float = 10) -> tuple[int, bytes]: return resp.status, resp.read() except urllib.error.HTTPError as exc: return exc.code, b"" + except (urllib.error.URLError, OSError, TimeoutError): + # Transient stack stall (e.g. the SLM ladder saturating the box during + # the candidate wait): poll loops must degrade to retry, not crash. + # Status 0 = "no answer"; every caller treats it as keep-waiting/fail. + return 0, b"" def http_json(url: str, timeout: float = 10) -> tuple[int, object]: @@ -197,22 +265,29 @@ def fingerprint_split_table(before: dict, after: dict) -> str: def step1_up() -> None: - print("== Step 1: docker compose up -d --build + health wait ==") - out = compose("up", "-d", "--build", timeout=BUILD_TIMEOUT_S) + print("== Step 1: docker compose --profile slm-4b up -d --build + health wait ==") + # --profile slm-4b: the onboarding service's SLM tier sidecar (R-T12-pre-b). + out = compose("--profile", "slm-4b", "up", "-d", "--build", timeout=BUILD_TIMEOUT_S) if out.returncode != 0: print(out.stdout[-2000:]) print(out.stderr[-2000:]) sys.exit("FATAL: compose up failed; cannot run the smoke.") ok_gw = wait_http_ready(f"{GATEWAY}/api/stats", "gateway", READY_TIMEOUT_S) ok_web = wait_http_ready(f"{WEB}/", "web", READY_TIMEOUT_S) + # onboarding readiness surface: /api/rules serves the M2 review API the + # asserts poll (gateway up + migrations applied). + ok_rules = wait_http_ready(f"{GATEWAY}/api/rules", "gateway M2 rules API", READY_TIMEOUT_S) ok_group = wait_group_present(READY_TIMEOUT_S) - if not (ok_gw and ok_web and ok_group): - sys.exit("FATAL: stack not ready (gateway/web/kafka group 'pipeline').") + if not (ok_gw and ok_web and ok_rules and ok_group): + sys.exit("FATAL: stack not ready (gateway/web/rules API/kafka group 'pipeline').") def run_simulator() -> dict[str, int]: print(f"== Step 2: simulator --eps {SIM_EPS} --duration {SIM_DURATION} ==") - out = compose("run", "--rm", "-T", "simulator", + # --profile sim BEFORE run: the simulator service is profile-gated + # (docker-compose.yml), and `docker compose run` refuses a gated service + # whose profile is not active (R-T12-pre-b). + out = compose("--profile", "sim", "run", "--rm", "-T", "simulator", "--eps", SIM_EPS, "--duration", SIM_DURATION, timeout=SIM_TIMEOUT_S) sent = {name: int(n) for name, n in _SIM_LINE.findall(out.stdout)} if out.returncode != 0 or "total" not in sent: @@ -234,6 +309,24 @@ def assert_a(before_raw: int, total_sent: int) -> None: f"sent={total_sent} raw_events delta={delta} (before={before_raw} after={raw_after})") +def active_rule_fingerprints() -> set[str]: + """Fingerprints with an ACTIVE rule (GET /api/rules?status=active); empty + set when the endpoint is down, so the unparsed check below stays strict. + + Assert B's 'auto_* must have parsed == 0' encodes M1's world where no auto + fingerprint ever had a rule. M2's hero loop legitimately activates one — + and the DB accumulates across runs, so from the first approved candidate + onward that fingerprint parses BY DESIGN (the pipeline parses live traffic + under active rules). The guarantee being regression-tested is conditional: + an auto fingerprint with NO active rule must stay unparsed. Whenever no + auto-fingerprint rule is active, this returns empty and assert_b behaves + exactly as M1 wrote it.""" + status, body = http_json(f"{GATEWAY}/api/rules?status=active") + if status != 200 or not body: + return set() + return {row["fingerprint_id"] for row in body.get("rules", [])} + + def assert_b(before_fps: dict) -> None: print("== Assert B: parsed split via /api/stats by_fingerprint ==") after = stats_by_fingerprint() @@ -254,8 +347,9 @@ def assert_b(before_fps: dict) -> None: autos = {fp: row for fp, row in after.items() if fp.startswith("auto_")} auto_delta = sum(row["total"] - before_fps.get(fp, {}).get("total", 0) for fp, row in autos.items()) + ruled_fps = active_rule_fingerprints() & set(autos) for fp, row in autos.items(): - if row["parsed"] != 0: + if row["parsed"] != 0 and fp not in ruled_fps: ok = False problems.append(f"{fp}: parsed={row['parsed']} (must be 0)") if not autos or auto_delta == 0: @@ -263,8 +357,11 @@ def assert_b(before_fps: dict) -> None: problems.append("no auto_* fingerprint saw traffic (AcmeGW unparsed path unexercised)") total_sent_b = seeded_delta + auto_delta details.append(f" seeded delta={seeded_delta}, auto delta={auto_delta}, sum={total_sent_b}") + auto_note = "" if not ruled_fps else \ + f" (active-rule auto fps parsed by design: {', '.join(sorted(ruled_fps))})" check("B parsed split", ok, "; ".join(problems) if problems else - f"5 seeded fingerprints fully parsed, auto_* fully unparsed{details[0]}\n{details[1]}") + f"5 seeded fingerprints fully parsed, rule-less auto_* fully unparsed{auto_note}" + f"{details[0]}\n{details[1]}") def assert_c() -> None: @@ -310,21 +407,19 @@ def assert_c() -> None: + ("" if ok else "; problems: " + "; ".join(problems[:5]))) -def assert_d_replay() -> None: - print("== Assert D: forced replay (group delete + pipeline restart) ==") - pre_raw = db_count("SELECT count(*) FROM raw_events") - pre_batches = db_count("SELECT count(*) FROM raw_batches") - pre_fps = stats_by_fingerprint() - +def replay_dance() -> tuple[bool, str]: + """The M1 forced-replay dance: stop pipeline -> wait for the consumer to + leave the group -> delete the consumer group -> start pipeline -> wait for + lag 0. Extracted verbatim from assert_d_replay (Task 12) so Assert H can + re-run the same dance after the M2 asserts; every branch here behaved + identically inside assert_d_replay before the extraction.""" # Order matters: the pipeline is stopped BEFORE the group delete. Deleted # while running, the live consumer's auto-rejoin recreates the group and # re-commits its offsets within seconds, so a restart afterwards would have # nothing to redeliver (vacuous replay, observed as raw_batches unchanged). stop = compose("stop", "pipeline", timeout=180) if stop.returncode != 0: - check("D replay idempotency", False, - f"compose stop pipeline failed: {stop.stderr.strip()[:200]}") - return + return False, f"compose stop pipeline failed: {stop.stderr.strip()[:200]}" deadline = time.monotonic() + 60 saw_members_zero = False last_detail = "group describe never succeeded" @@ -342,29 +437,60 @@ def assert_d_replay() -> None: listing = rpk("group", "list") if not group_listed(listing): - check("D replay idempotency", False, - f"kafka group 'pipeline' absent before delete (replay would be vacuous): " - f"{listing.stdout.strip()!r}") - return + return False, (f"kafka group 'pipeline' absent before delete (replay would be " + f"vacuous): {listing.stdout.strip()!r}") delete = rpk("group", "delete", "pipeline") if delete.returncode != 0: - check("D replay idempotency", False, - f"rpk group delete failed rc={delete.returncode}: {delete.stderr.strip()[:200]}") - return + return False, (f"rpk group delete failed rc={delete.returncode}: " + f"{delete.stderr.strip()[:200]}") print(" group 'pipeline' deleted (committed offsets dropped); starting pipeline") start = compose("start", "pipeline", timeout=180) if start.returncode != 0: - check("D replay idempotency", False, f"compose start pipeline failed: " - f"{start.stderr.strip()[:200]}") - return + return False, f"compose start pipeline failed: {start.stderr.strip()[:200]}" lag_ok, lag_detail = wait_group_lag_zero(LAG_TIMEOUT_S) if not lag_ok: - check("D replay idempotency", False, f"group lag never reached 0: {lag_detail}") - return + return False, f"group lag never reached 0: {lag_detail}" print(f" replay drained ({lag_detail})") + return True, lag_detail + + +def wait_batches_stable(window_s: float = 3.0, timeout_s: float = 90.0) -> int: + """After the replay dance reports lag 0, wait until raw_batches stops + moving and return its count. + + The worker commits offsets only AFTER its persist transactions, so lag 0 + should imply the replay's batches are durable — but a lag poll can hit + the window where the recreated group exists WITHOUT committed offsets + (observed live: rpk reported 0 before the redelivered batches landed), + which would make the post-replay snapshots early. Gating on the batch + count being quiet across a full poll window removes that race. + """ + deadline = time.monotonic() + timeout_s + last = db_count("SELECT count(*) FROM raw_batches") + last_change = time.monotonic() + while time.monotonic() < deadline: + time.sleep(POLL_INTERVAL_S) + current = db_count("SELECT count(*) FROM raw_batches") + if current != last: + last, last_change = current, time.monotonic() + elif time.monotonic() - last_change >= window_s: + return current + return last + + +def assert_d_replay() -> None: + print("== Assert D: forced replay (group delete + pipeline restart) ==") + pre_raw = db_count("SELECT count(*) FROM raw_events") + pre_batches = db_count("SELECT count(*) FROM raw_batches") + pre_fps = stats_by_fingerprint() + + danced, dance_detail = replay_dance() + if not danced: + check("D replay idempotency", False, dance_detail) + return + post_batches = wait_batches_stable() post_raw = db_count("SELECT count(*) FROM raw_events") - post_batches = db_count("SELECT count(*) FROM raw_batches") post_fps = stats_by_fingerprint() summary = ( f"raw_events {pre_raw}->{post_raw}, raw_batches {pre_batches}->{post_batches}, " @@ -425,6 +551,383 @@ def assert_e_chain() -> None: + ("" if ok else "; problems: " + "; ".join(problems[:5]))) +# --- M2: onboarding hero loop (Task 12 asserts F/G/H) --------------------------- + +def http_post_json(url: str, payload: dict, timeout: float = 30) -> tuple[int, object]: + """POST JSON, return (status, decoded body). 4xx/5xx bodies are decoded + too (the gateway's 422 carries {checks, notes} the assert should quote).""" + req = urllib.request.Request( + url, data=json.dumps(payload).encode("utf-8"), method="POST", + headers={"Content-Type": "application/json", "Accept": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + status, raw = resp.status, resp.read() + except urllib.error.HTTPError as exc: + status, raw = exc.code, exc.read() + except (urllib.error.URLError, OSError, TimeoutError): + return 0, None # network-level failure; no auto-retry (see below) + try: + return status, json.loads(raw.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + return status, None + + +def read_corpus_lines(path: Path) -> list[str]: + lines = [ln.strip() for ln in path.read_text(encoding="utf-8").splitlines() if ln.strip()] + if not lines: + sys.exit(f"FATAL: corpus {path} is empty.") + return lines + + +def corpus_fingerprint(lines: list[str], name: str) -> str: + """The corpus's single fingerprint_id, computed host-side exactly as the + pipeline mints it (same ulpf_core.fingerprint function).""" + fps = {fingerprint_id(ln) for ln in lines} + if len(fps) != 1: + sys.exit(f"FATAL: {name} corpus spans {len(fps)} fingerprints: {sorted(fps)}") + return fps.pop() + + +def ensure_zenwall_corpus() -> list[str]: + """G's corpus: simulator/data/raw_logs_zenwall.txt; copied from the golden + original when absent (the copy is inert to the simulator — load_corpora + iterates the fixed CORPUS_KEYS, not the directory).""" + if not ZENWALL_CORPUS.exists(): + ZENWALL_CORPUS.write_text(ZENWALL_GOLDEN.read_text(encoding="utf-8"), + encoding="utf-8", newline="\n") + print(f" copied golden zenwall corpus -> {ZENWALL_CORPUS.relative_to(ROOT)}") + return read_corpus_lines(ZENWALL_CORPUS) + + +def post_corpus(lines: list[str], source_id: str) -> int: + """POST the whole corpus x5 to the collector (new HTTP raw events each + time: raw_id is topic/partition/offset-derived, so repeats accumulate).""" + accepted = 0 + for _ in range(5): + status, body = http_post_json( + f"{COLLECTOR}/v1/ingest", {"source_id": source_id, "lines": lines}) + if status != 202 or not isinstance(body, dict): + sys.exit(f"FATAL: POST {source_id} corpus -> {status}: {body!r}") + accepted += int(body.get("accepted", 0)) + return accepted + + +def wait_samples_total(fp: str, minimum: int, timeout_s: float) -> int: + """Poll /api/onboarding/samples?fingerprint= until total >= minimum.""" + deadline = time.monotonic() + timeout_s + total = -1 + while time.monotonic() < deadline: + status, body = http_json(f"{GATEWAY}/api/onboarding/samples?fingerprint={fp}") + if status == 200 and isinstance(body, dict): + total = int(body.get("total", 0)) + if total >= minimum: + return total + time.sleep(POLL_INTERVAL_S) + return total + + +def pending_candidate_for(fp: str) -> dict | None: + status, body = http_json(f"{GATEWAY}/api/rules?status=pending_review") + if status == 200 and isinstance(body, dict): + for rule in body.get("rules", []): + if rule.get("fingerprint_id") == fp: + return rule + return None + + +def wait_pending_candidate(fp: str, timeout_s: float) -> dict | None: + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + candidate = pending_candidate_for(fp) + if candidate is not None: + return candidate + time.sleep(POLL_INTERVAL_S) + return None + + +def supersede_counts(fp: str) -> tuple[int, int]: + """(unsuperseded, superseded) normalized_events rows for the fingerprint — + the pre-approve snapshot pair R-T12-pre-d's delta assert needs.""" + with psycopg.connect(DSN) as conn, conn.cursor() as cur: + unsup = q1(cur, "SELECT count(*) FROM normalized_events " + "WHERE fingerprint_id=%s AND superseded_by_event_id IS NULL", (fp,)) + sup = q1(cur, "SELECT count(*) FROM normalized_events " + "WHERE fingerprint_id=%s AND superseded_by_event_id IS NOT NULL", (fp,)) + return unsup, sup + + +def wait_fingerprint_quiet(fp: str, window_s: float = 3.0, timeout_s: float = 90.0) -> int: + """Wait until the fingerprint's normalized_events row count stops moving + and return the quiet total (unsuperseded + superseded). + + The manual path (G) creates its candidate within seconds of posting the + corpus, so the pre-approve snapshots can fire while posted events are + still in flight (collector -> kafka -> pipeline). Without this gate the + late rows land AFTER the approve-time sweep and parse in-flight under the + now-active rule, making the current view grow past the pre-approve + unsuperseded count (observed live in run B: total 50 -> 100). Gating the + snapshots on a quiet count makes the per-run deltas deterministic; assert + semantics are unchanged.""" + deadline = time.monotonic() + timeout_s + last = sum(supersede_counts(fp)) + last_change = time.monotonic() + while time.monotonic() < deadline: + time.sleep(POLL_INTERVAL_S) + current = sum(supersede_counts(fp)) + if current != last: + last, last_change = current, time.monotonic() + elif time.monotonic() - last_change >= window_s: + return last + return last + + +def sample_count(fp: str) -> int: + with psycopg.connect(DSN) as conn, conn.cursor() as cur: + return q1(cur, "SELECT count(*) FROM onboarding_samples WHERE fingerprint_id=%s", (fp,)) + + +def validate_local_rule(fp: str, rule_spec, lines: list[str]) -> object: + """Fast-fail discipline (as G): run the hand-authored rule through the SAME + deterministic gate the gateway enforces server-side, against the corpus + lines in the onboarding split shape (5 prompt / rest held-out).""" + pattern, mappings = rule_spec + rule = Rule(fingerprint_id=fp, version=1, pattern=pattern, provenance="human", + mappings=[Mapping(source_field=s, ocsf_path=o) for s, o in mappings]) + return validate_candidate(rule, lines[:5], lines[5:20]) + + +def post_manual_candidate(fp: str, rule_spec) -> tuple[int, dict | None]: + pattern, mappings = rule_spec + return http_post_json(f"{GATEWAY}/api/rules/manual", { + "fingerprint_id": fp, + "pattern": pattern, + "mappings": [{"source_field": s, "ocsf_path": o} for s, o in mappings], + "actor": "smoke", + }) + + +def acquire_candidate(fp: str, lines: list[str], rule_spec) -> tuple[dict | None, bool, str]: + """R-T12-pre-a candidate acquisition for F: poll pending_review for an SLM + candidate (ULPF_SMOKE_SLM_WAIT_S, default 300s); on timeout author one + deterministically — local validate_candidate fast-fail, then POST + /api/rules/manual -> 201 (manual creation is gate-passed by construction). + Returns (candidate, ok, acquisition_detail); candidate None on failure with + the reason in detail.""" + print(f" polling pending_review for an SLM candidate " + f"(timeout {SLM_WAIT_TIMEOUT_S}s; qwen3:4b on an 8GB host is expected " + f"to store 0 rules — the manual fallback is the ruled outcome)") + candidate = wait_pending_candidate(fp, SLM_WAIT_TIMEOUT_S) + if candidate is not None: + validation = candidate.get("validation") or {} + checks = validation.get("checks") or {} + ok = validation.get("passed") is True and bool(checks) and all(checks.values()) + detail = (f"SLM candidate id={candidate.get('id')} v{candidate.get('version')} " + f"validation.passed={validation.get('passed')} checks={checks}") + return candidate, ok, detail + + report = validate_local_rule(fp, rule_spec, lines) + if not report.passed: + return None, False, (f"SLM wait expired AND hand-authored rule failed the local " + f"gate: checks={report.checks} notes={report.notes[:3]}") + status, body = post_manual_candidate(fp, rule_spec) + print(f" SLM wait expired after {SLM_WAIT_TIMEOUT_S}s — recorded limitation of the " + f"default qwen3:4b tier (0 rules stored, 0.0 held-out confidence; see " + f"docs/m2-slm-sanity.md). Falling back to the hand-authored manual candidate.") + if status != 201 or not isinstance(body, dict) or "rule_id" not in body: + return None, False, (f"SLM wait expired AND POST /api/rules/manual -> {status}: " + f"{body!r}") + detail = (f"SLM wait expired after {SLM_WAIT_TIMEOUT_S}s (qwen3:4b stores 0 rules on " + f"this host — recorded limitation, docs/m2-slm-sanity.md): manual candidate " + f"id={body['rule_id']} v{body.get('version')} POST -> 201") + return {"id": body["rule_id"], "version": body.get("version")}, True, detail + +def wait_reparse_complete(fp: str, version: int, timeout_s: float) -> dict | None: + """Poll /api/audit until a reparse_complete row exists with + detail.version == the approved rule's version (R-T12-pre-d: the audit row + is the deterministic completion signal — a parsed==total poll cannot + signal completion on a re-run, because with a v1 rule active new lines + parse in-flight before the approval).""" + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + status, body = http_json(f"{GATEWAY}/api/audit?fingerprint={fp}&limit={AUDIT_PAGE}") + if status == 200 and isinstance(body, dict): + for row in body.get("audit", []): + detail = row.get("detail") or {} + if row.get("action") == "reparse_complete" and detail.get("version") == version: + return detail + time.sleep(POLL_INTERVAL_S) + return None + + +def audit_has(fp: str, action: str, want: dict) -> bool: + """True iff the fingerprint's audit trail has an `action` row whose detail + contains every key=value in want.""" + status, body = http_json(f"{GATEWAY}/api/audit?fingerprint={fp}&limit={AUDIT_PAGE}") + if status != 200 or not isinstance(body, dict): + return False + for row in body.get("audit", []): + detail = row.get("detail") or {} + if row.get("action") == action and all(detail.get(k) == v for k, v in want.items()): + return True + return False + + +def approve_and_reparse(fp: str, candidate: dict) -> tuple[bool, list[str], list[str]]: + """Approve the candidate and verify the R11 backlog sweep by per-run + deltas (R-T12-pre-d): superseded-count DELTA == the pre-approve + unsuperseded count, completion via the reparse_complete audit row for the + approved version, and the current view fully parsed at an unchanged total. + Returns (ok, detail_parts, problems).""" + problems: list[str] = [] + candidate_id = candidate.get("id") + + # Per-run deltas (R-T12-pre-d). The quiet gate first: posted backlog must + # have fully landed before the pre-approve snapshots (see the helper). + quiet_total = wait_fingerprint_quiet(fp) + + # Pre-approve snapshots (per-run deltas — the DB accumulates prior runs). + pre_fps = stats_by_fingerprint().get(fp, {}) + unsup_pre, sup_pre = supersede_counts(fp) + if unsup_pre + sup_pre != quiet_total: + problems.append(f"fingerprint row count moved during snapshot " + f"(quiet={quiet_total}, snapshot={unsup_pre + sup_pre})") + + status, body = http_post_json( + f"{GATEWAY}/api/rules/{fp}/candidates/{candidate_id}/approve", {"actor": "smoke"}) + if status != 200 or not isinstance(body, dict): + msg = f"approve POST -> {status}: {body!r}" + return False, [msg], [msg] + version = body.get("version") + parts = [f"approve -> 200 v{version}"] + + reparse = wait_reparse_complete(fp, version, REPARSE_TIMEOUT_S) + if reparse is None: + problems.append(f"no reparse_complete v{version} audit row within " + f"{REPARSE_TIMEOUT_S:.0f}s") + else: + parts.append(f"reparse_complete v{version} inserted={reparse.get('inserted')} " + f"superseded={reparse.get('superseded')}") + + sup_post = supersede_counts(fp)[1] + sup_delta = sup_post - sup_pre + parts.append(f"superseded delta {sup_pre}->{sup_post} (delta {sup_delta})") + if sup_delta != unsup_pre: + problems.append(f"superseded delta {sup_delta} != pre-approve unsuperseded " + f"{unsup_pre}") + if reparse is not None and (reparse.get("inserted") != unsup_pre + or reparse.get("superseded") != unsup_pre): + problems.append(f"reparse_complete counts {reparse.get('inserted')}/" + f"{reparse.get('superseded')} != {unsup_pre}") + + post_fp = stats_by_fingerprint().get(fp, {}) + parts.append(f"stats total {pre_fps.get('total')}->{post_fp.get('total')}, " + f"parsed -> {post_fp.get('parsed')}") + if post_fp.get("parsed") != post_fp.get("total"): + problems.append(f"current view not fully parsed: {post_fp}") + elif post_fp.get("total") != unsup_pre: + problems.append(f"current-view total {post_fp.get('total')} changed from the " + f"pre-approve unsuperseded count {unsup_pre}") + if not audit_has(fp, "rule_approved", {"rule_id": candidate_id, "version": version}): + problems.append(f"no rule_approved audit row for candidate {candidate_id} v{version}") + else: + parts.append("audit rule_approved present") + return not problems, parts, problems + + +def assert_hero_loop(check_name: str, label: str, fp: str, lines: list[str], + source_id: str, rule_spec, slm_poll: bool) -> None: + """F/G shared body: post the corpus x5, wait for onboarding samples, take a + pending_review candidate through approve -> backlog re-parse (F polls for + an SLM candidate first per R-T12-pre-a; G posts the hand-authored rule + directly — deterministic, no SLM).""" + print(f"== Assert {check_name}: {label} (fingerprint {fp}) ==") + accepted = post_corpus(lines, source_id) + total = wait_samples_total(fp, 20, SAMPLES_WAIT_TIMEOUT_S) + if total < 20: + check(check_name, False, f"onboarding samples for {fp} never reached 20 " + f"(posted={accepted}, last total={total})") + return + + if slm_poll: + candidate, ok, detail = acquire_candidate(fp, lines, rule_spec) + else: + report = validate_local_rule(fp, rule_spec, lines) + if not report.passed: + check(check_name, False, f"hand-authored rule failed the local gate: " + f"checks={report.checks} notes={report.notes[:3]}") + return + status, body = post_manual_candidate(fp, rule_spec) + ok = status == 201 and isinstance(body, dict) and "rule_id" in body + detail = (f"manual candidate id={body.get('rule_id')} v{body.get('version')} " + f"POST -> {status}" if ok else + f"POST /api/rules/manual -> {status}: {body!r}") + candidate = ({"id": body["rule_id"], "version": body.get("version")} + if ok else None) + if not ok: + check(check_name, False, detail) + return + + reparse_ok, parts, problems = approve_and_reparse(fp, candidate) + detail_line = f"{detail}; " + "; ".join(parts) + if not reparse_ok: + detail_line += "; PROBLEMS: " + "; ".join(problems) + check(check_name, reparse_ok, detail_line) + + +def assert_f_hero(lines: list[str], fp: str) -> None: + assert_hero_loop("F hero loop (newapp)", "SLM candidate -> approve -> backlog re-parse", + fp, lines, NEWAPP_SOURCE, NEWAPP_RULE, slm_poll=True) + + +def assert_g_manual(lines: list[str], zfp: str) -> None: + assert_hero_loop("G manual rule (zenwall)", + "hand-authored rule -> manual create -> approve -> re-parse", + zfp, lines, ZENWALL_SOURCE, ZENWALL_RULE, slm_poll=False) + + +def assert_h(fp: str, zfp: str) -> None: + print("== Assert H: replay stability with active rules (samples included) ==") + pre_raw = db_count("SELECT count(*) FROM raw_events") + pre_norm = db_count("SELECT count(*) FROM normalized_events") + pre_batches = db_count("SELECT count(*) FROM raw_batches") + pre_fps = stats_by_fingerprint() + pre_fp_samples = sample_count(fp) + pre_zfp_samples = sample_count(zfp) + + danced, dance_detail = replay_dance() + if not danced: + check("H replay stability", False, dance_detail) + return + post_batches = wait_batches_stable() + + post_raw = db_count("SELECT count(*) FROM raw_events") + post_norm = db_count("SELECT count(*) FROM normalized_events") + post_fps = stats_by_fingerprint() + post_fp_samples = sample_count(fp) + post_zfp_samples = sample_count(zfp) + + problems = [] + if post_raw != pre_raw: + problems.append(f"raw_events {pre_raw}->{post_raw}") + if post_norm != pre_norm: + problems.append(f"normalized_events {pre_norm}->{post_norm}") + if post_batches <= pre_batches: + problems.append(f"raw_batches did not grow: {pre_batches}->{post_batches} " + "(replay was vacuous)") + if post_fps != pre_fps: + problems.append("by_fingerprint changed:\n" + fingerprint_split_table(pre_fps, post_fps)) + if post_fp_samples != pre_fp_samples: + problems.append(f"samples[{fp}] {pre_fp_samples}->{post_fp_samples}") + if post_zfp_samples != pre_zfp_samples: + problems.append(f"samples[{zfp}] {pre_zfp_samples}->{post_zfp_samples}") + check("H replay stability", not problems, + f"raw {pre_raw} unchanged, normalized {pre_norm} unchanged, raw_batches " + f"{pre_batches}->{post_batches}, by_fingerprint unchanged, samples[{fp}]=" + f"{post_fp_samples} unchanged, samples[{zfp}]={post_zfp_samples} unchanged " + "(unique raw_id dedup live with active rules)" + + ("" if not problems else "; PROBLEMS: " + "; ".join(problems))) + + def wait_quiet(window_s: float = 4.0, timeout_s: float = 45.0) -> int: """Wait until raw_events stops moving, then return its count. @@ -446,7 +949,7 @@ def wait_quiet(window_s: float = 4.0, timeout_s: float = 45.0) -> int: def main() -> int: - print(f"ULPF M1 smoke - {time.strftime('%Y-%m-%d %H:%M:%S')}") + print(f"ULPF smoke (M1 A-E + M2 F-H) - {time.strftime('%Y-%m-%d %H:%M:%S')}") step1_up() print("== Snapshot before (delta base) ==") @@ -472,6 +975,19 @@ def main() -> int: assert_d_replay() assert_e_chain() + # M2 (Task 12): the onboarding hero loop. Fingerprints are computed + # host-side with the pipeline's own function; rules are approved through + # the gateway and the R11 backlog sweep is verified by per-run deltas. + newapp_lines = read_corpus_lines(NEWAPP_CORPUS) + fp = corpus_fingerprint(newapp_lines, "newapp") + assert_f_hero(newapp_lines, fp) + + zenwall_lines = ensure_zenwall_corpus() + zfp = corpus_fingerprint(zenwall_lines, "zenwall") + assert_g_manual(zenwall_lines, zfp) + + assert_h(fp, zfp) + failed = [name for name, ok, _ in results if not ok] print(f"\nSMOKE RESULT: {'PASS' if not failed else 'FAIL'} " f"({len(results) - len(failed)}/{len(results)} checks)" diff --git a/services/collector/app.py b/services/collector/app.py index ab14e6b..e048c9b 100644 --- a/services/collector/app.py +++ b/services/collector/app.py @@ -11,7 +11,7 @@ import uvicorn from fastapi import FastAPI, HTTPException -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, Field, ValidationError from ulpf_core.models import RawEnvelope from collector.producer import KafkaProducer @@ -106,6 +106,13 @@ async def handle_syslog_tcp(producer, reader: asyncio.StreamReader, writer: asyn producer.produce_raw(env) except (ConnectionError, asyncio.IncompleteReadError): pass # client went away mid-line; nothing to salvage + except ValueError: + # readline() past the StreamReader limit (run_syslog_tcp passes + # limit=MAX_LINE_CHARS): an oversized TCP line kills the connection — + # a byte stream has no line boundary to resync to. UDP, by contrast, + # drops only the one oversized datagram (README documents this). + log.warning("tcp %s: line exceeds %d char cap; closing connection", + source_id, MAX_LINE_CHARS) except Exception: log.exception("tcp %s: handler error", source_id) finally: @@ -115,9 +122,14 @@ async def handle_syslog_tcp(producer, reader: asyncio.StreamReader, writer: asyn async def run_syslog_tcp(producer, host: str = "0.0.0.0", port: int = SYSLOG_PORT): - """TCP line server; serves until cancelled.""" + """TCP line server; serves until cancelled. + + limit= caps the StreamReader at MAX_LINE_CHARS: readline() raises ValueError + on a longer line, which closes the connection (README: TCP kills the + connection where UDP drops just the event — the documented asymmetry).""" server = await asyncio.start_server( - lambda r, w: handle_syslog_tcp(producer, r, w), host, port + lambda r, w: handle_syslog_tcp(producer, r, w), host, port, + limit=MAX_LINE_CHARS, ) log.info("syslog tcp listening on %s:%d", host, port) async with server: @@ -126,8 +138,8 @@ async def run_syslog_tcp(producer, host: str = "0.0.0.0", port: int = SYSLOG_POR class IngestRequest(BaseModel): source_id: str - lines: list[str] - format_hint: str | None = None + lines: list[str] = Field(max_length=1000) # 1000 lines per request, max + format_hint: str | None = Field(default=None, max_length=64) def build_app(producer, udp: bool = True): diff --git a/services/collector/tests/test_collector.py b/services/collector/tests/test_collector.py index 3895576..bdc6ecd 100644 --- a/services/collector/tests/test_collector.py +++ b/services/collector/tests/test_collector.py @@ -29,6 +29,25 @@ def test_http_rejects_oversized_line(client): r = c.post("/v1/ingest", json={"source_id": "x", "lines": ["y" * 70000]}) assert r.status_code == 422 +def test_http_rejects_more_than_1000_lines(client): + c, _prod = client + r = c.post("/v1/ingest", json={"source_id": "x", "lines": ["ok"] * 1001}) + assert r.status_code == 422 + +def test_http_accepts_1000_lines_and_64_char_format_hint(client): + # Boundary: the caps themselves are accepted (IngestRequest Field limits). + c, prod = client + r = c.post("/v1/ingest", json={"source_id": "x", "lines": ["a"] * 1000, + "format_hint": "f" * 64}) + assert r.status_code == 202 and r.json() == {"accepted": 1000} + assert prod.produced[0]["format_hint"] == "f" * 64 + +def test_http_rejects_oversized_format_hint(client): + c, _prod = client + r = c.post("/v1/ingest", json={"source_id": "x", "lines": ["ok"], + "format_hint": "f" * 65}) + assert r.status_code == 422 + def test_syslog_handler_wraps_datagram(): from collector.app import make_envelope env = make_envelope("syslog-udp", "fw01", "Aug 27 14:32:07 fw01 kernel: X") diff --git a/services/gateway/Dockerfile b/services/gateway/Dockerfile index 14d7ad5..9d63693 100644 --- a/services/gateway/Dockerfile +++ b/services/gateway/Dockerfile @@ -1,17 +1,27 @@ # services/gateway/Dockerfile — FastAPI gateway: stats, current-view events, -# raw trace, SSE, chain head. Read-only DB access as gateway_role (SELECT only). +# raw trace, SSE, chain head (read-only DB access as gateway_role), plus the +# M2 rule-lifecycle write path (writes.py, rules_role over WRITE_DATABASE_URL). # Build context is the repo root (compose build.context: ..; see /.dockerignore). FROM python:3.12-slim WORKDIR /app -# No ulpf-core/ocsf-schema dependency: the gateway speaks plain SQL via psycopg. +# BOTH libs first: writes.py revalidates candidates via ulpf_core.validation +# (Task 3 gate), whose parse() FAILS CLOSED without ocsf_schema's validator — +# the write path must not ship without it. Then deps (layer caching), then the +# service package. /app contains gateway/ and libs/ so `python -m gateway.app` +# imports cleanly. +COPY libs/ulpf-core/pyproject.toml /app/libs/ulpf-core/pyproject.toml +COPY libs/ulpf-core/ulpf_core/ /app/libs/ulpf-core/ulpf_core/ +COPY libs/ocsf-schema/pyproject.toml /app/libs/ocsf-schema/pyproject.toml +COPY libs/ocsf-schema/ocsf_schema/ /app/libs/ocsf-schema/ocsf_schema/ COPY services/gateway/requirements.txt /tmp/requirements.txt -RUN pip install --no-cache-dir -r /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt \ + && pip install --no-cache-dir -e /app/libs/ulpf-core -e /app/libs/ocsf-schema -# /app contains gateway/ so `python -m gateway.app` imports cleanly. COPY services/gateway/__init__.py /app/gateway/__init__.py COPY services/gateway/app.py /app/gateway/app.py COPY services/gateway/queries.py /app/gateway/queries.py +COPY services/gateway/writes.py /app/gateway/writes.py CMD ["python", "-m", "gateway.app"] diff --git a/services/gateway/app.py b/services/gateway/app.py index 73eb813..374ca99 100644 --- a/services/gateway/app.py +++ b/services/gateway/app.py @@ -19,8 +19,9 @@ import uvicorn from fastapi import FastAPI, HTTPException, Query from fastapi.responses import StreamingResponse +from pydantic import BaseModel -from gateway import queries +from gateway import queries, writes log = logging.getLogger("gateway") @@ -65,10 +66,57 @@ async def sse_events(queries_module, poll_seconds, heartbeat_seconds, max_polls) await asyncio.sleep(poll_seconds) -def build_app(queries_module=queries, poll_seconds=SSE_POLL_SECONDS, - heartbeat_seconds=SSE_HEARTBEAT_SECONDS, max_polls=None): - """FastAPI app factory; the queries module is injected (tests monkeypatch - gateway.queries attributes — routes resolve them at request time).""" +# --- M2 write path + review surfaces (Task 7; additive to the frozen M1 API) -- +# Request bodies (actor defaults to "anonymous" — no auth in MVP). + +class MappingBody(BaseModel): + source_field: str + ocsf_path: str + + +class OverrideBody(BaseModel): + pattern: str + mappings: list[MappingBody] + + +class ApproveBody(BaseModel): + actor: str = "anonymous" + reason: str | None = None + edited_mappings: list[MappingBody] | None = None + override: OverrideBody | None = None + + +class RejectBody(BaseModel): + actor: str = "anonymous" + reason: str | None = None + + +class ManualBody(BaseModel): + fingerprint_id: str + pattern: str + mappings: list[MappingBody] + actor: str = "anonymous" + confidence: float | None = None + + +def _write_http_error(exc: writes.WriteError) -> HTTPException: + """writes.* typed errors -> the API contract: 404 unknown, 409 conflict, + 422 failed validation carrying the gate's checks + notes (the Review UI + renders them).""" + if isinstance(exc, writes.ValidationError): + return HTTPException(status_code=422, + detail={"checks": exc.checks, "notes": exc.notes}) + if isinstance(exc, writes.ConflictError): + return HTTPException(status_code=409, detail=str(exc)) + return HTTPException(status_code=404, detail=str(exc)) # NotFoundError + + +def build_app(queries_module=queries, writes_module=writes, + poll_seconds=SSE_POLL_SECONDS, heartbeat_seconds=SSE_HEARTBEAT_SECONDS, + max_polls=None): + """FastAPI app factory; the queries/writes modules are injected (tests + monkeypatch gateway.queries / gateway.writes attributes — routes resolve + them at request time).""" app = FastAPI(title="ulpf-gateway", version="0.1.0") @@ -107,6 +155,88 @@ async def stream_events(): headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, ) + # ---------- M2: rules, samples status, audit; rule lifecycle ---------- + + @app.get("/api/rules") + async def list_rules(status: str | None = None): + return {"rules": await asyncio.to_thread(queries_module.fetch_rules, status)} + + @app.get("/api/rules/{fingerprint_id}") + async def rule_history(fingerprint_id: str): + return await asyncio.to_thread( + queries_module.fetch_rule_history, fingerprint_id) + + @app.get("/api/onboarding/samples") + async def samples_status(fingerprint: str | None = None): + rows = await asyncio.to_thread(queries_module.fetch_samples_status, fingerprint) + if fingerprint is not None: + return rows[0] # single-fingerprint form; fetch_* zero-fills unknown fps + return {"samples": rows} + + @app.get("/api/audit") + async def audit_trail( + fingerprint: str | None = None, + limit: int = Query(default=queries.DEFAULT_LIMIT, ge=1, le=queries.MAX_LIMIT), + ): + return {"audit": await asyncio.to_thread( + queries_module.fetch_audit, fingerprint, limit)} + + @app.post("/api/rules/manual", status_code=201) + async def create_manual(body: ManualBody): + # Two-step manual authoring: this stores a validated pending_review + # candidate; a human approves it through the SAME endpoint as SLM + # candidates (one activation path, uniform audit). + try: + return await asyncio.to_thread( + writes_module.create_manual_candidate, body.fingerprint_id, + body.pattern, [m.model_dump() for m in body.mappings], + actor=body.actor, confidence=body.confidence) + except writes.WriteError as exc: + raise _write_http_error(exc) from exc + + @app.post("/api/rules/{fingerprint_id}/candidates/{candidate_id}/approve") + async def approve(fingerprint_id: str, candidate_id: int, body: ApproveBody): + try: + return await asyncio.to_thread( + writes_module.approve_candidate, fingerprint_id, candidate_id, + actor=body.actor, reason=body.reason, + edited_mappings=None if body.edited_mappings is None + else [m.model_dump() for m in body.edited_mappings], + override=None if body.override is None else body.override.model_dump()) + except writes.WriteError as exc: + raise _write_http_error(exc) from exc + + @app.post("/api/rules/{fingerprint_id}/candidates/{candidate_id}/reject") + async def reject(fingerprint_id: str, candidate_id: int, + body: RejectBody | None = None): + body = body or RejectBody() # actor/reason are optional in the contract + try: + return await asyncio.to_thread( + writes_module.reject_candidate, fingerprint_id, candidate_id, + actor=body.actor, reason=body.reason) + except writes.WriteError as exc: + raise _write_http_error(exc) from exc + + @app.post("/api/rules/{fingerprint_id}/deactivate") + async def deactivate(fingerprint_id: str, body: RejectBody | None = None): + body = body or RejectBody() + try: + return await asyncio.to_thread( + writes_module.deactivate_rule, fingerprint_id, + actor=body.actor, reason=body.reason) + except writes.WriteError as exc: + raise _write_http_error(exc) from exc + + @app.post("/api/rules/{rule_id}/reactivate") + async def reactivate(rule_id: int, body: RejectBody | None = None): + body = body or RejectBody() + try: + return await asyncio.to_thread( + writes_module.reactivate_rule, rule_id, + actor=body.actor, reason=body.reason) + except writes.WriteError as exc: + raise _write_http_error(exc) from exc + return app diff --git a/services/gateway/queries.py b/services/gateway/queries.py index 81af2d9..8420052 100644 --- a/services/gateway/queries.py +++ b/services/gateway/queries.py @@ -2,7 +2,9 @@ # # gateway_role is SELECT-only (Task 5 DDL), and every value goes through %s # placeholders — parameterized SQL only; nothing user-controlled is ever -# interpolated into a statement string. psycopg is imported lazily inside +# interpolated into a statement string. The M2 write path lives in writes.py +# (rules_role over WRITE_DATABASE_URL); this module stays read-only over the +# M1 read DSN. psycopg is imported lazily inside # _connect() so unit tests (which monkeypatch these functions) need neither a # DB nor the driver on the test path (mirrors services/pipeline/db.py). # @@ -39,9 +41,13 @@ def clamp_limit(limit: int) -> int: def zero_fill_statuses(rows) -> dict[str, int]: """by_status GROUP BY rows (dict_row dicts: {"status": ..., "n": ...}) -> - contract dict with all four keys, zero-filled.""" + contract dict with all four keys, zero-filled. A status key outside + _STATUS_KEYS is a contract breach, not a shape to widen — raise (T9).""" out = {status: 0 for status in _STATUS_KEYS} for row in rows: + if row["status"] not in out: + raise ValueError( + f"unexpected status {row['status']!r}; expected one of {_STATUS_KEYS}") out[row["status"]] = row["n"] return out @@ -71,7 +77,7 @@ def fetch_stats() -> dict: "AND parsed_at >= now() - interval '1 minute'" ) events_last_minute = cur.fetchone()["n"] - return { + out = { "raw_total": raw_total, "by_status": by_status, "by_fingerprint": by_fingerprint, @@ -80,6 +86,11 @@ def fetch_stats() -> dict: "note": "dlq_total is not counted in M1 (needs broker-side inspection of " "pipeline.dlq); always null for M1", } + if len(by_fingerprint) >= MAX_LIMIT: + # T9: the hard cap truncates by_fingerprint — say so. Key stays ABSENT + # below the cap (additive; M1 bodies are unchanged when not truncated). + out["by_fingerprint_truncated"] = True + return out def fetch_events(status=None, fingerprint=None, limit=DEFAULT_LIMIT, before=None): @@ -149,3 +160,88 @@ def fetch_chain_heads() -> list[dict]: "FROM raw_batches ORDER BY partition_id, batch_seq DESC" ) return cur.fetchall() + + +# --- M2 review surfaces (Task 7; consumed by the web Review Queue / Rules) ----- + +_RULE_COLUMNS = ( + "id, fingerprint_id, version, pattern, mappings, provenance, confidence, " + "status, created_by, created_at, activated_at, deactivated_at, validation" +) + + +def fetch_rules(status=None): + """GET /api/rules rows: RuleRow columns only; every status unless filtered + (controller ruling), newest version first per fingerprint.""" + with _connect() as conn, conn.cursor() as cur: + cur.execute( + f"SELECT {_RULE_COLUMNS} FROM rules " + "WHERE (%s::text IS NULL OR status = %s) " + "ORDER BY fingerprint_id, version DESC", + (status, status), + ) + return cur.fetchall() + + +def fetch_rule_history(fingerprint_id: str) -> dict: + """GET /api/rules/{fp} body: every version of the fingerprint (newest + first) plus its audit trail (entity = fingerprint_id, latest 200).""" + with _connect() as conn, conn.cursor() as cur: + cur.execute( + f"SELECT {_RULE_COLUMNS} FROM rules " + "WHERE fingerprint_id = %s ORDER BY version DESC", + (fingerprint_id,), + ) + rules = cur.fetchall() + cur.execute( + "SELECT id, ts, actor, action, entity, detail FROM audit_log " + "WHERE entity = %s ORDER BY id DESC LIMIT 200", + (fingerprint_id,), + ) + audit = cur.fetchall() + return {"rules": rules, "audit": audit} + + +def fetch_samples_status(fingerprint=None): + """GET /api/onboarding/samples rows: per-fingerprint sample accounting — + total, per-role counts (zero-filled), latest capture. Omitted fingerprint + -> one row per fingerprint; given -> the single row (zero-filled when the + fingerprint has no samples yet).""" + with _connect() as conn, conn.cursor() as cur: + cur.execute( + "SELECT fingerprint_id, count(*) AS total, " + "count(*) FILTER (WHERE role = 'prompt') AS prompt, " + "count(*) FILTER (WHERE role = 'held_out') AS held_out, " + "count(*) FILTER (WHERE role = 'unused') AS unused, " + "max(captured_at) AS latest_captured_at " + "FROM onboarding_samples " + "WHERE (%s::text IS NULL OR fingerprint_id = %s) " + "GROUP BY fingerprint_id ORDER BY fingerprint_id", + (fingerprint, fingerprint), + ) + rows = cur.fetchall() + if fingerprint is not None and not rows: + return [{"fingerprint_id": fingerprint, "total": 0, + "by_role": {"prompt": 0, "held_out": 0, "unused": 0}, + "latest_captured_at": None}] + return [{ + "fingerprint_id": row["fingerprint_id"], + "total": row["total"], + "by_role": {"prompt": row["prompt"] or 0, "held_out": row["held_out"] or 0, + "unused": row["unused"] or 0}, + "latest_captured_at": row["latest_captured_at"], + } for row in rows] + + +def fetch_audit(fingerprint=None, limit=DEFAULT_LIMIT): + """GET /api/audit rows: the audit trail, latest first, optionally scoped to + one fingerprint (entity = the fingerprint_id).""" + limit = clamp_limit(limit) + with _connect() as conn, conn.cursor() as cur: + cur.execute( + "SELECT id, ts, actor, action, entity, detail FROM audit_log " + "WHERE (%s::text IS NULL OR entity = %s) " + "ORDER BY id DESC LIMIT %s", + (fingerprint, fingerprint, limit), + ) + return cur.fetchall() diff --git a/services/gateway/tests/test_gateway.py b/services/gateway/tests/test_gateway.py index a55372f..66b13b0 100644 --- a/services/gateway/tests/test_gateway.py +++ b/services/gateway/tests/test_gateway.py @@ -270,3 +270,441 @@ def test_r9_current_view_predicate_in_every_normalized_query(queries): for fn_name in ("fetch_stats", "fetch_events", "fetch_raw_trace", "poll_events"): src = inspect.getsource(getattr(queries, fn_name)) assert "superseded_by_event_id IS NULL" in src, f"{fn_name} missing R9 predicate" + + +# ================= M2 (Task 7): rules/samples/audit reads + write path ======= +# Everything below is ADDITIVE — the frozen M1 contract above is untouched. +# Same monkeypatch pattern: new GETs return the queries-module fixtures +# verbatim; POSTs forward to gateway.writes (asserted to run off the event +# loop in a worker thread). The T9 fixes pin fetch_stats' truncation flag and +# zero_fill_statuses' strictness through a fake _connect. + +RULE_ROW_KEYS = { + "id", "fingerprint_id", "version", "pattern", "mappings", "provenance", "confidence", + "status", "created_by", "created_at", "activated_at", "deactivated_at", "validation", +} +AUDIT_ROW_KEYS = {"id", "ts", "actor", "action", "entity", "detail"} + + +def rule_row(**overrides) -> dict: + """One RuleRow as psycopg dict_row returns it (JSONB loaded, real datetimes).""" + row = { + "id": 7, + "fingerprint_id": "fp_ssh_denied", + "version": 3, + "pattern": r"^.*?DENIED\s+(?P.*)$", + "mappings": [{"source_field": "SRC", "ocsf_path": "src_endpoint.ip"}], + "provenance": "slm", + "confidence": 0.93, + "status": "active", + "created_by": "onboarding", + "created_at": datetime(2026, 9, 19, 9, 0, 0, tzinfo=UTC), + "activated_at": datetime(2026, 9, 19, 9, 5, 0, tzinfo=UTC), + "deactivated_at": None, + "validation": {"passed": True, "checks": {"caps_and_allowlist": True}, + "held_out_match_rate": 1.0, "notes": [], "previews": [], + "prompt_count": 5, "held_out_count": 15}, + } + row.update(overrides) + return row + + +def audit_row(**overrides) -> dict: + row = { + "id": 41, + "ts": datetime(2026, 9, 19, 9, 5, 0, tzinfo=UTC), + "actor": "amy", + "action": "rule_approved", + "entity": "fp_ssh_denied", + "detail": {"rule_id": 7, "version": 3, "actor": "amy"}, + } + row.update(overrides) + return row + + +class _FakeRowsConn: + """Fake dict_row connection for the read helpers: records SQL, replays + scripted fetch results FIFO (consumed per fetchone / fetchall call).""" + + def __init__(self, fetchone=(), fetchall=()): + self.sql = [] + self.params = [] + self._fetchone = [dict(r) for r in fetchone] + self._fetchall = [[dict(r) for r in batch] for batch in fetchall] + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def cursor(self): + conn = self + + class Cur: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params=None): + conn.sql.append(sql) + conn.params.append(params) + + def fetchone(self): + return conn._fetchone.pop(0) + + def fetchall(self): + return conn._fetchall.pop(0) + + return Cur() + + +def _raises(exc): + def _fn(*args, **kwargs): + raise exc + + return _fn + + +# ---------- GET /api/rules + GET /api/rules/{fp} ---------- + +def test_rules_list_shape(client, queries, monkeypatch): + rows = [rule_row(), rule_row(id=8, version=4, status="pending_review", + activated_at=None)] + monkeypatch.setattr(queries, "fetch_rules", lambda status=None: rows) + r = client.get("/api/rules") + assert r.status_code == 200 + assert r.json() == {"rules": [jsonable(row) for row in rows]} + assert set(r.json()["rules"][0]) == RULE_ROW_KEYS # exact RuleRow keys + + +def test_rules_status_filter_forwarded(client, queries, monkeypatch): + seen = {} + + def fake(status=None): + seen["status"] = status + return [] + + monkeypatch.setattr(queries, "fetch_rules", fake) + assert client.get("/api/rules", params={"status": "pending_review"}).status_code == 200 + assert seen["status"] == "pending_review" + assert client.get("/api/rules").status_code == 200 # no filter -> all statuses + assert seen["status"] is None + + +def test_rule_history_shape(client, queries, monkeypatch): + body = {"rules": [rule_row(version=3), rule_row(version=2, status="superseded")], + "audit": [audit_row()]} + monkeypatch.setattr(queries, "fetch_rule_history", lambda fp: body) + r = client.get("/api/rules/fp_ssh_denied") + assert r.status_code == 200 + assert r.json() == { + "rules": [jsonable(row) for row in body["rules"]], + "audit": [jsonable(row) for row in body["audit"]], + } + + +# ---------- GET /api/onboarding/samples ---------- + +def test_samples_status_single_fingerprint_form(client, queries, monkeypatch): + row = {"fingerprint_id": "fp_ssh_denied", "total": 20, + "by_role": {"prompt": 5, "held_out": 15, "unused": 0}, + "latest_captured_at": datetime(2026, 9, 19, 10, 0, tzinfo=UTC)} + seen = {} + + def fake(fingerprint=None): + seen["fingerprint"] = fingerprint + return [row] + + monkeypatch.setattr(queries, "fetch_samples_status", fake) + r = client.get("/api/onboarding/samples", params={"fingerprint": "fp_ssh_denied"}) + assert r.status_code == 200 + assert r.json() == jsonable(row) # fp given -> the single-fingerprint shape + assert seen["fingerprint"] == "fp_ssh_denied" + + +def test_samples_status_all_fingerprints_form(client, queries, monkeypatch): + rows = [{"fingerprint_id": "fp_a", "total": 20, + "by_role": {"prompt": 5, "held_out": 15, "unused": 0}, + "latest_captured_at": None}] + monkeypatch.setattr(queries, "fetch_samples_status", lambda fingerprint=None: rows) + r = client.get("/api/onboarding/samples") + assert r.status_code == 200 + assert r.json() == {"samples": rows} # omitted -> that shape per fingerprint + + +# ---------- GET /api/audit ---------- + +def test_audit_endpoint_forwards_filters(client, queries, monkeypatch): + rows = [audit_row()] + seen = {} + + def fake(fingerprint=None, limit=50): + seen.update(fingerprint=fingerprint, limit=limit) + return rows + + monkeypatch.setattr(queries, "fetch_audit", fake) + r = client.get("/api/audit", params={"fingerprint": "fp_ssh_denied", "limit": 10}) + assert r.status_code == 200 + assert r.json() == {"audit": [jsonable(row) for row in rows]} + assert set(r.json()["audit"][0]) == AUDIT_ROW_KEYS + assert seen == {"fingerprint": "fp_ssh_denied", "limit": 10} + + +def test_audit_limit_bounds_rejected(client): + assert client.get("/api/audit", params={"limit": 501}).status_code == 422 + assert client.get("/api/audit", params={"limit": 0}).status_code == 422 + + +# ---------- read helpers over a fake conn (real SQL pinned) ---------- + +def test_fetch_rules_filters_status_and_orders(queries, monkeypatch): + conn = _FakeRowsConn(fetchall=[[rule_row()]]) + monkeypatch.setattr(queries, "_connect", lambda: conn) + assert queries.fetch_rules("pending_review") == [rule_row()] + assert "WHERE (%s::text IS NULL OR status = %s)" in conn.sql[0] + assert "ORDER BY fingerprint_id, version DESC" in conn.sql[0] + assert conn.params[0] == ("pending_review", "pending_review") + + +def test_fetch_rule_history_returns_rules_and_scoped_audit(queries, monkeypatch): + conn = _FakeRowsConn(fetchall=[ + [rule_row(version=3), rule_row(version=2, status="superseded")], + [audit_row()], + ]) + monkeypatch.setattr(queries, "_connect", lambda: conn) + body = queries.fetch_rule_history("fp_ssh_denied") + assert body["rules"][0]["version"] == 3 # versions newest-first + assert body["audit"] == [audit_row()] + assert "ORDER BY version DESC" in conn.sql[0] + assert "FROM audit_log" in conn.sql[1] and "ORDER BY id DESC LIMIT 200" in conn.sql[1] + assert conn.params[1] == ("fp_ssh_denied",) + + +def test_fetch_samples_status_groups_by_fingerprint_and_role(queries, monkeypatch): + latest = datetime(2026, 9, 19, 10, 0, tzinfo=UTC) + conn = _FakeRowsConn(fetchall=[[ + {"fingerprint_id": "fp_a", "total": 20, "prompt": 5, "held_out": 15, + "unused": 0, "latest_captured_at": latest}, + ]]) + monkeypatch.setattr(queries, "_connect", lambda: conn) + assert queries.fetch_samples_status() == [{ + "fingerprint_id": "fp_a", "total": 20, + "by_role": {"prompt": 5, "held_out": 15, "unused": 0}, + "latest_captured_at": latest, + }] + assert "GROUP BY fingerprint_id" in conn.sql[0] + assert conn.params[0] == (None, None) + + +def test_fetch_samples_status_zero_fills_unknown_fingerprint(queries, monkeypatch): + conn = _FakeRowsConn(fetchall=[[]]) + monkeypatch.setattr(queries, "_connect", lambda: conn) + assert queries.fetch_samples_status("fp_none") == [{ + "fingerprint_id": "fp_none", "total": 0, + "by_role": {"prompt": 0, "held_out": 0, "unused": 0}, + "latest_captured_at": None, + }] + assert conn.params[0] == ("fp_none", "fp_none") + + +def test_fetch_audit_scopes_entity_and_clamps_limit(queries, monkeypatch): + conn = _FakeRowsConn(fetchall=[[audit_row()]]) + monkeypatch.setattr(queries, "_connect", lambda: conn) + assert queries.fetch_audit(fingerprint="fp_ssh_denied", limit=10**9) == [audit_row()] + assert "FROM audit_log" in conn.sql[0] and "ORDER BY id DESC LIMIT %s" in conn.sql[0] + assert conn.params[0] == ("fp_ssh_denied", "fp_ssh_denied", 500) + + +# ---------- T9 fixes ---------- + +def test_fetch_stats_flags_by_fingerprint_truncation_at_cap(queries, monkeypatch): + conn = _FakeRowsConn( + fetchone=[{"n": 1}, {"n": 0}], + fetchall=[[], [{"fingerprint_id": f"fp_{i}", "total": 1, "parsed": 1} + for i in range(queries.MAX_LIMIT)]], + ) + monkeypatch.setattr(queries, "_connect", lambda: conn) + out = queries.fetch_stats() + assert out["by_fingerprint_truncated"] is True # LIMIT cap hit -> say so + + +def test_fetch_stats_omits_truncation_key_below_cap(queries, monkeypatch): + conn = _FakeRowsConn( + fetchone=[{"n": 1}, {"n": 0}], + fetchall=[[], [{"fingerprint_id": f"fp_{i}", "total": 1, "parsed": 1} + for i in range(queries.MAX_LIMIT - 1)]], + ) + monkeypatch.setattr(queries, "_connect", lambda: conn) + assert "by_fingerprint_truncated" not in queries.fetch_stats() # additive only + + +def test_zero_fill_statuses_rejects_unknown_status_key(queries): + with pytest.raises(ValueError): + queries.zero_fill_statuses([{"status": "weird", "n": 1}]) + # the known four still zero-fill + assert queries.zero_fill_statuses([{"status": "parsed", "n": 2}])["parsed"] == 2 + + +# ---------- POST write endpoints (forward to gateway.writes) ---------- + +def test_approve_endpoint_forwards_and_runs_in_worker_thread(client, monkeypatch): + import threading + + from gateway import writes + + seen = {} + + def fake(fp, cid, **kwargs): + seen["fp"], seen["cid"], seen["kwargs"] = fp, cid, kwargs + seen["main_thread"] = threading.current_thread() is threading.main_thread() + return {"rule_id": 9, "version": 3, "status": "active"} + + monkeypatch.setattr(writes, "approve_candidate", fake) + r = client.post("/api/rules/fp_ssh_denied/candidates/9/approve", + json={"actor": "amy", "reason": "checked held-out"}) + assert r.status_code == 200 + assert r.json() == {"rule_id": 9, "version": 3, "status": "active"} + assert (seen["fp"], seen["cid"]) == ("fp_ssh_denied", 9) + assert seen["kwargs"]["actor"] == "amy" + assert seen["kwargs"]["reason"] == "checked held-out" + assert seen["main_thread"] is False # asyncio.to_thread, not the event loop + + +def test_approve_edited_mappings_forwarded_as_dicts(client, monkeypatch): + from gateway import writes + + seen = {} + + def fake(fp, cid, **kwargs): + seen.update(kwargs) + return {"rule_id": 10, "version": 4, "status": "active"} + + monkeypatch.setattr(writes, "approve_candidate", fake) + r = client.post("/api/rules/fp/candidates/9/approve", json={ + "actor": "amy", + "edited_mappings": [{"source_field": "SRC", "ocsf_path": "src_endpoint.ip"}], + }) + assert r.status_code == 200 + assert seen["edited_mappings"] == [{"source_field": "SRC", + "ocsf_path": "src_endpoint.ip"}] + assert seen["override"] is None + + +def test_approve_error_mapping_404_and_409(client, monkeypatch): + from gateway import writes + + monkeypatch.setattr(writes, "approve_candidate", _raises(writes.NotFoundError("gone"))) + assert client.post("/api/rules/fp/candidates/9/approve", json={}).status_code == 404 + + monkeypatch.setattr(writes, "approve_candidate", + _raises(writes.ConflictError("not pending"))) + assert client.post("/api/rules/fp/candidates/9/approve", json={}).status_code == 409 + + +def _gate_failure_report(): + from ulpf_core.validation import CandidateReport + + return CandidateReport( + passed=False, + checks={"caps_and_allowlist": False, "samples_parse": False, + "held_out_match_all": False, "ip_fields_valid": True, + "port_fields_valid": True, "no_orphan_mappings": True, + "adversarial_probe": True, "no_hardcoded_literals": True}, + held_out_match_rate=0.0, + notes=["compile error: boom"], + ) + + +def test_manual_422_carries_gate_checks_and_notes(client, monkeypatch): + from gateway import writes + + report = _gate_failure_report() + monkeypatch.setattr(writes, "create_manual_candidate", + _raises(writes.ValidationError(report))) + r = client.post("/api/rules/manual", json={ + "fingerprint_id": "fp_a", "pattern": "^(", "mappings": [], "actor": "amy"}) + assert r.status_code == 422 + # FastAPI HTTPException detail: the gate's checks/notes, faithful for the UI + assert r.json()["detail"]["checks"] == report.checks + assert r.json()["detail"]["notes"] == report.notes + + +def test_approve_revalidation_failure_maps_to_422(client, monkeypatch): + from gateway import writes + + report = _gate_failure_report() + monkeypatch.setattr(writes, "approve_candidate", _raises(writes.ValidationError(report))) + r = client.post("/api/rules/fp/candidates/9/approve", json={ + "edited_mappings": [{"source_field": "SRC", "ocsf_path": "bogus.path"}]}) + assert r.status_code == 422 + assert r.json()["detail"]["notes"] == report.notes + + +def test_manual_endpoint_201_and_forwards(client, monkeypatch): + from gateway import writes + + seen = {} + + def fake(fp, pattern, mappings, **kwargs): + seen.update(fp=fp, pattern=pattern, mappings=mappings, **kwargs) + return {"rule_id": 12, "version": 1, "status": "pending_review"} + + monkeypatch.setattr(writes, "create_manual_candidate", fake) + r = client.post("/api/rules/manual", json={ + "fingerprint_id": "fp_new", "pattern": "^.*?(?P.*)$", + "mappings": [{"source_field": "msg", "ocsf_path": "message"}], + "actor": "amy", "confidence": 0.5}) + assert r.status_code == 201 + assert r.json() == {"rule_id": 12, "version": 1, "status": "pending_review"} + assert seen["fp"] == "fp_new" + assert seen["mappings"] == [{"source_field": "msg", "ocsf_path": "message"}] + assert seen["actor"] == "amy" and seen["confidence"] == 0.5 + + +def test_reject_endpoint_forwards(client, monkeypatch): + from gateway import writes + + seen = {} + + def fake(fp, cid, **kwargs): + seen.update(fp=fp, cid=cid, **kwargs) + return {"status": "rejected"} + + monkeypatch.setattr(writes, "reject_candidate", fake) + r = client.post("/api/rules/fp/candidates/9/reject", + json={"actor": "amy", "reason": "bad"}) + assert r.status_code == 200 + assert r.json() == {"status": "rejected"} + assert (seen["fp"], seen["cid"]) == ("fp", 9) + assert seen["actor"] == "amy" and seen["reason"] == "bad" + + +def test_deactivate_and_reactivate_endpoints_forward(client, monkeypatch): + from gateway import writes + + seen = {} + + def fake_deactivate(fp, **kwargs): + seen["deactivate"] = (fp, kwargs) + return {"rule_id": 5, "version": 2, "status": "deactivated"} + + def fake_reactivate(rule_id, **kwargs): + seen["reactivate"] = (rule_id, kwargs) + return {"rule_id": 5, "version": 2, "status": "active"} + + monkeypatch.setattr(writes, "deactivate_rule", fake_deactivate) + monkeypatch.setattr(writes, "reactivate_rule", fake_reactivate) + + r = client.post("/api/rules/fp_ssh_denied/deactivate", json={"actor": "amy"}) + assert r.status_code == 200 + assert r.json() == {"rule_id": 5, "version": 2, "status": "deactivated"} + assert seen["deactivate"] == ("fp_ssh_denied", {"actor": "amy", "reason": None}) + + r = client.post("/api/rules/5/reactivate") # empty body allowed + assert r.status_code == 200 + assert r.json() == {"rule_id": 5, "version": 2, "status": "active"} + assert seen["reactivate"][0] == 5 + assert seen["reactivate"][1]["actor"] == "anonymous" # default actor diff --git a/services/gateway/tests/test_writes.py b/services/gateway/tests/test_writes.py new file mode 100644 index 0000000..3abb5ea --- /dev/null +++ b/services/gateway/tests/test_writes.py @@ -0,0 +1,504 @@ +# services/gateway/tests/test_writes.py — the gateway write path (Task 7), +# pinned through a recording fake conn (the onboarding test_store style; NEVER +# a live DB). +# +# The fake records (sql, params) in execution order and brackets each +# `with conn.transaction():` block with TXN-START / TXN-COMMIT (or +# TXN-ROLLBACK) markers, so "supersede active -> activate -> audit, one txn" +# is an assertion on the recorded sequence. Fetch results are consumed FIFO; +# `raise_on` scripts one-shot exceptions (the IntegrityError retry); `rowcounts` +# scripts cur.rowcount for MUTATION statements FIFO (default 1) — the TOCTOU +# predicate contract. psycopg appears ONLY as its exception class (the retry +# contract); Json wrappers are unwrapped duck-typed via .obj. +# +# The revalidation fixtures drive the REAL Task 3 gate (validate_candidate) — +# the same ACME rule/lines that pass in libs/ulpf-core's own suite. +import pytest +from psycopg import IntegrityError + +from gateway import writes + +ACME_PROMPT = ("ACMEGW fw01 2026-09-19T10:00:03Z DROP IN=eth0 OUT= SRC=198.51.100.4 " + "DST=203.0.113.9 SPT=52114 DPT=8443") +ACME_HELD_OUT = ("ACMEGW fw02 2026-09-19T10:00:11Z ACCEPT IN=eth1 OUT= SRC=198.51.100.9 " + "DST=203.0.113.4 SPT=49300 DPT=443") +PATTERN = r"^.*?(?:DROP|ACCEPT)\s+(?P.*)$" +MAPPINGS = [ + {"source_field": "SRC", "ocsf_path": "src_endpoint.ip"}, + {"source_field": "DST", "ocsf_path": "dst_endpoint.ip"}, + {"source_field": "SPT", "ocsf_path": "src_endpoint.port"}, + {"source_field": "DPT", "ocsf_path": "dst_endpoint.port"}, +] +SAMPLE_ROWS = [{"raw_text": ACME_PROMPT, "role": "prompt"}, + {"raw_text": ACME_HELD_OUT, "role": "held_out"}] + + +def candidate_row(**overrides) -> dict: + """One pending candidate as psycopg dict_row returns it.""" + row = { + "id": 9, + "fingerprint_id": "fp_a", + "version": 2, + "status": "pending_review", + "pattern": PATTERN, + "mappings": MAPPINGS, + "provenance": "slm", + "confidence": 0.87, + } + row.update(overrides) + return row + + +class FakeConn: + """Fake psycopg dict_row connection: records SQL, replays scripted fetch + results, raises scripted one-shot exceptions on matching SQL.""" + + def __init__(self, fetchone=(), fetchall=(), raise_on=(), rowcounts=()): + self.sql: list = [] + self.params: list = [] + self.txns_opened = 0 + self._fetchone = list(fetchone) + self._fetchall = list(fetchall) + self._raise_on = list(raise_on) # (sql-substring, exception) pairs + self._rowcounts = list(rowcounts) # FIFO rowcount per mutation execute + + def transaction(self): + conn = self + + class Txn: + def __enter__(self): + conn.txns_opened += 1 + conn.sql.append("TXN-START") + conn.params.append(None) # keep sql/params index-aligned + + def __exit__(self, exc_type, exc, tb): + conn.sql.append("TXN-ROLLBACK" if exc_type is not None else "TXN-COMMIT") + conn.params.append(None) + return False + + return Txn() + + def cursor(self): + conn = self + + class Cur: + rowcount = -1 # psycopg semantics: set by execute for DML + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params=None): + conn.sql.append(sql) + conn.params.append(params) + # Only mutations consume the scripted rowcount FIFO; SELECT + # rowcount is never checked by writes.py. + if sql.lstrip().startswith(("INSERT", "UPDATE", "DELETE")): + self.rowcount = conn._rowcounts.pop(0) if conn._rowcounts else 1 + for i, (needle, exc) in enumerate(conn._raise_on): + if needle in sql: + conn._raise_on.pop(i) # one-shot + raise exc + + def fetchone(self): + if not conn._fetchone: + return None + row = conn._fetchone.pop(0) + return None if row is None else dict(row) # scripted None = no row + + def fetchall(self): + return [dict(r) for r in conn._fetchall.pop(0)] if conn._fetchall else [] + + return Cur() + + +def _unwrap(value): + """psycopg's Json adapter -> plain Python (duck-typed: no wrapper here).""" + return value.obj if hasattr(value, "obj") else value + + +def _audit_rows(conn): + """(params, unwrapped detail) per audit INSERT, in execution order.""" + out = [] + for i, sql in enumerate(conn.sql): + if isinstance(sql, str) and sql.startswith("INSERT INTO audit_log"): + params = conn.params[i] + out.append((params, _unwrap(params[3]))) + return out + + +def _statements(conn, prefix): + return [(sql, conn.params[i]) for i, sql in enumerate(conn.sql) + if isinstance(sql, str) and sql.startswith(prefix)] + + +# --- approve: plain (status flip only, partial-index ordering, one audit) ---- + + +def test_approve_plain_supersedes_active_first_then_activates_candidate(): + conn = FakeConn(fetchone=[candidate_row(), {"version": 2}]) + result = writes.approve_candidate("fp_a", 9, actor="amy", reason="looks right", + conn=conn) + + assert result == {"rule_id": 9, "version": 2, "status": "active"} + assert conn.txns_opened == 1 # single transaction + assert conn.sql[0] == "TXN-START" and conn.sql[-1] == "TXN-COMMIT" + + sql = [s for s in conn.sql if isinstance(s, str)] + supersede = next(s for s in sql if "status = 'superseded'" in s) + activate = next(s for s in sql if "status = 'active', activated_at = now()" in s) + # Partial-unique gotcha: the prior active row MUST flip to superseded + # before the candidate goes active (per-statement index check). + assert sql.index(supersede) < sql.index(activate) + assert "WHERE fingerprint_id = %s AND status = 'active'" in supersede + # Plain approve touches status + timestamps ONLY (content is UPDATE-denied). + assert conn.params[conn.sql.index(activate)] == (9,) + # TOCTOU predicate: only a still-pending row may flip (0 rows -> 409). + assert "AND status = 'pending_review'" in activate + + audits = _audit_rows(conn) + assert len(audits) == 1 # exactly one audit INSERT + (actor, action, entity, _), detail = audits[0] + assert (actor, action, entity) == ("amy", "rule_approved", "fp_a") + assert detail["actor"] == "amy" and detail["reason"] == "looks right" + assert detail["prior_version"] == 2 + assert detail["rule_id"] == 9 and detail["version"] == 2 + + +def test_approve_without_prior_active_records_null_prior_version(): + conn = FakeConn(fetchone=[candidate_row(), None]) + result = writes.approve_candidate("fp_a", 9, conn=conn) + assert result["status"] == "active" + (_p, detail) = _audit_rows(conn)[0] + assert detail["prior_version"] is None + + +# --- approve: with edits / full override (new version row, draft consumed) --- + + +def test_approve_with_edited_mappings_mints_new_active_row_and_consumes_draft(): + conn = FakeConn( + fetchone=[candidate_row(), {"version": 2}, {"next_version": 3}, {"id": 12}], + fetchall=[SAMPLE_ROWS], + ) + edited = [dict(MAPPINGS[0])] + result = writes.approve_candidate("fp_a", 9, actor="amy", reason="fix src", + edited_mappings=edited, conn=conn) + + assert result == {"rule_id": 12, "version": 3, "status": "active"} + assert conn.txns_opened == 1 and conn.sql[-1] == "TXN-COMMIT" + + sql = [s for s in conn.sql if isinstance(s, str)] + samples = next(s for s in sql if "FROM onboarding_samples" in s) + version_sel = next(s for s in sql if "COALESCE(MAX(version), 0) + 1" in s) + supersede = next(s for s in sql if "status = 'superseded'" in s) + insert = next(s for s in sql if s.startswith("INSERT INTO rules")) + flip = next(s for s in sql if "SET status = 'rejected'" in s) + # Gate inputs fetched first; supersede BEFORE the new row is born active + # (per-statement partial index); the draft is flipped after the mint. + assert sql.index(samples) < sql.index(supersede) < sql.index(insert) + assert sql.index(version_sel) < sql.index(insert) + assert ("WHERE fingerprint_id = %s AND role IN ('prompt','held_out')") in samples + assert "ORDER BY captured_at, id" in samples + + ip = conn.params[conn.sql.index(insert)] + assert ip[0] == "fp_a" and ip[1] == 3 # version = MAX+1 minted inside the txn + assert ip[2] == PATTERN # edited mappings keep the candidate pattern + assert _unwrap(ip[3]) == edited + assert ip[4] == "active" # status bound as a parameter, not interpolated + assert ip[5] == "slm-edited" + assert ip[6] == 0.87 # candidate confidence carried through an edit + assert ip[7] == "amy" # created_by = the approving actor + validation = _unwrap(ip[8]) + assert validation["passed"] is True # revalidated through the REAL gate + + assert conn.params[conn.sql.index(flip)] == (9,) # draft candidate consumed + assert "AND status = 'pending_review'" in flip # TOCTOU predicate + audits = _audit_rows(conn) + assert len(audits) == 1 + (actor, action, entity, _), detail = audits[0] + assert (actor, action, entity) == ("amy", "rule_approved", "fp_a") + assert detail["consumed_by_edit"] == 3 + assert detail["prior_version"] == 2 and detail["reason"] == "fix src" + + +def test_approve_full_override_inserts_human_row_with_override_content(): + conn = FakeConn( + fetchone=[candidate_row(), {"version": 2}, {"next_version": 4}, {"id": 13}], + fetchall=[SAMPLE_ROWS], + ) + override = {"pattern": PATTERN, "mappings": [dict(m) for m in MAPPINGS]} + result = writes.approve_candidate("fp_a", 9, actor="amy", override=override, + conn=conn) + + assert result == {"rule_id": 13, "version": 4, "status": "active"} + insert = _statements(conn, "INSERT INTO rules")[0] + ip = insert[1] + assert ip[2] == override["pattern"] # the override pattern, not the draft's + assert _unwrap(ip[3]) == override["mappings"] + assert ip[4] == "active" # status bound as a parameter + assert ip[5] == "human" # provenance records the human origin + assert ip[6] is None # the SLM confidence no longer describes this content + assert ip[7] == "amy" + assert _unwrap(ip[8])["passed"] is True + (_p, detail) = _audit_rows(conn)[0] + assert detail["consumed_by_edit"] == 4 + + +# --- approve: failure paths --------------------------------------------------- + + +def test_approve_unknown_candidate_not_found(): + conn = FakeConn(fetchone=[None]) + with pytest.raises(writes.NotFoundError): + writes.approve_candidate("fp_a", 999, conn=conn) + assert not _statements(conn, "UPDATE rules") + + +def test_approve_non_pending_conflicts_without_writes(): + conn = FakeConn(fetchone=[candidate_row(status="rejected")]) + with pytest.raises(writes.ConflictError): + writes.approve_candidate("fp_a", 9, conn=conn) + sql = [s for s in conn.sql if isinstance(s, str)] + assert not [s for s in sql if s.startswith("UPDATE rules")] + assert not [s for s in sql if s.startswith("INSERT INTO rules")] + assert "TXN-ROLLBACK" in sql + + +# --- reject ------------------------------------------------------------------- + + +def test_reject_flips_candidate_and_audits_once(): + conn = FakeConn(fetchone=[candidate_row()]) + result = writes.reject_candidate("fp_a", 9, actor="amy", reason="wrong shape", + conn=conn) + + assert result == {"status": "rejected"} + assert conn.txns_opened == 1 + updates = _statements(conn, "UPDATE rules") + assert "SET status = 'rejected'" in updates[0][0] + assert "AND status = 'pending_review'" in updates[0][0] # TOCTOU predicate + assert updates[0][1] == (9,) + (actor, action, entity, _), detail = _audit_rows(conn)[0] + assert (actor, action, entity) == ("amy", "rule_rejected", "fp_a") + assert detail["reason"] == "wrong shape" and detail["version"] == 2 + + +def test_reject_non_pending_conflicts(): + conn = FakeConn(fetchone=[candidate_row(status="active")]) + with pytest.raises(writes.ConflictError): + writes.reject_candidate("fp_a", 9, conn=conn) + + +def test_reject_unknown_candidate_not_found(): + conn = FakeConn(fetchone=[None]) + with pytest.raises(writes.NotFoundError): + writes.reject_candidate("fp_a", 999, conn=conn) + + +# --- manual authoring (two-step: candidate here, approve via approve) --------- + + +def test_create_manual_candidate_success_stores_pending_human_row(): + conn = FakeConn(fetchone=[{"next_version": 1}, {"id": 33}], fetchall=[SAMPLE_ROWS]) + result = writes.create_manual_candidate("fp_a", PATTERN, MAPPINGS, actor="amy", + confidence=0.5, conn=conn) + + assert result == {"rule_id": 33, "version": 1, "status": "pending_review"} + assert conn.txns_opened == 1 and conn.sql[-1] == "TXN-COMMIT" + insert = _statements(conn, "INSERT INTO rules")[0] + ip = insert[1] + assert ip[0] == "fp_a" and ip[1] == 1 + assert ip[4] == "pending_review" # status bound as a parameter + assert ip[5] == "human" and ip[6] == 0.5 and ip[7] == "amy" + assert _unwrap(ip[8])["passed"] is True # validated through the REAL gate + (actor, action, entity, _), detail = _audit_rows(conn)[0] + assert (actor, action, entity) == ("amy", "candidate_created", "fp_a") + assert detail["rule_id"] == 33 and detail["version"] == 1 + + +def test_create_manual_candidate_validation_failure_writes_nothing(): + conn = FakeConn(fetchall=[SAMPLE_ROWS]) + with pytest.raises(writes.ValidationError) as caught: + writes.create_manual_candidate("fp_a", r"^NOMATCH.*$", MAPPINGS, actor="amy", + conn=conn) + assert caught.value.checks["held_out_match_all"] is False + assert caught.value.notes # gate notes carried for the 422 body + sql = [s for s in conn.sql if isinstance(s, str)] + assert not [s for s in sql if s.startswith("INSERT INTO rules")] + assert "TXN-ROLLBACK" in sql + + +def test_create_manual_candidate_with_no_samples_degrades_to_pass(): + # The unseen-format path: the Task 3 gate passes with a "no samples" note. + conn = FakeConn(fetchone=[{"next_version": 1}, {"id": 34}], fetchall=[[]]) + result = writes.create_manual_candidate("fp_new", PATTERN, MAPPINGS, actor="amy", + conn=conn) + assert result["status"] == "pending_review" + validation = _unwrap(_statements(conn, "INSERT INTO rules")[0][1][8]) + assert validation["passed"] is True + assert any("no samples" in note for note in validation["notes"]) + + +# --- IntegrityError retry (concurrent candidate INSERT -> once, then 409) ----- + + +def test_manual_integrity_error_retries_once_and_succeeds(): + conn = FakeConn( + fetchone=[{"next_version": 1}, {"next_version": 1}, {"id": 33}], + fetchall=[SAMPLE_ROWS, SAMPLE_ROWS], + raise_on=[("INSERT INTO rules", IntegrityError("rules_one_pending"))], + ) + result = writes.create_manual_candidate("fp_a", PATTERN, MAPPINGS, actor="amy", + conn=conn) + assert result["rule_id"] == 33 + assert len(_statements(conn, "INSERT INTO rules")) == 2 # retried once + assert conn.txns_opened == 2 + + +def test_manual_integrity_error_twice_conflicts(): + conn = FakeConn( + fetchone=[{"next_version": 1}, {"next_version": 1}], + fetchall=[SAMPLE_ROWS, SAMPLE_ROWS], + raise_on=[("INSERT INTO rules", IntegrityError("rules_one_pending")), + ("INSERT INTO rules", IntegrityError("rules_one_pending"))], + ) + with pytest.raises(writes.ConflictError): + writes.create_manual_candidate("fp_a", PATTERN, MAPPINGS, actor="amy", conn=conn) + assert len(_statements(conn, "INSERT INTO rules")) == 2 # retry, then give up + + +# --- deactivate / reactivate -------------------------------------------------- + + +def test_deactivate_targets_the_active_row_and_audits(): + conn = FakeConn(fetchone=[{"id": 5, "version": 2}]) + result = writes.deactivate_rule("fp_a", actor="amy", reason="drift", conn=conn) + + assert result == {"rule_id": 5, "version": 2, "status": "deactivated"} + assert conn.txns_opened == 1 + update = _statements(conn, "UPDATE rules")[0] + assert "status = 'deactivated', deactivated_at = now()" in update[0] + assert "AND status = 'active'" in update[0] # TOCTOU predicate + assert update[1] == (5,) + (actor, action, entity, _), detail = _audit_rows(conn)[0] + assert (actor, action, entity) == ("amy", "rule_deactivated", "fp_a") + assert detail == {"rule_id": 5, "version": 2, "actor": "amy", "reason": "drift"} + + +def test_deactivate_without_active_rule_not_found(): + conn = FakeConn(fetchone=[None]) + with pytest.raises(writes.NotFoundError): + writes.deactivate_rule("fp_a", conn=conn) + + +def test_reactivate_sets_active_and_audits(): + conn = FakeConn(fetchone=[ + {"id": 5, "fingerprint_id": "fp_a", "version": 2, "status": "deactivated"}, + None, # no other active version + ]) + result = writes.reactivate_rule(5, actor="amy", reason="false positive", conn=conn) + + assert result == {"rule_id": 5, "version": 2, "status": "active"} + update = _statements(conn, "UPDATE rules")[0] + assert "status = 'active', activated_at = now()" in update[0] + assert "AND status IN ('deactivated', 'superseded')" in update[0] + assert update[1] == (5,) + (actor, action, entity, _), detail = _audit_rows(conn)[0] + assert (actor, action, entity) == ("amy", "rule_reactivated", "fp_a") + assert detail["reason"] == "false positive" + + +def test_reactivate_conflicts_while_another_version_is_active(): + conn = FakeConn(fetchone=[ + {"id": 5, "fingerprint_id": "fp_a", "version": 2, "status": "deactivated"}, + {"id": 7}, # v3 is already active + ]) + with pytest.raises(writes.ConflictError): + writes.reactivate_rule(5, conn=conn) + assert not _statements(conn, "UPDATE rules") + + +def test_reactivate_unknown_rule_not_found(): + conn = FakeConn(fetchone=[None]) + with pytest.raises(writes.NotFoundError): + writes.reactivate_rule(999, conn=conn) + + +# --- fix round 1 (T7-F1): only deactivated/superseded rows reactivate --------- +# +# approve is the ONE activation path; a pending_review candidate (or a +# rejected row) must not be activatable through the reactivate back door — +# that would mint an active rule whose audit trail shows rule_reactivated +# with no rule_approved. + + +def test_reactivate_pending_candidate_conflicts_without_writes(): + conn = FakeConn(fetchone=[candidate_row()]) + with pytest.raises(writes.ConflictError): + writes.reactivate_rule(9, actor="amy", conn=conn) + assert not _statements(conn, "UPDATE rules") + assert not _audit_rows(conn) # no rule_reactivated row on the 409 + assert "TXN-ROLLBACK" in conn.sql + + +def test_reactivate_rejected_rule_conflicts(): + conn = FakeConn(fetchone=[candidate_row(status="rejected")]) + with pytest.raises(writes.ConflictError): + writes.reactivate_rule(9, conn=conn) + assert not _statements(conn, "UPDATE rules") + assert not _audit_rows(conn) + + +# --- fix round 1: TOCTOU predicates, rowcount 409s, retry scoping ------------- + + +def test_approve_activate_rowcount_zero_conflicts_without_audit(): + # The expected-status predicate on the activate UPDATE is the TOCTOU + # backstop: rowcount 0 means a concurrent writer moved the candidate out + # from under us between the pre-check and the statement. Mutations run in + # order supersede, activate — so the FIFO is [1, 0]. + conn = FakeConn(fetchone=[candidate_row(), {"version": 2}], rowcounts=[1, 0]) + with pytest.raises(writes.ConflictError): + writes.approve_candidate("fp_a", 9, conn=conn) + assert not _audit_rows(conn) # no audit row for a mutation that never landed + assert "TXN-ROLLBACK" in conn.sql + + +def test_reject_rowcount_zero_conflicts_without_audit(): + conn = FakeConn(fetchone=[candidate_row()], rowcounts=[0]) + with pytest.raises(writes.ConflictError): + writes.reject_candidate("fp_a", 9, conn=conn) + assert not _audit_rows(conn) + assert "TXN-ROLLBACK" in conn.sql + + +def test_update_integrity_error_conflicts_without_retry(): + # Retry is scoped to the candidate INSERT only: an UPDATE-side unique race + # (rules_one_active lost between supersede and activate) is answered 409 + # immediately — a retry would just re-lose — with the exception logged. + conn = FakeConn( + fetchone=[candidate_row(), {"version": 2}], + raise_on=[("status = 'active', activated_at = now()", + IntegrityError("rules_one_active"))], + ) + with pytest.raises(writes.ConflictError): + writes.approve_candidate("fp_a", 9, conn=conn) + activates = _statements(conn, "UPDATE rules SET status = 'active', activated_at") + assert len(activates) == 1 # single shot: no retry on the UPDATE path + + +def test_reactivate_rowcount_zero_conflicts_without_audit(): + # TOCTOU on reactivate itself: the row flipped pending between the + # pre-check and the UPDATE (IN ('deactivated','superseded') no longer + # matches) — 409, nothing written. + conn = FakeConn(fetchone=[ + {"id": 5, "fingerprint_id": "fp_a", "version": 2, "status": "deactivated"}, + None, + ], rowcounts=[0]) + with pytest.raises(writes.ConflictError): + writes.reactivate_rule(5, conn=conn) + assert not _audit_rows(conn) + assert "TXN-ROLLBACK" in conn.sql diff --git a/services/gateway/writes.py b/services/gateway/writes.py new file mode 100644 index 0000000..bce4cc0 --- /dev/null +++ b/services/gateway/writes.py @@ -0,0 +1,484 @@ +# services/gateway/writes.py — the gateway's write path (Task 7): the rule +# lifecycle transitions behind migration 003's least-privilege rules_role. +# +# The grant surface IS the design: rules_role may INSERT rules, UPDATE ONLY +# (status, activated_at, deactivated_at), INSERT audit_log, and SELECT +# onboarding_samples. Content columns are UPDATE-denied — so approve-with-edits +# INSERTs a fresh version row and flips the draft candidate to rejected; it can +# never rewrite content in place. +# +# Version minting (X-b′): candidates are created pending_review at +# MAX(version)+1 for the fingerprint; plain approve activates the candidate row +# itself (status + timestamps only); approve-with-edits INSERTs version = MAX+1 +# with the edited content and consumes the draft. Versions are NOT gap-free — +# documented, accepted. +# +# Partial-unique ordering gotcha: rules_one_active is a partial unique index, +# checked per-statement and not deferrable — the prior active row MUST be +# flipped to superseded BEFORE the new row goes active, or the later statement +# raises a unique violation. +# +# TOCTOU: every status-transition UPDATE carries its expected-status predicate +# (AND status = 'pending_review' / 'active' / IN ('deactivated','superseded')) +# and checks cur.rowcount — 0 rows means a concurrent writer moved the row +# between the pre-check and the statement: ConflictError, never a silent no-op +# that would still write the audit row. +# +# House style (T7): connections run AUTOCOMMIT; each public function wraps +# exactly one `with conn.transaction():` block; every mutation is paired with +# an audit INSERT (entity = the fingerprint_id; gateway lifecycle actions: +# rule_approved, rule_rejected, rule_deactivated, rule_reactivated; actor is a +# request field defaulting to "anonymous" — no auth in MVP). psycopg is +# imported lazily so unit tests drive everything through fake conns (never a +# live DB; mirrors services/onboarding/store.py). +from __future__ import annotations + +import logging +import os +from typing import TYPE_CHECKING + +from ulpf_core.models import Mapping, Rule +from ulpf_core.validation import report_to_json, validate_candidate + +if TYPE_CHECKING: + import psycopg + +log = logging.getLogger("gateway.writes") + + +class WriteError(Exception): + """Base for gateway write failures; app maps subclasses onto HTTP codes.""" + + +class NotFoundError(WriteError): + """404: unknown candidate/rule/fingerprint.""" + + +class ConflictError(WriteError): + """409: not pending, another version already active, or a lost insert race.""" + + +class ValidationError(WriteError): + """422: the deterministic gate rejected the candidate; carries the report + so the API body can render checks + notes.""" + + def __init__(self, report): + super().__init__("candidate failed validation") + self.report = report + + @property + def checks(self) -> dict: + return self.report.checks + + @property + def notes(self) -> list: + return self.report.notes + + +def _connect_write() -> psycopg.Connection: + """One connection per request, as rules_role; dict rows and AUTOCOMMIT per + house style (multi-statement operations wrap one transaction() block).""" + import psycopg + from psycopg.rows import dict_row + + return psycopg.connect(os.environ["WRITE_DATABASE_URL"], row_factory=dict_row, + autocommit=True) + + +# --- shared statement pieces --------------------------------------------------- + +_CANDIDATE_SQL = ( + "SELECT id, fingerprint_id, version, status, pattern, mappings, provenance, confidence " + "FROM rules WHERE id = %s AND fingerprint_id = %s" +) +_SAMPLES_SQL = ( + "SELECT raw_text, role FROM onboarding_samples " + "WHERE fingerprint_id = %s AND role IN ('prompt','held_out') " + "ORDER BY captured_at, id" +) +_AUDIT_SQL = "INSERT INTO audit_log (actor, action, entity, detail) VALUES (%s, %s, %s, %s)" + +# Status-transition UPDATEs. The expected-status predicate in each WHERE is +# the TOCTOU backstop (rowcount 0 -> ConflictError); the SET side touches only +# the columns migration 003 grants to rules_role. +_ACTIVATE_PENDING_SQL = ( + "UPDATE rules SET status = 'active', activated_at = now() " + "WHERE id = %s AND status = 'pending_review'" +) +_REJECT_PENDING_SQL = ( + "UPDATE rules SET status = 'rejected' " + "WHERE id = %s AND status = 'pending_review'" +) +_DEACTIVATE_ACTIVE_SQL = ( + "UPDATE rules SET status = 'deactivated', deactivated_at = now() " + "WHERE id = %s AND status = 'active'" +) +_REACTIVATE_SQL = ( + "UPDATE rules SET status = 'active', activated_at = now() " + "WHERE id = %s AND status IN ('deactivated', 'superseded')" +) + + +# --- plumbing ------------------------------------------------------------------- + +_CONFLICT_MSG = "concurrent modification; the rule state changed — retry" + + +def _execute(op, conn, /, *args): + """Run op once; open/close the connection when none is injected (tests + always inject a fake).""" + if conn is None: + conn = _connect_write() + try: + return op(conn, *args) + finally: + conn.close() + return op(conn, *args) + + +def _run(op, conn, /, *args): + """Single-shot run. Any IntegrityError — a unique-index race on an UPDATE, + or a CHECK/FK violation — is logged with its detail and answered + ConflictError (409), never retried and never a raw 500. Only the + candidate-INSERT path retries (see _run_candidate_insert).""" + import psycopg + + try: + return _execute(op, conn, *args) + except psycopg.IntegrityError as exc: + log.warning("constraint violation, answering 409 without retry: %s", exc) + raise ConflictError(_CONFLICT_MSG) from exc + + +def _run_candidate_insert(op, conn, /, *args): + """The brief's pinned race, scoped to ops whose mutating statement is the + rules INSERT (manual create, approve-with-content): a concurrent writer + winning the version/pending/active race retries the whole txn ONCE — the + re-run re-reads state inside a fresh transaction and either wins or + conflicts. UPDATE-side races take the single-shot 409 from _run; CHECK/FK + violations are not retried pointlessly.""" + import psycopg + + last_exc: psycopg.IntegrityError | None = None + for attempt in (1, 2): + try: + return _execute(op, conn, *args) + except psycopg.IntegrityError as exc: + last_exc = exc # kept: `as exc` is deleted at except-block exit + log.warning("rules INSERT integrity error (attempt %d/2): %s", + attempt, exc) + raise ConflictError(_CONFLICT_MSG) from last_exc + + +def _audit(cur, action: str, entity: str, detail: dict, *, actor: str) -> None: + from psycopg.types.json import Json # deferred: keeps psycopg off the import path + + cur.execute(_AUDIT_SQL, (actor, action, entity, Json(detail))) + + +def _mutate(cur, sql: str, params, conflict: str) -> None: + """Run a status-transition UPDATE and enforce its expected-status + predicate: rowcount 0 means another writer changed the row between the + pre-check and this statement — ConflictError, never a silent no-op that + would still write the audit row.""" + cur.execute(sql, params) + if cur.rowcount == 0: + raise ConflictError(conflict) + + +def _load_candidate(cur, fingerprint_id: str, candidate_id: int) -> dict: + cur.execute(_CANDIDATE_SQL, (candidate_id, fingerprint_id)) + row = cur.fetchone() + if row is None: + raise NotFoundError( + f"candidate {candidate_id} not found for fingerprint {fingerprint_id!r}") + return row + + +def _require_pending(candidate: dict) -> None: + if candidate["status"] != "pending_review": + raise ConflictError( + f"candidate {candidate['id']} is {candidate['status']!r}, not pending_review") + + +def _revalidation_lines(cur, fingerprint_id: str) -> tuple[list, list]: + """(prompt_lines, held_out_lines) in capture order — the gate's inputs.""" + cur.execute(_SAMPLES_SQL, (fingerprint_id,)) + prompt: list[str] = [] + held_out: list[str] = [] + for row in cur.fetchall(): + (prompt if row["role"] == "prompt" else held_out).append(row["raw_text"]) + return prompt, held_out + + +def _supersede_active(cur, fingerprint_id: str): + """Flip the fingerprint's active row to superseded FIRST (partial-unique + gotcha: the index is per-statement, so activating a new row while the old + one is still active would raise). Returns the prior active version, or + None when nothing was active.""" + cur.execute( + "UPDATE rules SET status = 'superseded' " + "WHERE fingerprint_id = %s AND status = 'active' RETURNING version", + (fingerprint_id,), + ) + row = cur.fetchone() + return row["version"] if row else None + + +def _next_version(cur, fingerprint_id: str) -> int: + """MAX(version)+1 for the fingerprint, computed INSIDE the txn.""" + cur.execute( + "SELECT COALESCE(MAX(version), 0) + 1 AS next_version FROM rules " + "WHERE fingerprint_id = %s", + (fingerprint_id,), + ) + return cur.fetchone()["next_version"] + + +def _insert_rule(cur, *, fingerprint_id: str, version: int, pattern: str, mappings, + provenance: str, confidence, status: str, created_by: str, + report) -> int: + from psycopg.types.json import Json + + # status is a module-controlled vocabulary value ('active' / + # 'pending_review'), never request input — but it is bound as a parameter + # like every other value (parameterized SQL only); the rules status CHECK + # constraint is the real guard. + cur.execute( + "INSERT INTO rules (fingerprint_id, version, pattern, mappings, status, " + "provenance, confidence, created_by, validation) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id", + (fingerprint_id, version, pattern, Json(mappings), status, provenance, + confidence, created_by, Json(report_to_json(report))), + ) + return cur.fetchone()["id"] + + +# --- approve -------------------------------------------------------------------- + +def approve_candidate(fingerprint_id: str, candidate_id: int, *, actor: str = "anonymous", + reason: str | None = None, edited_mappings=None, override=None, + conn: psycopg.Connection | None = None) -> dict: + """Approve a pending candidate — the ONE activation path. + + Plain approve flips the candidate row itself to active (status + timestamps + only; content columns are UPDATE-denied by grant). With edited_mappings or + a full override, the edited rule is revalidated through the deterministic + gate against the fingerprint's samples, a NEW row version = MAX+1 is + INSERTed already active (provenance 'slm-edited' / 'human'), and the draft + candidate is flipped to rejected (audited with consumed_by_edit).""" + # Only the with-content path contains a rules INSERT, so only it gets the + # candidate-insert retry; the plain path single-shots (see _run). + runner = (_run if edited_mappings is None and override is None + else _run_candidate_insert) + return runner(_approve_once, conn, fingerprint_id, candidate_id, actor, reason, + edited_mappings, override) + + +def _edited_content(candidate: dict, edited_mappings, override): + """(pattern, mappings, provenance, confidence) for the approve-with-edits + row: edited_mappings keep the candidate pattern + confidence with + provenance 'slm-edited'; a full override replaces pattern and mappings + wholesale with provenance 'human' (the SLM confidence no longer describes + that content).""" + if override is not None: + return override["pattern"], override["mappings"], "human", None + return candidate["pattern"], edited_mappings, "slm-edited", candidate["confidence"] + + +def _approve_once(conn, fingerprint_id, candidate_id, actor, reason, edited_mappings, + override): + with conn.transaction(), conn.cursor() as cur: + candidate = _load_candidate(cur, fingerprint_id, candidate_id) + _require_pending(candidate) + + if edited_mappings is None and override is None: + # Plain approve: supersede the prior active row BEFORE this one goes + # active (partial-unique gotcha), then flip status + timestamps only. + prior_version = _supersede_active(cur, fingerprint_id) + _mutate(cur, _ACTIVATE_PENDING_SQL, (candidate_id,), + f"candidate {candidate_id} is no longer pending_review " + "(concurrent modification)") + _audit(cur, "rule_approved", fingerprint_id, { + "rule_id": candidate_id, + "version": candidate["version"], + "actor": actor, + "reason": reason, + "prior_version": prior_version, + }, actor=actor) + return {"rule_id": candidate_id, "version": candidate["version"], + "status": "active"} + + # Approve-with-edits: validate FIRST (a failing gate must not mutate + # anything), then supersede, mint the new active version, consume the draft. + pattern, mappings, provenance, confidence = _edited_content( + candidate, edited_mappings, override) + prompt_lines, held_out_lines = _revalidation_lines(cur, fingerprint_id) + rule = Rule(fingerprint_id=fingerprint_id, version=candidate["version"], + pattern=pattern, provenance=provenance, + mappings=[Mapping(**m) for m in mappings]) + report = validate_candidate(rule, prompt_lines, held_out_lines) + if not report.passed: + raise ValidationError(report) + + prior_version = _supersede_active(cur, fingerprint_id) + version = _next_version(cur, fingerprint_id) + new_id = _insert_rule(cur, fingerprint_id=fingerprint_id, version=version, + pattern=pattern, mappings=mappings, provenance=provenance, + confidence=confidence, status="active", created_by=actor, + report=report) + _mutate(cur, _REJECT_PENDING_SQL, (candidate_id,), + f"candidate {candidate_id} is no longer pending_review " + "(concurrent modification)") + _audit(cur, "rule_approved", fingerprint_id, { + "rule_id": new_id, + "version": version, + "actor": actor, + "reason": reason, + "prior_version": prior_version, + "consumed_by_edit": version, + "consumed_candidate_id": candidate_id, + "provenance": provenance, + }, actor=actor) + return {"rule_id": new_id, "version": version, "status": "active"} + + +# --- reject --------------------------------------------------------------------- + +def reject_candidate(fingerprint_id: str, candidate_id: int, *, actor: str = "anonymous", + reason: str | None = None, + conn: psycopg.Connection | None = None) -> dict: + """Reject a pending candidate (terminal).""" + return _run(_reject_once, conn, fingerprint_id, candidate_id, actor, reason) + + +def _reject_once(conn, fingerprint_id, candidate_id, actor, reason): + with conn.transaction(), conn.cursor() as cur: + candidate = _load_candidate(cur, fingerprint_id, candidate_id) + _require_pending(candidate) + _mutate(cur, _REJECT_PENDING_SQL, (candidate_id,), + f"candidate {candidate_id} is no longer pending_review " + "(concurrent modification)") + _audit(cur, "rule_rejected", fingerprint_id, { + "rule_id": candidate_id, + "version": candidate["version"], + "actor": actor, + "reason": reason, + }, actor=actor) + return {"status": "rejected"} + + +# --- manual authoring (step 1 of 2; approval rides approve_candidate) ------------ + +def create_manual_candidate(fingerprint_id: str, pattern: str, mappings, *, + actor: str = "anonymous", confidence: float | None = None, + conn: psycopg.Connection | None = None) -> dict: + """Validate a hand-written rule through the SAME deterministic gate as SLM + candidates and store it pending_review (provenance 'human', created_by = + actor). A human then approves it through the same approve endpoint — one + activation path, uniform audit. Validation failure writes nothing and + raises ValidationError (app -> 422 with checks + notes). With no samples + the gate degrades to pass-with-note, so an unseen format can be authored.""" + return _run_candidate_insert(_create_manual_once, conn, fingerprint_id, pattern, + list(mappings), actor, confidence) + + +def _create_manual_once(conn, fingerprint_id, pattern, mappings, actor, confidence): + with conn.transaction(), conn.cursor() as cur: + prompt_lines, held_out_lines = _revalidation_lines(cur, fingerprint_id) + # version=1 is a generation placeholder; the real version is minted below. + rule = Rule(fingerprint_id=fingerprint_id, version=1, pattern=pattern, + provenance="human", mappings=[Mapping(**m) for m in mappings]) + report = validate_candidate(rule, prompt_lines, held_out_lines) + if not report.passed: + raise ValidationError(report) + version = _next_version(cur, fingerprint_id) + rule_id = _insert_rule(cur, fingerprint_id=fingerprint_id, version=version, + pattern=pattern, mappings=mappings, provenance="human", + confidence=confidence, status="pending_review", + created_by=actor, report=report) + _audit(cur, "candidate_created", fingerprint_id, { + "rule_id": rule_id, + "version": version, + "provenance": "human", + "confidence": confidence, + }, actor=actor) + return {"rule_id": rule_id, "version": version, "status": "pending_review"} + + +# --- deactivate / reactivate ------------------------------------------------------ + +def deactivate_rule(fingerprint_id: str, *, actor: str = "anonymous", + reason: str | None = None, + conn: psycopg.Connection | None = None) -> dict: + """Deactivate the fingerprint's ACTIVE rule (fingerprint reverts to + raw-only; samples re-accumulate and onboarding re-runs).""" + return _run(_deactivate_once, conn, fingerprint_id, actor, reason) + + +def _deactivate_once(conn, fingerprint_id, actor, reason): + with conn.transaction(), conn.cursor() as cur: + cur.execute( + "SELECT id, version FROM rules " + "WHERE fingerprint_id = %s AND status = 'active'", + (fingerprint_id,), + ) + row = cur.fetchone() + if row is None: + raise NotFoundError(f"no active rule for fingerprint {fingerprint_id!r}") + _mutate(cur, _DEACTIVATE_ACTIVE_SQL, (row["id"],), + f"rule {row['id']} is no longer active (concurrent modification)") + _audit(cur, "rule_deactivated", fingerprint_id, { + "rule_id": row["id"], + "version": row["version"], + "actor": actor, + "reason": reason, + }, actor=actor) + return {"rule_id": row["id"], "version": row["version"], + "status": "deactivated"} + + +def reactivate_rule(rule_id: int, *, actor: str = "anonymous", + reason: str | None = None, + conn: psycopg.Connection | None = None) -> dict: + """Reactivate a rule row (status -> active, activated_at = now). Only + deactivated and superseded rows come back — the rollback path; 409 for a + pending/rejected/active target (approve is the ONE activation path) and + when another version of the fingerprint is already active + (rules_one_active).""" + return _run(_reactivate_once, conn, rule_id, actor, reason) + + +def _reactivate_once(conn, rule_id, actor, reason): + with conn.transaction(), conn.cursor() as cur: + cur.execute( + "SELECT id, fingerprint_id, version, status FROM rules WHERE id = %s", + (rule_id,), + ) + row = cur.fetchone() + if row is None: + raise NotFoundError(f"rule {rule_id} not found") + if row["status"] in ("active", "pending_review", "rejected"): + raise ConflictError( + f"rule {rule_id} is {row['status']!r}; only deactivated or " + "superseded rules reactivate") + cur.execute( + "SELECT id FROM rules " + "WHERE fingerprint_id = %s AND status = 'active' AND id <> %s", + (row["fingerprint_id"], rule_id), + ) + other = cur.fetchone() + if other is not None: + raise ConflictError( + f"rule {other['id']} is already active for fingerprint " + f"{row['fingerprint_id']!r}") + _mutate(cur, _REACTIVATE_SQL, (rule_id,), + f"rule {rule_id} changed state concurrently; expected " + "deactivated or superseded") + _audit(cur, "rule_reactivated", row["fingerprint_id"], { + "rule_id": rule_id, + "version": row["version"], + "actor": actor, + "reason": reason, + }, actor=actor) + return {"rule_id": rule_id, "version": row["version"], "status": "active"} diff --git a/services/onboarding/Dockerfile b/services/onboarding/Dockerfile new file mode 100644 index 0000000..c18effd --- /dev/null +++ b/services/onboarding/Dockerfile @@ -0,0 +1,27 @@ +# services/onboarding/Dockerfile — M2 cold-path candidate generation: +# sample split -> SLM/JSON candidate -> validation gate -> pending_review. +# Build context is the repo root (compose build.context: ..; see /.dockerignore). +FROM python:3.12-slim + +WORKDIR /app + +# BOTH libs first (the validation gate imports ocsf_schema.validator via +# ulpf_core), then deps (layer caching), then the service package. +# /app contains onboarding/ and libs/ so `python -m onboarding.app` imports cleanly. +COPY libs/ulpf-core/pyproject.toml /app/libs/ulpf-core/pyproject.toml +COPY libs/ulpf-core/ulpf_core/ /app/libs/ulpf-core/ulpf_core/ +COPY libs/ocsf-schema/pyproject.toml /app/libs/ocsf-schema/pyproject.toml +COPY libs/ocsf-schema/ocsf_schema/ /app/libs/ocsf-schema/ocsf_schema/ +COPY services/onboarding/requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt \ + && pip install --no-cache-dir -e /app/libs/ulpf-core -e /app/libs/ocsf-schema + +COPY services/onboarding/__init__.py /app/onboarding/__init__.py +COPY services/onboarding/config.py /app/onboarding/config.py +COPY services/onboarding/samples.py /app/onboarding/samples.py +COPY services/onboarding/generate.py /app/onboarding/generate.py +COPY services/onboarding/store.py /app/onboarding/store.py +COPY services/onboarding/reparse.py /app/onboarding/reparse.py +COPY services/onboarding/app.py /app/onboarding/app.py + +CMD ["python", "-m", "onboarding.app"] diff --git a/services/onboarding/__init__.py b/services/onboarding/__init__.py new file mode 100644 index 0000000..fd79047 --- /dev/null +++ b/services/onboarding/__init__.py @@ -0,0 +1,3 @@ +# services/onboarding — M2 cold-path service: sample split -> SLM/JSON +# candidate generation -> deterministic validation gate -> pending_review rule +# row, every step audit-logged (plan Task 5). diff --git a/services/onboarding/app.py b/services/onboarding/app.py new file mode 100644 index 0000000..50d6375 --- /dev/null +++ b/services/onboarding/app.py @@ -0,0 +1,175 @@ +# services/onboarding/app.py — entrypoint for BOTH onboarding convergent +# loops (cold path; the hot path is never touched from here). +# +# Candidate loop (run_candidate_loop), per fingerprint: has_active_or_pending +# guard -> load_samples -> assign_roles -> mark_roles -> audit(samples_split) +# -> JSON fast-path (ALL prompt lines parse as JSON) or SLM generate_candidate +# -> validate_candidate -> passed: insert_candidate (audits candidate_created) +# / failed: record_attempt + audit(candidate_failed). Reparse loop +# (run_reparse_loop): R11 backlog supersede sweep (services/onboarding/reparse). +# Every DB failure and SLM unreachability is logged and the loop continues — +# fail-closed posture: onboarding pends, the hot path stays untouched. +import logging +import threading + +from ulpf_core.validation import validate_candidate + +from onboarding.config import Config +from onboarding.generate import ( + OllamaClient, + generate_candidate, + generate_json_rule, + is_json_line, +) +from onboarding.reparse import run_reparse_loop +from onboarding.samples import assign_roles +from onboarding.store import ( + audit, + connect_db, + count_samples, + has_active_or_pending, + insert_candidate, + load_samples, + mark_roles, + ready_fingerprints, + record_attempt, +) + +log = logging.getLogger("onboarding.app") + + +def process_fingerprint(cfg: Config, fingerprint_id: str) -> bool: + """One candidate cycle for one fingerprint; True iff a candidate was + stored. Any failure is logged here and reported as False — never raised + into the loop.""" + with connect_db(cfg.database_url) as conn: + has_active, has_pending = has_active_or_pending(conn, fingerprint_id) + if has_active or has_pending: + log.info("skip %s: active or pending rule exists", fingerprint_id) + return False + rows = load_samples(conn, fingerprint_id, cfg.sample_threshold) + + if len(rows) < cfg.prompt_size + cfg.held_out_size: + log.info("skip %s: %d samples below split size", fingerprint_id, len(rows)) + return False + + split = assign_roles(rows, cfg.prompt_size, cfg.held_out_size) + prompt_lines = [row["raw_text"] for row in split["prompt"]] + held_out_lines = [row["raw_text"] for row in split["held_out"]] + + with connect_db(cfg.database_url) as conn: + mark_roles(conn, split) + audit(conn, "samples_split", fingerprint_id, { + "prompt": len(split["prompt"]), + "held_out": len(split["held_out"]), + "unused": len(split["unused"]), + }) + + try: + if all(is_json_line(line) for line in prompt_lines): + # JSON fast-path: deterministic alias matching, NO model call. All + # samples (prompt + held-out) feed the confidence statistic — the + # rule derives from key-name aliases, so there is nothing to leak + # into it and the confidence covers the held-out split too. + rule = generate_json_rule(fingerprint_id, prompt_lines + held_out_lines) + else: + client = OllamaClient(cfg.ollama_url, cfg.ollama_model) + rule = generate_candidate(fingerprint_id, prompt_lines, client) + except Exception as exc: + # GenerationError / SLM unreachable / JSON edge — fail closed. Full + # traceback to the log; the summary goes to the attempt record. + log.exception("generation failed for %s", fingerprint_id) + _record_failure(cfg, fingerprint_id, f"generation failed: {exc}") + return False + + report = validate_candidate(rule, prompt_lines, held_out_lines) + if not report.passed: + summary = "; ".join(report.notes[:3]) or "validation failed" + _record_failure(cfg, fingerprint_id, f"validation failed: {summary}") + return False + + with connect_db(cfg.database_url) as conn: + # insert_candidate audits candidate_created itself (ruling: actor = + # the created_by value); the audit detail carries the REAL version — + # the rule object's version field is only the generation placeholder. + rule_id = insert_candidate(conn, rule, report) + log.info("candidate stored for %s (rules.id=%s, provenance=%s, confidence=%s)", + fingerprint_id, rule_id, rule.provenance, getattr(rule, "confidence", None)) + return True + + +def _record_failure(cfg: Config, fingerprint_id: str, error: str) -> None: + """record_attempt + audit(candidate_failed); itself failure-tolerant — + a bookkeeping write must never kill the loop. + + samples_seen records the fingerprint's TOTAL sample count (store. + count_samples — the same units _READY_SQL's s.total is compared against), + NOT the capped loaded count: with the loaded count, a fingerprint whose + backlog kept growing during an outage satisfies total - samples_seen >= + retry_new_samples forever and is re-attempted every poll (T5-F1). + """ + log.warning("candidate failed for %s: %s", fingerprint_id, error) + try: + with connect_db(cfg.database_url) as conn: + total = count_samples(conn, fingerprint_id) + record_attempt(conn, fingerprint_id, total, error) + audit(conn, "candidate_failed", fingerprint_id, + {"error": error, "samples_seen": total}) + except Exception: + log.exception("failed to record the attempt for %s", fingerprint_id) + + +def run_candidate_loop(cfg: Config, stop: threading.Event) -> None: + """Poll for ready fingerprints and run one candidate cycle each, serially + (cold path). `stop.set()` ends the loop promptly (wait() is sliced by the + event). Every exception is logged; the loop continues — convergence is + re-attempted on the next poll.""" + log.info("onboarding candidate loop started (threshold=%d, poll=%.1fs, " + "slm=%s@%s)", cfg.sample_threshold, cfg.onboarding_poll_s, + cfg.ollama_model, cfg.ollama_url) + while not stop.is_set(): + try: + with connect_db(cfg.database_url) as conn: + fingerprints = ready_fingerprints(conn, cfg) + for fingerprint_id in fingerprints: + if stop.is_set(): + break + try: + process_fingerprint(cfg, fingerprint_id) + except Exception: + log.exception("fingerprint %s failed; continuing", fingerprint_id) + except Exception: + log.exception("candidate loop iteration failed; will retry") + stop.wait(cfg.onboarding_poll_s) + log.info("onboarding candidate loop stopped") + + +def main() -> None: + logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s" + ) + cfg = Config.from_env() + # The service runs BOTH convergent loops (controller ruling): candidate + # generation + the R11 backlog re-parse, as daemon threads sharing ONE + # stop handle. SIGINT lands in this (main) thread, which sets stop and + # lets the daemons wind down. + stop = threading.Event() + loops = [ + threading.Thread(target=run_candidate_loop, args=(cfg, stop), daemon=True), + threading.Thread(target=run_reparse_loop, args=(cfg, stop), daemon=True), + ] + for loop in loops: + loop.start() + try: + # Timeout-ed joins: a bare join() cannot be interrupted by Ctrl+C on + # Windows, so poll liveness instead of blocking forever. + while any(loop.is_alive() for loop in loops): + for loop in loops: + loop.join(0.5) + except KeyboardInterrupt: + stop.set() + log.info("shutdown requested") + + +if __name__ == "__main__": + main() diff --git a/services/onboarding/config.py b/services/onboarding/config.py new file mode 100644 index 0000000..cb6bdf4 --- /dev/null +++ b/services/onboarding/config.py @@ -0,0 +1,31 @@ +# services/onboarding/config.py — onboarding service configuration. +# +# Compose env contract (Task 11 sets exactly these names): DATABASE_URL, +# OLLAMA_URL, OLLAMA_MODEL. Defaults are the compose-internal endpoints +# (binding ruling): the DSN follows M1's per-service constant pattern (env +# overrides), the SLM tier is qwen3:4b behind ollama-4b:11434. +import os +from dataclasses import dataclass + + +@dataclass +class Config: + database_url: str = "postgresql://onboarding_role:onboarding_dev@postgres:5432/ulpf" + ollama_url: str = "http://ollama-4b:11434" + ollama_model: str = "qwen3:4b" + sample_threshold: int = 20 + prompt_size: int = 5 + held_out_size: int = 15 + retry_new_samples: int = 10 + onboarding_poll_s: float = 5.0 + reparse_poll_s: float = 2.0 + reparse_batch: int = 500 + + @classmethod + def from_env(cls) -> "Config": + base = cls() + return cls( + database_url=os.environ.get("DATABASE_URL", base.database_url), + ollama_url=os.environ.get("OLLAMA_URL", base.ollama_url), + ollama_model=os.environ.get("OLLAMA_MODEL", base.ollama_model), + ) diff --git a/services/onboarding/generate.py b/services/onboarding/generate.py new file mode 100644 index 0000000..2a6eefe --- /dev/null +++ b/services/onboarding/generate.py @@ -0,0 +1,423 @@ +# services/onboarding/generate.py — SLM candidate generation + JSON fast-path. +# +# Ported from legacy-demo/onboarding/generate_rule.py (the proven demo logic): +# the prompt (both few-shots + anti-hardcoding rule 4), fence-tolerant JSON +# extraction, the retry loop with escalating temperature and error-feedback +# turns, and the deterministic JSON-alias fast-path. Deliberate differences: +# - the SLM sits behind the SLMClient Protocol and OllamaClient imports +# `ollama` lazily inside chat() — no test or import path touches it; +# - exhausted retries raise GenerationError (fail-closed: app.py records the +# attempt and moves on) instead of returning the demo's last 0-confidence +# rule; +# - confidence rides GeneratedRule (the rules.confidence REAL column) via +# the ActiveRule subclass precedent from services/pipeline/rules.py — the +# ulpf_core Rule model deliberately carries no confidence field; +# - provenance is 'slm' (the rules table CHECK), not the demo's +# 'slm-generated'; +# - the OCSF target list comes from ulpf_core.parsing.OCSF_TARGETS and the +# extension key=value grammar from EXTENSION_KV (single definitions — +# never recompiled here); +# - set iteration is sorted before alias matching so the generated rule is +# deterministic across runs (the demo iterated a raw set). +import json +import logging +import re +from typing import Protocol + +from ulpf_core.models import Mapping, Rule +from ulpf_core.parsing import EXTENSION_KV, JSON_SENTINEL, OCSF_TARGETS + +log = logging.getLogger("onboarding.generate") + + +class GenerationError(RuntimeError): + """The SLM could not produce a usable rule within max_attempts.""" + + +class GeneratedRule(Rule): + """Rule plus generation-time confidence (rules.confidence REAL). + + A GeneratedRule IS a Rule — parse() and the Task 3 validation gate are + unaffected — mirroring the ActiveRule precedent in + services/pipeline/rules.py (an id/extra riding a Rule subclass). + """ + + confidence: float + + +class SLMClient(Protocol): + def chat(self, prompt: str, temperature: float) -> str: ... + + +class OllamaClient: + """SLMClient over a local Ollama server. `ollama` is imported lazily in + chat() so this module (and the unit tests) load without it installed.""" + + def __init__(self, base_url: str, model: str): + self.base_url = base_url + self.model = model + + def chat(self, prompt: str, temperature: float) -> str: + from ollama import Client # deferred: unit tests never need ollama + + response = Client(host=self.base_url).chat( + model=self.model, + messages=[{"role": "user", "content": prompt}], + format="json", + options={"temperature": temperature}, + # qwen3 emits chain-of-thought before the answer by default; on + # the CPU-only target tier that is ~20-40 min per attempt and can + # fill the 4096-token context before any JSON exists (live- + # verified Task 11). Reasoning buys nothing for rule extraction. + think=False, + ) + return response["message"]["content"] + + +# Keep this short and targeted — do NOT dump the full OCSF spec into the prompt. +_OCSF_TARGET_LIST = sorted(OCSF_TARGETS) + +# One worked example, in a DIFFERENT vendor format but the SAME general +# shape (pipe-delimited header + space-separated key=value extension) as +# CEF/LEEF — this teaches the model the right structural pattern instead +# of inventing a garbled hybrid. A structurally-unrelated example (e.g. +# plain syslog) taught the model the wrong lesson in testing. +FEW_SHOT_EXAMPLE_INPUT = """Sample log lines: +LEEF:2.0|Acme|NetGuard|3.2|401|src=198.51.100.7 srcPort=44521 dst=192.168.1.5 dstPort=443 proto=TCP action=allow msg=Session started +LEEF:2.0|Acme|NetGuard|3.2|402|src=203.0.113.9 srcPort=51002 dst=192.168.1.6 dstPort=22 proto=TCP action=deny msg=Blocked login attempt +""" + +FEW_SHOT_EXAMPLE_OUTPUT = { + "pattern": ( + r"^.*\|(?P.*)$" + ), + "field_mappings": [ + {"source_field": "src", "ocsf_path": "src_endpoint.ip"}, + {"source_field": "srcPort", "ocsf_path": "src_endpoint.port"}, + {"source_field": "dst", "ocsf_path": "dst_endpoint.ip"}, + {"source_field": "dstPort", "ocsf_path": "dst_endpoint.port"}, + {"source_field": "action", "ocsf_path": "action"}, + {"source_field": "msg", "ocsf_path": "message"}, + ], +} + +# Second few-shot example: prefix-boilerplate log with iptables-style keys. +# Teaches two things the LEEF example doesn't: (1) skip an arbitrary leading +# prefix (timestamp/hostname/process) rather than anchoring tightly, and +# (2) IN=/OUT=/MAC=/LEN= are interface/hardware fields, NOT endpoints — +# only SRC=/DST=/SPT=/DPT=-style keys are actual source/destination data. +FEW_SHOT_EXAMPLE_INPUT_2 = """Sample log lines: +Sep 3 10:05:12 gw02 kernel: FW-DROP: IN=eth1 OUT= MAC=00:1a:2b SRC=10.0.0.5 DST=10.0.0.9 LEN=40 PROTO=TCP SPT=443 DPT=8080 FLAGS=ACK +Sep 5 22:41:03 gw07 kernel: FW-DROP: IN=eth2 OUT= MAC=aa:bb:cc SRC=10.1.4.2 DST=10.1.4.9 LEN=60 PROTO=UDP SPT=53 DPT=33211 FLAGS= +""" + +FEW_SHOT_EXAMPLE_OUTPUT_2 = { + "pattern": ( + r"^.*?FW-DROP:\s*(?P.*)$" + ), + "field_mappings": [ + {"source_field": "SRC", "ocsf_path": "src_endpoint.ip"}, + {"source_field": "DST", "ocsf_path": "dst_endpoint.ip"}, + {"source_field": "SPT", "ocsf_path": "src_endpoint.port"}, + {"source_field": "DPT", "ocsf_path": "dst_endpoint.port"}, + ], +} + + +def build_prompt(sample_lines: list[str]) -> str: + joined_samples = "\n".join(sample_lines) + return f"""You are a log-parsing rule generator. You will be given raw log lines +of semi-structured text that all share the same format. Produce a Python +regular expression with NAMED capture groups (?P...) that matches +the structure of these lines, plus a mapping from each captured group to +a target field. + +1. Many formats have a HEADER before the meaningful content — either + pipe-delimited (CEF, LEEF) or a free-text prefix like a timestamp and + hostname (syslog-style). Do NOT try to match the header field-by-field. + Use a GREEDY generic skip instead: + - pipe-delimited header: use "^.*\\|" to skip to the last pipe + - free-text prefix: use "^.*?" (non-greedy) to skip to wherever the + actual meaningful key=value content begins + This works regardless of exactly how many header fields exist. + +2. If the meaningful content consists of space-separated key=value pairs + (e.g., src=1.2.3.4 dst=5.6.7.8), do NOT try to match each pair individually. + Instead, capture the ENTIRE key=value remainder of the line in a single + named group called (?P.*). Our system will automatically parse + the key=value pairs inside the extension blob. The source_field in your + mappings must exactly match the key name from the log (e.g. "src", "dstPort"). + +3. In firewall/iptables-style logs, IN=, OUT=, MAC=, and LEN= are + NETWORK INTERFACE / hardware fields — they are NOT source or + destination addresses. Do not map them to endpoint IPs. The real + endpoint data is in keys like SRC=, DST=, SPT=, DPT= (or clearly + analogous names). Only capture and map fields that are actually + source/destination/port/action/message data. + +4. CRITICAL — do not hardcode any literal text that is specific to just + the sample lines shown to you (a specific hostname, process name, PID, + date, or exact interface name). Your pattern must generalize to lines + you have NOT seen, which will have DIFFERENT hostnames/timestamps/etc. + Anything that varies conceptually must be matched with a generic wildcard. + +Only map fields that clearly correspond to one of these target fields — +leave other captured groups unmapped if they don't fit: +{", ".join(_OCSF_TARGET_LIST)} + +Respond with ONLY a JSON object, no other text, no markdown code fences, +in exactly this shape. Two worked examples: + +Example 1 (pipe-delimited header): +{FEW_SHOT_EXAMPLE_INPUT} +Expected output: +{json.dumps(FEW_SHOT_EXAMPLE_OUTPUT, indent=2)} + +Example 2 (free-text prefix, interface fields present but NOT mapped): +{FEW_SHOT_EXAMPLE_INPUT_2} +Expected output: +{json.dumps(FEW_SHOT_EXAMPLE_OUTPUT_2, indent=2)} + +Now do the same for these sample lines: +{joined_samples} + +Respond with ONLY the JSON object. +""" + + +def is_json_line(line: str) -> bool: + try: + json.loads(line) + return True + except json.JSONDecodeError: + return False + + +def extract_json(raw_response: str) -> dict: + """Model sometimes wraps JSON in ```json fences or adds stray text — strip that.""" + text = raw_response.strip() + text = re.sub(r"^```(?:json)?\s*", "", text) + text = re.sub(r"\s*```$", "", text) + # If there's still leading/trailing junk, grab the outermost {...} + match = re.search(r"\{.*\}", text, re.DOTALL) + if not match: + raise ValueError(f"No JSON object found in model response: {raw_response[:200]}") + return json.loads(match.group(0)) + + +def _resolve_json_path(obj: dict, dotted_path: str): + """Walk a nested dict using a dotted path like 'alert.severity'. + Returns None if any part of the path is missing (never raises).""" + current = obj + for part in dotted_path.split("."): + if not isinstance(current, dict) or part not in current: + return None + current = current[part] + return current + + +def _flatten(obj: dict, prefix: str = "") -> dict: + """Flatten a nested dict into {dotted.path: value} pairs.""" + flat = {} + for key, value in obj.items(): + path = f"{prefix}.{key}" if prefix else key + if isinstance(value, dict): + flat.update(_flatten(value, path)) + else: + flat[path] = value + return flat + + +# Common key-name aliases seen in real security log JSON, matched against +# the LEAF (last segment) of a flattened dotted path, case-insensitively. +_JSON_FIELD_ALIASES = { + "src_endpoint.ip": ["src_ip", "source_ip", "srcip"], + "src_endpoint.port": ["src_port", "source_port", "srcport"], + "dst_endpoint.ip": ["dest_ip", "dst_ip", "destination_ip", "dstip"], + "dst_endpoint.port": ["dest_port", "dst_port", "destination_port", "dstport"], + "time": ["timestamp", "time"], + "severity_id": ["severity", "sev"], + "action": ["action", "event_type"], + "message": ["message", "msg", "signature"], +} + + +def generate_json_rule(fingerprint_id: str, sample_lines: list[str]) -> GeneratedRule: + """Deterministic path for JSON-formatted logs — NO model call. JSON is + already self-describing, so we flatten it and match field names against + known aliases directly. This is more reliable and much faster than asking + an SLM to reverse-engineer a regex for data that's already structured. + """ + parsed_lines = [] + for line in sample_lines: + try: + parsed_lines.append(json.loads(line)) + except json.JSONDecodeError: + continue + + if not parsed_lines: + raise ValueError(f"No sample lines for '{fingerprint_id}' parsed as valid JSON") + + # Union of flattened keys across all sample lines, so we catch fields + # that only appear in some lines. Sorted before matching: "first match + # wins per target field" must not depend on set iteration order. + all_flat_keys = set() + for obj in parsed_lines: + all_flat_keys.update(_flatten(obj).keys()) + + field_mappings = [] + for ocsf_path, aliases in _JSON_FIELD_ALIASES.items(): + for flat_key in sorted(all_flat_keys): + leaf = flat_key.split(".")[-1].lower() + if leaf in aliases: + field_mappings.append(Mapping(source_field=flat_key, ocsf_path=ocsf_path)) + break # first match wins per target field + + if not field_mappings: + # A zero-mapping rule passes every validation check vacuously — a + # useless pending_review candidate that blocks the fingerprint until + # a human rejects it (T5-M2). Refuse instead. + raise GenerationError( + f"no mappable fields via alias table for fingerprint '{fingerprint_id}' " + f"(keys seen: {sorted(all_flat_keys)})") + + return GeneratedRule( + fingerprint_id=fingerprint_id, + pattern=JSON_SENTINEL, + mappings=field_mappings, + confidence=_json_confidence(sample_lines, field_mappings), + provenance="slm", + version=1, + ) + + +def _json_confidence(lines: list[str], mappings: list[Mapping]) -> float: + """Legacy _compute_json_confidence: fraction of lines that parse as JSON + AND have every mapped path resolve to a non-null value.""" + if not lines: + return 0.0 + hits = 0 + for line in lines: + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if all(_resolve_json_path(obj, m.source_field) is not None for m in mappings): + hits += 1 + return round(hits / len(lines), 2) + + +def _parse_extension_kv(extension_str: str) -> dict[str, str]: + """Extension-blob key=value scan via the shared ulpf_core grammar.""" + pairs: dict[str, str] = {} + if not extension_str: + return pairs + for match in EXTENSION_KV.finditer(extension_str): + key = match.group(1) + value = match.group(2) if match.group(2) is not None else match.group(3) + pairs[key] = value.strip() + return pairs + + +def _regex_confidence(pattern: str, mappings: list[Mapping], lines: list[str]) -> float: + """Legacy _compute_confidence semantics: fraction of lines where the + pattern matches AND every mapped field resolves to a non-empty value + (extension-blob keys included).""" + try: + compiled = re.compile(pattern) + except re.error: + return 0.0 + + if not lines: + return 0.0 + + expected_groups = [m.source_field for m in mappings] + hits = 0 + for line in lines: + match = compiled.search(line) + if not match: + continue + fields = dict(match.groupdict()) + extension_text = fields.get("extension") + if extension_text: + fields.update(_parse_extension_kv(extension_text)) + if all(fields.get(group) for group in expected_groups): + hits += 1 + return round(hits / len(lines), 2) + + +def generate_candidate(fingerprint_id: str, prompt_lines: list[str], client: SLMClient, + max_attempts: int = 3) -> GeneratedRule: + """Ask the SLM for a candidate rule, retrying with an escalated temperature + and an error-feedback turn after each malformed or non-matching response. + + Raises GenerationError after max_attempts (fail-closed — the app loop + records the attempt and leaves the hot path untouched). + """ + if not prompt_lines: + raise GenerationError(f"no prompt lines for fingerprint '{fingerprint_id}'") + + base_prompt = build_prompt(prompt_lines) + prompt = base_prompt + last_error: Exception | None = None + + for attempt in range(1, max_attempts + 1): + temperature = 0.1 + (attempt - 1) * 0.2 + try: + content = client.chat(prompt, temperature) + except Exception as exc: + # ANY failure from the transport call is a transport failure for + # retry purposes (T5-M3): builtin ConnectionError, but also + # httpx.ConnectError / ollama.ResponseError, which are NOT + # builtin subclasses — without this the ladder would fail fast + # on attempt 1. Counted like any failed attempt; never fed back + # as rule feedback. The exc_info log surfaces the outage and + # keeps the broad catch lint-legal (BLE001). + log.warning("SLM chat failed (attempt %d/%d): %s", attempt, max_attempts, + exc, exc_info=True) + last_error = exc + continue + try: + parsed = extract_json(content) + + pattern = parsed["pattern"] + raw_mappings = parsed["field_mappings"] + mappings = [ + Mapping(source_field=m["source_field"], ocsf_path=m["ocsf_path"]) + for m in raw_mappings + ] + + if pattern == JSON_SENTINEL: + if not mappings: + # Same vacuity as the fast-path (T5-M2): a sentinel rule + # with zero mappings passes every check vacuously — send + # it back through the feedback loop instead. + raise ValueError("JSON rule carried zero field mappings") + confidence = _json_confidence(prompt_lines, mappings) + else: + re.compile(pattern) # must compile before we trust it + confidence = _regex_confidence(pattern, mappings, prompt_lines) + + if confidence == 0.0: + raise ValueError( + "Generated rule yielded 0.0 confidence. It failed to match the " + "samples. Try a simpler regex.") + + return GeneratedRule( + fingerprint_id=fingerprint_id, + pattern=pattern, + mappings=mappings, + confidence=confidence, + provenance="slm", + version=1, + ) + except (json.JSONDecodeError, KeyError, TypeError, ValueError, re.error) as exc: + last_error = exc + prompt = (base_prompt + + f"\n\nYour previous response failed with error: {exc}\n" + "Please try again and fix the issue.") + continue + + raise GenerationError( + f"Failed to generate a valid rule for fingerprint '{fingerprint_id}' " + f"after {max_attempts} attempts. Last error: {last_error}") diff --git a/services/onboarding/reparse.py b/services/onboarding/reparse.py new file mode 100644 index 0000000..e738814 --- /dev/null +++ b/services/onboarding/reparse.py @@ -0,0 +1,213 @@ +# services/onboarding/reparse.py — R11 backlog re-parse: convergent sweep. +# +# When a rule is approved at version N, every normalized_events row for the +# fingerprint still counted under an OLDER version (or unparsed, version 0) is +# stale. This module finds such fingerprints, re-parses their raw backlog +# under the new version in batches, INSERTs the new-version rows (event_id +# minted exactly as the pipeline mints them: event_id_for(raw_id, version)), +# and supersedes the old rows via normalized_events.superseded_by_event_id — +# the single column-scoped UPDATE grant migration 003 gives onboarding_role, +# the only mutable cell in the event store. Insert + supersede share ONE +# transaction per batch, so a crash never leaves a duplicated or orphaned +# backlog row: the next poll re-finds whatever stayed stale and the sweep +# converges (a re-run over swept rows is a no-op by construction). +from __future__ import annotations + +import logging +import threading +import uuid +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING + +from ulpf_core.ids import event_id_for +from ulpf_core.models import Mapping, Rule +from ulpf_core.parsing import parse + +from onboarding.config import Config +from onboarding.store import audit, connect_db + +if TYPE_CHECKING: + import psycopg + +log = logging.getLogger("onboarding.reparse") + +# Active rules whose fingerprint still has unsuperseded rows parsed under an +# older version (COALESCE(rule_version, 0): unparsed rows are stale from any +# version). r.provenance is additive to the plan's STALE_SQL: Rule requires it +# (controller ruling — disclosed, not guessed). +STALE_SQL = """ +SELECT r.id, r.fingerprint_id, r.version, r.pattern, r.mappings, r.provenance +FROM rules r +WHERE r.status = 'active' AND EXISTS ( + SELECT 1 FROM normalized_events ne + WHERE ne.fingerprint_id = r.fingerprint_id + AND ne.superseded_by_event_id IS NULL + AND COALESCE(ne.rule_version, 0) < r.version) +""" +BATCH_SQL = """ +SELECT ne.event_id, ne.raw_id, ne.raw_received_at, raw.raw_text +FROM normalized_events ne +JOIN raw_events raw ON raw.raw_id = ne.raw_id AND raw.received_at = ne.raw_received_at +WHERE ne.fingerprint_id = %s AND ne.superseded_by_event_id IS NULL + AND COALESCE(ne.rule_version, 0) < %s +ORDER BY ne.raw_received_at +LIMIT %s +""" +SUPERSEDE_SQL = ("UPDATE normalized_events SET superseded_by_event_id = %s " + "WHERE event_id = %s AND parsed_at = %s") + +_STALE_KEYS = ("id", "fingerprint_id", "version", "pattern", "mappings", "provenance") + +# The staged-COPY insert below duplicates pipeline.db.persist_normalized's +# block (temp table -> COPY -> INSERT .. ON CONFLICT (event_id, parsed_at) DO +# NOTHING). Deliberate duplication, NOT an import: services don't cross-import +# (M1 pythonpath ruling — the pipeline package is not on the onboarding image), +# so the ~15-line staging helper lives here too. +_NE_COLUMNS = ( + "event_id", "raw_id", "raw_received_at", "parsed_at", "fingerprint_id", + "rule_id", "rule_version", "status", "ocsf", +) + + +@dataclass(frozen=True) +class _NewEvent: + """One new-version normalized_events row (field-for-field the pipeline's + NormalizedRow — duplicated for the same no-cross-import reason).""" + + event_id: uuid.UUID + raw_id: uuid.UUID + raw_received_at: datetime + parsed_at: datetime + fingerprint_id: str + rule_id: int + rule_version: int + status: str + ocsf: dict | None + + +def find_stale(conn: psycopg.Connection) -> list[dict]: + """Active rules with stale backlog, one dict per rule keyed by + _STALE_KEYS (id = the rules-table serial id, needed for attribution).""" + with conn.cursor() as cur: + cur.execute(STALE_SQL) + return [dict(zip(_STALE_KEYS, row)) for row in cur.fetchall()] + + +def _rule_from_row(rule_row: dict) -> Rule: + """Rebuild the typed Rule parse() consumes; mappings arrive from Postgres + as a JSON list of dicts.""" + return Rule( + fingerprint_id=rule_row["fingerprint_id"], + version=rule_row["version"], + pattern=rule_row["pattern"], + mappings=[Mapping(**m) for m in rule_row["mappings"]], + provenance=rule_row["provenance"], + ) + + +def sweep_one(conn: psycopg.Connection, rule_row: dict, batch_size: int) -> dict: + """Re-parse one fingerprint's stale backlog under its active rule. + + Loops BATCH_SQL until a batch returns < batch_size rows — the stale select + drains as rows get superseded, so the sweep terminates and is idempotent + (a second sweep finds nothing and does nothing). Per batch, ONE + transaction: staged-COPY INSERT of the new rows, then SUPERSEDE_SQL per + old row. Audits reparse_complete once per sweep that did work. + """ + from psycopg.types.json import Json # deferred: keeps psycopg off the test path + + rule = _rule_from_row(rule_row) + inserted = superseded = 0 + while True: + with conn.cursor() as cur: + cur.execute(BATCH_SQL, (rule.fingerprint_id, rule.version, batch_size)) + old_rows = cur.fetchall() + if not old_rows: + break + + new_events = [] + for old_event_id, raw_id, raw_received_at, raw_text in old_rows: + doc, err = parse(raw_text, rule) + if err is not None: + status, ocsf = "parse_error", None + else: + status, ocsf = "parsed", doc + if doc["time"] is None: + # Worker parity: parse() leaves time None when the rule + # maps no timestamp; the pipeline fills ingestion time + # (received_at) before persisting (ulpf_core.parsing). + doc["time"] = raw_received_at.isoformat() + new_events.append(_NewEvent( + event_id=event_id_for(raw_id, rule.version), + raw_id=raw_id, + raw_received_at=raw_received_at, + parsed_at=raw_received_at, # := raw received_at (deterministic) + fingerprint_id=rule.fingerprint_id, + rule_id=rule_row["id"], + rule_version=rule.version, + status=status, + ocsf=ocsf, + )) + + with conn.transaction(), conn.cursor() as cur: + cur.execute("CREATE TEMP TABLE staging_ne (LIKE normalized_events INCLUDING DEFAULTS) ON COMMIT DROP") + with cur.copy(f"COPY staging_ne ({', '.join(_NE_COLUMNS)}) FROM STDIN") as copy: + for ev in new_events: + copy.write_row(( + ev.event_id, ev.raw_id, ev.raw_received_at, ev.parsed_at, + ev.fingerprint_id, ev.rule_id, ev.rule_version, ev.status, + Json(ev.ocsf) if ev.ocsf is not None else None, + )) + cur.execute( + f"INSERT INTO normalized_events ({', '.join(_NE_COLUMNS)}) " + f"SELECT {', '.join(_NE_COLUMNS)} FROM staging_ne " + "ON CONFLICT (event_id, parsed_at) DO NOTHING" + ) + for (old_event_id, _raw_id, old_parsed_at, _raw_text), ev in zip(old_rows, new_events): + # old_parsed_at == the old row's parsed_at: parsed_at is ALWAYS + # the raw's received_at (spec §4 deterministic minting), so the + # PK (event_id, parsed_at) is keyed by what BATCH_SQL selected. + cur.execute(SUPERSEDE_SQL, (ev.event_id, old_event_id, old_parsed_at)) + + inserted += len(new_events) + superseded += len(old_rows) + if len(old_rows) < batch_size: + break + + if inserted or superseded: # audit once per sweep that did work + audit(conn, "reparse_complete", rule.fingerprint_id, { + "rule_id": rule_row["id"], + "version": rule.version, + "inserted": inserted, + "superseded": superseded, + }) + log.info("reparse sweep %s: v%d inserted=%d superseded=%d", + rule.fingerprint_id, rule.version, inserted, superseded) + return {"inserted": inserted, "superseded": superseded, "status": "complete"} + + +def run_reparse_loop(cfg: Config, stop: threading.Event) -> None: + """Poll find_stale and sweep each stale fingerprint, serially (cold path). + `stop.set()` ends the loop promptly (wait() is sliced by the event). Every + exception is logged and the loop continues — convergence self-heals: the + next poll re-finds whatever the failed sweep left stale.""" + log.info("onboarding reparse loop started (batch=%d, poll=%.1fs)", + cfg.reparse_batch, cfg.reparse_poll_s) + while not stop.is_set(): + try: + with connect_db(cfg.database_url) as conn: + stale = find_stale(conn) + for rule_row in stale: + if stop.is_set(): + break + try: + with connect_db(cfg.database_url) as conn: + sweep_one(conn, rule_row, cfg.reparse_batch) + except Exception: + log.exception("reparse sweep failed for %s; continuing", + rule_row.get("fingerprint_id")) + except Exception: + log.exception("reparse loop iteration failed; will retry") + stop.wait(cfg.reparse_poll_s) + log.info("onboarding reparse loop stopped") diff --git a/services/onboarding/requirements.txt b/services/onboarding/requirements.txt new file mode 100644 index 0000000..40cc43f --- /dev/null +++ b/services/onboarding/requirements.txt @@ -0,0 +1,3 @@ +psycopg[binary]==3.2.1 +regex~=2024.7 # controller R8: no cp313 wheel for ==2024.7.24; ~= matches ulpf-core's own pin +ollama>=0.5.0 # SLM client; >=0.5 for chat(think=False) (Task 11); lazy import diff --git a/services/onboarding/samples.py b/services/onboarding/samples.py new file mode 100644 index 0000000..1044455 --- /dev/null +++ b/services/onboarding/samples.py @@ -0,0 +1,26 @@ +# services/onboarding/samples.py — the deterministic prompt/held-out split +# (spec §6.2 anti-leakage fix). +# +# assign_roles re-splits the (captured_at, id)-ordered sample set from scratch +# on every run (controller ruling): first prompt_size rows -> prompt, next +# held_out_size -> held_out, the rest -> unused. Pure + deterministic, so a +# re-run before any new samples arrive reproduces the same split and +# store.mark_roles persists it idempotently (scoped by id lists). The split +# itself never leaks: a row id lands in exactly one role. +PROMPT_SIZE = 5 +HELD_OUT_SIZE = 15 + + +def assign_roles(rows: list[dict], prompt_size: int = PROMPT_SIZE, + held_out_size: int = HELD_OUT_SIZE) -> dict[str, list[dict]]: + """Split (captured_at, id)-ordered sample rows into the three roles. + + The optional size kwargs default to the Config defaults; app.py passes the + configured values through, tests and other callers can use the bare + `assign_roles(rows)` form. + """ + return { + "prompt": rows[:prompt_size], + "held_out": rows[prompt_size:prompt_size + held_out_size], + "unused": rows[prompt_size + held_out_size:], + } diff --git a/services/onboarding/store.py b/services/onboarding/store.py new file mode 100644 index 0000000..a24689c --- /dev/null +++ b/services/onboarding/store.py @@ -0,0 +1,193 @@ +# services/onboarding/store.py — all onboarding DB access (cold path). +# +# Every function takes a live connection (one conn per call: the app loop +# opens and closes a connection around each step) and imports psycopg lazily, +# so unit tests drive everything through fake conns — never a live DB. +# Connections run AUTOCOMMIT (the T7 house style): each multi-statement +# operation wraps exactly one `with conn.transaction():` block, single +# statements rely on autocommit. +# +# Grant shape (migration 003): onboarding_role has SELECT+INSERT on rules, +# INSERT on audit_log, SELECT+UPDATE(role) on onboarding_samples, and full +# CRUD on onboarding_attempts — every statement below stays inside that. +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from ulpf_core.validation import report_to_json + +if TYPE_CHECKING: + import psycopg + +log = logging.getLogger("onboarding.store") + +# Audit ruling: onboarding-generated events use actor = the created_by value. +CREATED_BY = "onboarding" + +# TOTAL samples per fingerprint (any role) against the threshold, no active +# rule, no pending candidate, and (no attempt yet OR enough new samples since +# the last attempt) — the retry_new_samples gate. +_READY_SQL = """ +SELECT s.fingerprint_id +FROM ( + SELECT fingerprint_id, count(*) AS total + FROM onboarding_samples + GROUP BY fingerprint_id +) s +LEFT JOIN onboarding_attempts a ON a.fingerprint_id = s.fingerprint_id +WHERE s.total >= %s + AND NOT EXISTS ( + SELECT 1 FROM rules r + WHERE r.fingerprint_id = s.fingerprint_id AND r.status = 'active') + AND NOT EXISTS ( + SELECT 1 FROM rules r + WHERE r.fingerprint_id = s.fingerprint_id AND r.status = 'pending_review') + AND (a.fingerprint_id IS NULL OR s.total - a.samples_seen >= %s) +ORDER BY s.fingerprint_id +""" + + +def connect_db(url: str) -> psycopg.Connection: + """psycopg connection for onboarding_role; autocommit per house style.""" + import psycopg # deferred: unit tests never need psycopg + + return psycopg.connect(url, autocommit=True) + + +def ready_fingerprints(conn: psycopg.Connection, cfg) -> list[str]: + """Fingerprint ids with enough samples to attempt, none blocked by an + active/pending rule, and (for repeat attempts) enough NEW samples.""" + with conn.cursor() as cur: + cur.execute(_READY_SQL, (cfg.sample_threshold, cfg.retry_new_samples)) + return [row[0] for row in cur.fetchall()] + + +def load_samples(conn: psycopg.Connection, fingerprint_id: str, limit: int) -> list[dict]: + """Up to `limit` sample rows for the fingerprint, ordered by + (captured_at, id) REGARDLESS of current role — the split is re-derived + from this ordered set on every run (controller ruling).""" + with conn.cursor() as cur: + cur.execute( + "SELECT id, raw_text FROM onboarding_samples " + "WHERE fingerprint_id = %s ORDER BY captured_at, id LIMIT %s", + (fingerprint_id, limit), + ) + return [{"id": row[0], "raw_text": row[1]} for row in cur.fetchall()] + + +def count_samples(conn: psycopg.Connection, fingerprint_id: str) -> int: + """TOTAL onboarding_samples rows for the fingerprint (any role) — the + same units _READY_SQL's s.total is counted in, so samples_seen + bookkeeping and the retry_new_samples throttle stay commensurate + (T5-F1: recording the capped loaded count defeated the throttle for + backlogged fingerprints).""" + with conn.cursor() as cur: + cur.execute( + "SELECT count(*) FROM onboarding_samples WHERE fingerprint_id = %s", + (fingerprint_id,), + ) + return cur.fetchone()[0] + + +def mark_roles(conn: psycopg.Connection, split: dict[str, list[dict]]) -> None: + """Persist the re-split scoped by id lists (idempotent on re-run). Always + three UPDATEs — an empty id list matches nothing but keeps the role + assignment complete (unused rows are forced back to 'unused').""" + with conn.transaction(), conn.cursor() as cur: + for role in ("prompt", "held_out", "unused"): + ids = [row["id"] for row in split.get(role, [])] + cur.execute( + "UPDATE onboarding_samples SET role = %s WHERE id = ANY(%s)", + (role, ids), + ) + log.info("roles marked: prompt=%d held_out=%d unused=%d", + len(split.get("prompt", [])), len(split.get("held_out", [])), + len(split.get("unused", []))) + + +def has_active_or_pending(conn: psycopg.Connection, + fingerprint_id: str) -> tuple[bool, bool]: + """(has_active, has_pending) for the fingerprint; the loop skips when + EITHER is true.""" + with conn.cursor() as cur: + cur.execute( + "SELECT status FROM rules " + "WHERE fingerprint_id = %s AND status IN ('active', 'pending_review')", + (fingerprint_id,), + ) + statuses = {row[0] for row in cur.fetchall()} + return "active" in statuses, "pending_review" in statuses + + +def insert_candidate(conn: psycopg.Connection, rule, report, + created_by: str = CREATED_BY) -> int: + """Store the candidate as ONE pending_review row and audit it. + + version = MAX(existing)+1 for the fingerprint (the rules UNIQUE + (fingerprint_id, version) contract), validation = the Task 3 report + verbatim via report_to_json, confidence from the GeneratedRule when the + rule carries one. The whole sequence (version select -> INSERT -> audit) + is one transaction. + """ + from psycopg.types.json import Json # deferred: keeps psycopg off the test path + + with conn.transaction(), conn.cursor() as cur: + cur.execute( + "SELECT COALESCE(MAX(version), 0) + 1 FROM rules WHERE fingerprint_id = %s", + (rule.fingerprint_id,), + ) + version = cur.fetchone()[0] + confidence = getattr(rule, "confidence", None) + cur.execute( + "INSERT INTO rules (fingerprint_id, version, pattern, mappings, provenance, " + "confidence, status, created_by, validation) " + "VALUES (%s, %s, %s, %s, %s, %s, 'pending_review', %s, %s) RETURNING id", + ( + rule.fingerprint_id, + version, + rule.pattern, + Json([{"source_field": m.source_field, "ocsf_path": m.ocsf_path} + for m in rule.mappings]), + rule.provenance, + confidence, + created_by, + Json(report_to_json(report)), + ), + ) + rule_id = cur.fetchone()[0] + audit(conn, "candidate_created", rule.fingerprint_id, { + "rule_id": rule_id, + "version": version, + "provenance": rule.provenance, + "confidence": confidence, + }, actor=created_by) + log.info("candidate #%d (v%d) for %s stored as pending_review", + rule_id, version, rule.fingerprint_id) + return rule_id + + +def record_attempt(conn: psycopg.Connection, fingerprint_id: str, + samples_seen: int, error: str | None) -> None: + """Upsert the generation-attempt bookkeeping row (one per fingerprint).""" + with conn.cursor() as cur: + cur.execute( + "INSERT INTO onboarding_attempts (fingerprint_id, samples_seen, error) " + "VALUES (%s, %s, %s) " + "ON CONFLICT (fingerprint_id) DO UPDATE SET " + "samples_seen = EXCLUDED.samples_seen, attempted_at = now(), " + "error = EXCLUDED.error", + (fingerprint_id, samples_seen, error), + ) + + +def audit(conn: psycopg.Connection, action: str, entity: str, detail: dict, + actor: str = CREATED_BY) -> None: + """One audit_log row; entity is the fingerprint_id (plain string).""" + from psycopg.types.json import Json # deferred: keeps psycopg off the test path + + with conn.cursor() as cur: + cur.execute( + "INSERT INTO audit_log (actor, action, entity, detail) VALUES (%s, %s, %s, %s)", + (actor, action, entity, Json(detail)), + ) diff --git a/services/onboarding/tests/__init__.py b/services/onboarding/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/onboarding/tests/test_app.py b/services/onboarding/tests/test_app.py new file mode 100644 index 0000000..901e9cf --- /dev/null +++ b/services/onboarding/tests/test_app.py @@ -0,0 +1,71 @@ +# services/onboarding/tests/test_app.py — wiring test for the failure path +# (T5-F1): samples_seen handed to record_attempt/audit must be the +# fingerprint's TOTAL sample count (store.count_samples), NOT the capped +# loaded count — otherwise the retry_new_samples throttle never engages for +# backlogged fingerprints. Minimal monkeypatch style: app.py imported its +# store collaborators by name, so patching the app module's globals is +# enough to observe the wiring. +from onboarding import app + + +class FakeStoreConn: + """Just the context-manager role of a psycopg connection.""" + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def test_failure_path_records_total_sample_count(monkeypatch): + seen = {} + + def fake_connect_db(url): + return FakeStoreConn() + + def fake_count_samples(conn, fp): + return 45 # TOTAL rows: e.g. 30 piled up while the SLM was down + + def fake_record_attempt(conn, fp, samples_seen, error): + seen["attempt"] = (fp, samples_seen, error) + + def fake_audit(conn, action, entity, detail, actor="onboarding"): + seen.setdefault("audits", []).append((action, entity, detail)) + + monkeypatch.setattr(app, "connect_db", fake_connect_db) + monkeypatch.setattr(app, "count_samples", fake_count_samples) + monkeypatch.setattr(app, "record_attempt", fake_record_attempt) + monkeypatch.setattr(app, "audit", fake_audit) + + app._record_failure(app.Config(), "fp_backlog", "validation failed: boom") + + assert seen["attempt"] == ("fp_backlog", 45, "validation failed: boom") + assert seen["audits"] == [ + ("candidate_failed", "fp_backlog", + {"error": "validation failed: boom", "samples_seen": 45}), + ] + + +def test_main_wires_both_convergent_loops(monkeypatch): + # Controller ruling: the onboarding service runs BOTH convergent loops + # (candidate generation + R11 backlog re-parse) as daemon threads sharing + # one stop handle. Both fakes return immediately, so main()'s join loop + # drains and returns — no threading harness needed. + invoked = [] + + def fake_candidate(cfg, stop): + invoked.append(("candidate", cfg, stop)) + + def fake_reparse(cfg, stop): + invoked.append(("reparse", cfg, stop)) + + monkeypatch.setattr(app, "run_candidate_loop", fake_candidate) + monkeypatch.setattr(app, "run_reparse_loop", fake_reparse) + + app.main() + + assert sorted(name for name, _, _ in invoked) == ["candidate", "reparse"] + # One Config and ONE shared threading.Event stop handle across both loops. + assert len({id(cfg) for _, cfg, _ in invoked}) == 1 + assert len({id(stop) for _, _, stop in invoked}) == 1 diff --git a/services/onboarding/tests/test_config.py b/services/onboarding/tests/test_config.py new file mode 100644 index 0000000..0af6e9a --- /dev/null +++ b/services/onboarding/tests/test_config.py @@ -0,0 +1,37 @@ +# services/onboarding/tests/test_config.py — pins the binding compose ruling +# (SLM tier qwen3:4b behind ollama-4b, compose-internal onboarding DSN, and +# the exact env names Task 11 sets in docker-compose). +from onboarding.config import Config + + +def test_defaults_match_the_compose_ruling(): + cfg = Config() + assert cfg.database_url == "postgresql://onboarding_role:onboarding_dev@postgres:5432/ulpf" + assert cfg.ollama_url == "http://ollama-4b:11434" + assert cfg.ollama_model == "qwen3:4b" + assert cfg.sample_threshold == 20 + assert cfg.prompt_size == 5 + assert cfg.held_out_size == 15 + assert cfg.retry_new_samples == 10 + assert cfg.onboarding_poll_s == 5.0 + assert cfg.reparse_poll_s == 2.0 + assert cfg.reparse_batch == 500 + + +def test_from_env_reads_exactly_the_three_compose_names(monkeypatch): + monkeypatch.setenv("DATABASE_URL", "postgresql://x:x@db:5432/ulpf") + monkeypatch.setenv("OLLAMA_URL", "http://slm:11434") + monkeypatch.setenv("OLLAMA_MODEL", "other:7b") + cfg = Config.from_env() + assert cfg.database_url == "postgresql://x:x@db:5432/ulpf" + assert cfg.ollama_url == "http://slm:11434" + assert cfg.ollama_model == "other:7b" + # Non-env fields keep their defaults. + assert cfg.sample_threshold == 20 + + +def test_from_env_without_env_keeps_defaults(monkeypatch): + for name in ("DATABASE_URL", "OLLAMA_URL", "OLLAMA_MODEL"): + monkeypatch.delenv(name, raising=False) + cfg = Config.from_env() + assert cfg == Config() diff --git a/services/onboarding/tests/test_generate.py b/services/onboarding/tests/test_generate.py new file mode 100644 index 0000000..657e474 --- /dev/null +++ b/services/onboarding/tests/test_generate.py @@ -0,0 +1,173 @@ +# services/onboarding/tests/test_generate.py — SLM prompt/extraction/retry +# semantics (ported from legacy-demo/onboarding/generate_rule.py) + the JSON +# alias fast-path. +# +# The SLM is faked with recording clients (the SLMClient Protocol); the +# `ollama` package is NEVER imported on the test path (OllamaClient defers it +# into .chat()), so this module runs on a host without ollama installed. +import json + +import pytest +from ulpf_core.models import Rule +from ulpf_core.parsing import JSON_SENTINEL + +from onboarding.generate import ( + GeneratedRule, + GenerationError, + build_prompt, + extract_json, + generate_candidate, + generate_json_rule, +) + +_SAMPLES = [ + ("LEEF:2.0|Acme|NetGuard|3.2|401|src=198.51.100.7 srcPort=44521 dst=192.168.1.5 " + "dstPort=443 proto=TCP action=allow msg=Session started"), + ("LEEF:2.0|Acme|NetGuard|3.2|402|src=203.0.113.9 srcPort=51002 dst=192.168.1.6 " + "dstPort=22 proto=TCP action=deny msg=Blocked login attempt"), +] + +_VALID = json.dumps({ + "pattern": r"^.*\|(?P.*)$", + "field_mappings": [ + {"source_field": "src", "ocsf_path": "src_endpoint.ip"}, + {"source_field": "msg", "ocsf_path": "message"}, + ], +}) + + +class FakeSLM: + """Replays scripted responses; records (prompt, temperature) per call.""" + + def __init__(self, responses): + self.responses = list(responses) + self.calls: list[tuple[str, float]] = [] + + def chat(self, prompt: str, temperature: float) -> str: + self.calls.append((prompt, temperature)) + return self.responses.pop(0) + + +class DownSLM: + """The SLM host is unreachable: every call raises ConnectionError.""" + + def __init__(self): + self.calls = 0 + + def chat(self, prompt: str, temperature: float) -> str: + self.calls += 1 + raise ConnectionError("ollama unreachable") + + +class BrokenSLM: + """A NON-ConnectionError transport failure (httpx.ConnectError and + ollama.ResponseError are not builtin subclasses) on every call.""" + + def __init__(self): + self.temperatures: list[float] = [] + + def chat(self, prompt: str, temperature: float) -> str: + self.temperatures.append(temperature) + raise RuntimeError("some other transport boom") + + +def test_build_prompt_contains_few_shots_and_samples(): + prompt = build_prompt(_SAMPLES) + assert "LEEF:2.0|Acme|NetGuard" in prompt # few-shot example 1 (LEEF) + assert "FW-DROP" in prompt # few-shot example 2 (iptables prefix) + assert _SAMPLES[0] in prompt and _SAMPLES[1] in prompt + assert "src_endpoint.ip" in prompt # the OCSF target list + assert "hardcode" in prompt # anti-hardcoding rule 4 + + +def test_extract_json_strips_fences_and_grabs_outermost_object(): + assert extract_json('```json\n{"pattern": "x"}\n```') == {"pattern": "x"} + assert extract_json(' Sure! {"a": {"b": 1}} hope that helps ') == {"a": {"b": 1}} + with pytest.raises(ValueError): + extract_json("no object here") + + +def test_generate_candidate_retries_with_error_feedback_and_escalating_temp(): + fake = FakeSLM(["totally not json", _VALID]) + rule = generate_candidate("fp_x", _SAMPLES, fake) + + assert len(fake.calls) == 2 # exactly one retry + assert [t for _, t in fake.calls] == pytest.approx([0.1, 0.3]) + # Error feedback: the second prompt carries the first attempt's failure. + assert "failed with error" in fake.calls[1][0] + assert "totally not json" in fake.calls[1][0] + + assert isinstance(rule, Rule) + assert isinstance(rule, GeneratedRule) + assert rule.provenance == "slm" + assert rule.pattern == r"^.*\|(?P.*)$" + assert rule.confidence == 1.0 # every sample matches and all fields resolve + + +def test_generate_candidate_wraps_connection_error_after_max_attempts(): + fake = DownSLM() + with pytest.raises(GenerationError): + generate_candidate("fp_x", _SAMPLES, fake, max_attempts=3) + assert fake.calls == 3 + + +def test_generate_candidate_retries_any_transport_error_type(): + fake = BrokenSLM() + with pytest.raises(GenerationError): + generate_candidate("fp_x", _SAMPLES, fake, max_attempts=3) + assert fake.temperatures == pytest.approx([0.1, 0.3, 0.5]) + + +def test_generate_candidate_without_lines_raises_generation_error(): + with pytest.raises(GenerationError): + generate_candidate("fp_x", [], FakeSLM([_VALID])) + + +def test_generate_candidate_ladder_on_sentinel_zero_mappings(): + # T5-M2 through the SLM ladder: a valid-JSON response carrying the + # __JSON__ sentinel but ZERO field mappings is the same vacuity as the + # fast path's — fed back through the escalating-temperature ladder, and + # when every attempt fails the same way the outcome is GenerationError. + zero_map = json.dumps({"pattern": JSON_SENTINEL, "field_mappings": []}) + fake = FakeSLM([zero_map, zero_map, zero_map]) + with pytest.raises(GenerationError, match="zero field mappings"): + generate_candidate("fp_x", _SAMPLES, fake, max_attempts=3) + + assert [t for _, t in fake.calls] == pytest.approx([0.1, 0.3, 0.5]) + # The retry prompt carries the previous attempt's failure (feedback loop). + assert "zero field mappings" in fake.calls[1][0] + + +def test_generate_json_rule_maps_aliases_and_sets_sentinel(): + lines = [ + ('{"src_ip": "10.0.0.1", "dst_ip": "10.0.0.2", "src_port": 443, ' + '"dst_port": 80, "msg": "hello"}'), + ('{"src_ip": "10.0.0.3", "dst_ip": "10.0.0.4", "src_port": 22, ' + '"dst_port": 8080, "msg": "blocked"}'), + ] + rule = generate_json_rule("fp_json", lines) + + assert isinstance(rule, Rule) + assert rule.pattern == JSON_SENTINEL + assert rule.provenance == "slm" + assert {m.source_field: m.ocsf_path for m in rule.mappings} == { + "src_ip": "src_endpoint.ip", + "dst_ip": "dst_endpoint.ip", + "src_port": "src_endpoint.port", + "dst_port": "dst_endpoint.port", + "msg": "message", + } + assert rule.confidence == 1.0 + + +def test_generate_json_rule_requires_parseable_lines(): + with pytest.raises(ValueError): + generate_json_rule("fp_json", ["not json at all"]) + + +def test_generate_json_rule_rejects_zero_mappings(): + # A JSON format hitting none of the 8 alias entries would yield + # mappings=[] — every gate check is vacuously true for it, so the rule + # must be refused instead of stored as a useless pending_review row. + with pytest.raises(GenerationError, match="no mappable fields via alias table"): + generate_json_rule("fp_json", ['{"widget": "x"}', '{"widget": "y"}']) diff --git a/services/onboarding/tests/test_reparse.py b/services/onboarding/tests/test_reparse.py new file mode 100644 index 0000000..7a2e856 --- /dev/null +++ b/services/onboarding/tests/test_reparse.py @@ -0,0 +1,259 @@ +# services/onboarding/tests/test_reparse.py — R11 backlog re-parse sweep, SQL +# pinned through a recording fake conn/cursor (the M1 db-test style; NEVER a +# live DB). Same shape as test_store.py plus a `copy()` on the fake cursor so +# the staged-COPY insert (the pipeline.db.persist_normalized pattern, +# duplicated per the no-cross-import ruling) is observable: every write_row +# lands in conn.copied in call order, psycopg's Json wrapper unwrapped via .obj. +import uuid +from datetime import UTC, datetime + +from ulpf_core.ids import event_id_for +from ulpf_core.models import Mapping, Rule +from ulpf_core.parsing import parse + +from onboarding.reparse import SUPERSEDE_SQL, find_stale, sweep_one + +RECEIVED_AT = datetime(2026, 9, 19, 12, 0, 0, tzinfo=UTC) + + +class FakeCopy: + def __init__(self, conn): + self._conn = conn + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def write_row(self, row): + self._conn.copied.append(tuple(row)) + + +class FakeConn: + """Fake psycopg connection: records SQL, replays scripted fetch results.""" + + def __init__(self, fetchall_results=()): + self.sql: list = [] + self.params: list = [] + self.copied: list = [] + self.txns_opened = 0 + self._fetchall = list(fetchall_results) + + def transaction(self): + conn = self + + class Txn: + def __enter__(self): + conn.txns_opened += 1 + conn.sql.append("TXN-START") + conn.params.append(None) # keep sql/params index-aligned + + def __exit__(self, exc_type, exc, tb): + conn.sql.append("TXN-ROLLBACK" if exc_type is not None else "TXN-COMMIT") + conn.params.append(None) + return False + + return Txn() + + def cursor(self): + conn = self + + class Cur: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params=None): + conn.sql.append(sql) + conn.params.append(params) + + def fetchall(self): + return list(conn._fetchall.pop(0)) if conn._fetchall else [] + + def copy(self, sql): + conn.sql.append(sql) + conn.params.append(None) + return FakeCopy(conn) + + return Cur() + + +def _unwrap(value): + """psycopg's Json adapter -> plain Python (duck-typed: no psycopg here).""" + return value.obj if hasattr(value, "obj") else value + + +def _rule_row(**over): + """One find_stale row (dict). provenance rides along per the controller + ruling: ulpf_core.models.Rule requires it, so STALE_SQL selects it too.""" + row = { + "id": 7, + "fingerprint_id": "fp_a", + "version": 2, + "pattern": r"(?P.*)", + "mappings": [{"source_field": "message", "ocsf_path": "message"}], + "provenance": "slm", + } + row.update(over) + return row + + +def _old_row(n=0, raw_text="host1 sshd: accepted"): + """One BATCH_SQL fetch row: (event_id, raw_id, raw_received_at, raw_text).""" + return ( + uuid.uuid5(uuid.NAMESPACE_URL, f"old-event-{n}"), + uuid.uuid5(uuid.NAMESPACE_URL, f"raw-{n}"), + RECEIVED_AT, + raw_text, + ) + + +# --- (a) find_stale: active rules over unsuperseded older-version rows ------- + + +def test_find_stale_selects_active_rules_over_unsuperseded_older_rows(): + conn = FakeConn(fetchall_results=[[ + (7, "fp_a", 2, "pat", [{"source_field": "m", "ocsf_path": "message"}], "slm"), + ]]) + rows = find_stale(conn) + + assert rows == [{ + "id": 7, + "fingerprint_id": "fp_a", + "version": 2, + "pattern": "pat", + "mappings": [{"source_field": "m", "ocsf_path": "message"}], + "provenance": "slm", + }] + # Whitespace-normalized: the predicate must keep the partial-index shape. + sql = " ".join(next(s for s in conn.sql if isinstance(s, str)).split()) + assert "FROM rules r" in sql + assert "r.status = 'active'" in sql + assert ("ne.superseded_by_event_id IS NULL AND " + "COALESCE(ne.rule_version, 0) < r.version") in sql + assert "EXISTS" in sql + assert "ne.fingerprint_id = r.fingerprint_id" in sql + assert conn.params[conn.sql.index(next( + s for s in conn.sql if isinstance(s, str)))] is None + + +# --- (b) sweep_one: INSERT + supersede in ONE txn, then reparse_complete ----- + + +def test_sweep_one_inserts_new_row_and_supersedes_in_one_txn_then_audits(): + conn = FakeConn(fetchall_results=[[_old_row()]]) + result = sweep_one(conn, _rule_row(), batch_size=500) + + assert result == {"inserted": 1, "superseded": 1, "status": "complete"} + sql = [s for s in conn.sql if isinstance(s, str)] + create = next(s for s in sql if s.startswith("CREATE TEMP TABLE staging_ne")) + copy_sql = next(s for s in sql if s.startswith("COPY staging_ne")) + insert_sql = next(s for s in sql if s.startswith("INSERT INTO normalized_events")) + supersede_ix = sql.index(SUPERSEDE_SQL) + audit_sql = next(s for s in sql if s.startswith("INSERT INTO audit_log")) + # One txn: staging COPY -> INSERT -> supersede; the audit follows the commit. + assert conn.txns_opened == 1 + assert (sql.index("TXN-START") < sql.index(create) < sql.index(copy_sql) + < sql.index(insert_sql) < supersede_ix < sql.index("TXN-COMMIT") + < sql.index(audit_sql)) + assert "ON CONFLICT (event_id, parsed_at) DO NOTHING" in insert_sql + + # New row: deterministic event_id, worker-parity fields. + assert len(conn.copied) == 1 + (event_id, raw_id, raw_received_at, parsed_at, fp, rule_id, rule_version, + status, _ocsf) = conn.copied[0] + assert event_id == event_id_for(raw_id, 2) + assert raw_id == _old_row()[1] + assert raw_received_at == RECEIVED_AT + assert parsed_at == RECEIVED_AT # parsed_at := raw received_at + assert (fp, rule_id, rule_version, status) == ("fp_a", 7, 2, "parsed") + + # The column-scoped UPDATE, old row keyed by (event_id, parsed_at). + assert conn.params[supersede_ix] == (event_id, _old_row()[0], RECEIVED_AT) + + # One reparse_complete audit for the sweep, counts in the detail. + assert conn.params[sql.index(audit_sql)][:3] == ("onboarding", "reparse_complete", "fp_a") + detail = _unwrap(conn.params[sql.index(audit_sql)][3]) + assert detail == {"rule_id": 7, "version": 2, "inserted": 1, "superseded": 1} + + +# --- (c) raw that fails under the new version: parse_error + still supersede - + + +def test_sweep_one_parse_failure_stores_parse_error_and_still_supersedes(): + conn = FakeConn(fetchall_results=[[_old_row()]]) + result = sweep_one(conn, _rule_row(pattern=r"^NEVER-MATCH$"), batch_size=500) + + assert result == {"inserted": 1, "superseded": 1, "status": "complete"} + (event_id, _raw_id, _raw_received_at, _parsed_at, _fp, rule_id, rule_version, + status, ocsf) = conn.copied[0] + # The parse_error row is attributed to the new rule_id/version. + assert (event_id, rule_id, rule_version, status, ocsf) == ( + event_id_for(_old_row()[1], 2), 7, 2, "parse_error", None) + # ...and the old row is still superseded, inside the same txn. + supersede_ix = next(i for i, s in enumerate(conn.sql) if s == SUPERSEDE_SQL) + assert conn.txns_opened == 1 + assert conn.sql.index("TXN-START") < supersede_ix < conn.sql.index("TXN-COMMIT") + assert conn.params[supersede_ix] == (event_id, _old_row()[0], RECEIVED_AT) + + +# --- (d) idempotency: nothing stale -> nothing inserted, updated or audited -- + + +def test_sweep_one_is_idempotent_once_nothing_is_stale(): + # The fake models the predicate: sweep #1 supersedes the row, so the same + # stale-select drains and sweep #2 must insert/update/audit nothing. + conn = FakeConn(fetchall_results=[[_old_row()], [], []]) + first = sweep_one(conn, _rule_row(), batch_size=500) + second = sweep_one(conn, _rule_row(), batch_size=500) + + assert first == {"inserted": 1, "superseded": 1, "status": "complete"} + assert second == {"inserted": 0, "superseded": 0, "status": "complete"} + assert len(conn.copied) == 1 + assert sum(1 for s in conn.sql if s == SUPERSEDE_SQL) == 1 + audits = [s for s in conn.sql + if isinstance(s, str) and s.startswith("INSERT INTO audit_log")] + assert len(audits) == 1 + + +# --- (e) doc["time"] is None -> filled with the raw received_at (parity) ----- + + +def test_sweep_one_fills_missing_doc_time_with_raw_received_at(): + # Baseline: the rule maps no timestamp, so parse() itself leaves time None + # — the fill below is the sweep's doing, exactly like the pipeline worker. + rule = Rule( + fingerprint_id="fp_a", version=2, pattern=r"(?P.*)", + mappings=[Mapping(source_field="message", ocsf_path="message")], + provenance="slm", + ) + doc, err = parse(_old_row()[3], rule) + assert err is None and doc["time"] is None + + conn = FakeConn(fetchall_results=[[_old_row()]]) + sweep_one(conn, _rule_row(), batch_size=500) + + assert conn.copied[0][7] == "parsed" + assert _unwrap(conn.copied[0][8])["time"] == RECEIVED_AT.isoformat() + + +# --- batch drain: full batches loop, one audit per sweep --------------------- + + +def test_sweep_one_loops_full_batches_and_audits_once_for_the_sweep(): + conn = FakeConn(fetchall_results=[ + [_old_row(1, "line one")], [_old_row(2, "line two")], [], + ]) + result = sweep_one(conn, _rule_row(), batch_size=1) + + assert result == {"inserted": 2, "superseded": 2, "status": "complete"} + assert conn.txns_opened == 2 # one txn per batch + assert len(conn.copied) == 2 + audits = [s for s in conn.sql + if isinstance(s, str) and s.startswith("INSERT INTO audit_log")] + assert len(audits) == 1 + detail = _unwrap(conn.params[conn.sql.index(audits[0])][3]) + assert detail == {"rule_id": 7, "version": 2, "inserted": 2, "superseded": 2} diff --git a/services/onboarding/tests/test_samples.py b/services/onboarding/tests/test_samples.py new file mode 100644 index 0000000..747bd3d --- /dev/null +++ b/services/onboarding/tests/test_samples.py @@ -0,0 +1,33 @@ +# services/onboarding/tests/test_samples.py — the deterministic prompt / +# held-out split (spec §6.2 anti-leakage guarantee). +# +# assign_roles consumes the (captured_at, id)-ordered row list load_samples +# produces and re-splits it from scratch: first prompt_size rows -> prompt, +# next held_out_size -> held_out, the rest -> unused. +from onboarding.samples import assign_roles + + +def _rows(n): + # Already ordered by (captured_at, id) — the load_samples contract. + return [{"id": i, "raw_text": f"line {i}"} for i in range(n)] + + +def test_twenty_rows_split_5_15_0(): + split = assign_roles(_rows(20)) + assert set(split) == {"prompt", "held_out", "unused"} + assert [r["id"] for r in split["prompt"]] == list(range(5)) + assert [r["id"] for r in split["held_out"]] == list(range(5, 20)) + assert split["unused"] == [] + + +def test_twenty_five_rows_split_5_15_5(): + split = assign_roles(_rows(25)) + assert [r["id"] for r in split["prompt"]] == list(range(5)) + assert [r["id"] for r in split["held_out"]] == list(range(5, 20)) + assert [r["id"] for r in split["unused"]] == list(range(20, 25)) + + +def test_split_is_non_overlapping_by_id(): + split = assign_roles(_rows(23)) + flat = [r["id"] for role in ("prompt", "held_out", "unused") for r in split[role]] + assert len(flat) == len(set(flat)) == 23 # leakage guarantee: no id in two roles diff --git a/services/onboarding/tests/test_store.py b/services/onboarding/tests/test_store.py new file mode 100644 index 0000000..afc4b0c --- /dev/null +++ b/services/onboarding/tests/test_store.py @@ -0,0 +1,271 @@ +# services/onboarding/tests/test_store.py — store SQL pinned through a +# recording fake conn/cursor (the M1 pipeline db-test style; NEVER a live DB). +# +# The fake records (sql, params) in execution order and brackets each +# `with conn.transaction():` block with TXN-START / TXN-COMMIT (or +# TXN-ROLLBACK) markers, so "version select -> rules INSERT -> audit INSERT, +# all inside one txn" is an assertion on the recorded sequence. Fetch results +# are consumed FIFO so one call can drive several queries. psycopg is not on +# the test path, so its Json wrapper is unwrapped duck-typed via .obj. +from ulpf_core.models import Mapping +from ulpf_core.validation import CandidateReport, report_to_json + +from onboarding.config import Config +from onboarding.generate import GeneratedRule +from onboarding.store import ( + audit, + count_samples, + has_active_or_pending, + insert_candidate, + load_samples, + mark_roles, + ready_fingerprints, + record_attempt, +) + + +class FakeConn: + """Fake psycopg connection: records SQL, replays scripted fetch results.""" + + def __init__(self, fetchone_results=(), fetchall_results=()): + self.sql: list = [] + self.params: list = [] + self.txns_opened = 0 + self._fetchone = list(fetchone_results) + self._fetchall = list(fetchall_results) + + def transaction(self): + conn = self + + class Txn: + def __enter__(self): + conn.txns_opened += 1 + conn.sql.append("TXN-START") + conn.params.append(None) # keep sql/params index-aligned + + def __exit__(self, exc_type, exc, tb): + conn.sql.append("TXN-ROLLBACK" if exc_type is not None else "TXN-COMMIT") + conn.params.append(None) + return False + + return Txn() + + def cursor(self): + conn = self + + class Cur: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params=None): + conn.sql.append(sql) + conn.params.append(params) + + def fetchone(self): + return list(conn._fetchone.pop(0)) if conn._fetchone else [1] + + def fetchall(self): + return list(conn._fetchall.pop(0)) if conn._fetchall else [] + + return Cur() + + +def _unwrap(value): + """psycopg's Json adapter -> plain Python (duck-typed: no psycopg here).""" + return value.obj if hasattr(value, "obj") else value + + +def _rule(confidence=0.93): + return GeneratedRule( + fingerprint_id="fp_a", + version=1, + pattern=r"^.*\|(?P.*)$", + mappings=[Mapping(source_field="src", ocsf_path="src_endpoint.ip")], + provenance="slm", + confidence=confidence, + ) + + +def _report(passed=True): + checks = { + "caps_and_allowlist", "samples_parse", "held_out_match_all", "ip_fields_valid", + "port_fields_valid", "no_orphan_mappings", "adversarial_probe", + "no_hardcoded_literals", + } + return CandidateReport( + passed=passed, + checks={key: passed for key in checks}, + held_out_match_rate=1.0 if passed else 0.4, + notes=["held-out line 3: boom"] if not passed else [], + previews=[], + prompt_count=5, + held_out_count=15, + ) + + +# --- insert_candidate: pending_review row + version + validation + audit ----- + + +def test_insert_candidate_pends_with_next_version_and_audits(): + conn = FakeConn(fetchone_results=[[1], [42]]) # version select, RETURNING id + rule_id = insert_candidate(conn, _rule(), _report()) + + assert rule_id == 42 + sql = [s for s in conn.sql if isinstance(s, str)] + version_sql = next(s for s in sql if "COALESCE(MAX(version), 0) + 1" in s) + insert_sql = next(s for s in sql if s.startswith("INSERT INTO rules")) + audit_sql = next(s for s in sql if s.startswith("INSERT INTO audit_log")) + # One txn: version select -> rules INSERT -> audit INSERT. + assert conn.txns_opened == 1 + assert (sql.index("TXN-START") < sql.index(version_sql) < sql.index(insert_sql) + < sql.index(audit_sql) < sql.index("TXN-COMMIT")) + + assert conn.params[sql.index(version_sql)] == ("fp_a",) + assert "pending_review" in insert_sql and "validation" in insert_sql + ip = [_unwrap(v) for v in conn.params[sql.index(insert_sql)]] + # (fingerprint_id, version, pattern, mappings, provenance, confidence, + # created_by, validation) + assert ip[0] == "fp_a" and ip[1] == 1 + assert ip[2] == r"^.*\|(?P.*)$" + assert ip[3] == [{"source_field": "src", "ocsf_path": "src_endpoint.ip"}] + assert ip[4] == "slm" and ip[5] == 0.93 and ip[6] == "onboarding" + assert ip[7] == report_to_json(_report()) + + assert conn.params[sql.index(audit_sql)][:3] == ( + "onboarding", "candidate_created", "fp_a") + detail = _unwrap(conn.params[sql.index(audit_sql)][3]) + assert detail == {"rule_id": 42, "version": 1, "provenance": "slm", + "confidence": 0.93} + + +def test_insert_candidate_without_confidence_stores_null(): + from ulpf_core.models import Rule + + plain = Rule( + fingerprint_id="fp_a", version=1, pattern="x", + mappings=[Mapping(source_field="s", ocsf_path="message")], provenance="slm", + ) + conn = FakeConn(fetchone_results=[[3], [7]]) + assert insert_candidate(conn, plain, _report()) == 7 + sql = [s for s in conn.sql if isinstance(s, str)] + insert_sql = next(s for s in sql if s.startswith("INSERT INTO rules")) + assert conn.params[sql.index(insert_sql)][5] is None + + +# --- ready_fingerprints: threshold + no active + no pending + retry gate ----- + + +def test_ready_fingerprints_gates_on_threshold_rules_and_new_samples(): + conn = FakeConn(fetchall_results=[[("fp_a",), ("fp_b",)]]) + cfg = Config() + fps = ready_fingerprints(conn, cfg) + + assert fps == ["fp_a", "fp_b"] + sql = next(s for s in conn.sql if isinstance(s, str) and "onboarding_samples" in s) + # TOTAL samples per fingerprint (any role) against the threshold. + assert "count(*)" in sql and "GROUP BY fingerprint_id" in sql + assert "s.total >= %s" in sql + assert "status = 'active'" in sql and "status = 'pending_review'" in sql + assert sql.count("NOT EXISTS") == 2 + # No attempt row yet, OR enough new samples since the last attempt. + assert "a.fingerprint_id IS NULL OR s.total - a.samples_seen >= %s" in sql + assert "JOIN onboarding_attempts" in sql + assert conn.params[conn.sql.index(sql)] == (cfg.sample_threshold, cfg.retry_new_samples) + + +# --- mark_roles: three id-scoped UPDATEs, rest forced unused ------------------ + + +def test_mark_roles_issues_three_scoped_updates(): + conn = FakeConn() + split = { + "prompt": [{"id": 1}, {"id": 2}], + "held_out": [{"id": 3}, {"id": 4}, {"id": 5}], + "unused": [{"id": 6}], + } + mark_roles(conn, split) + + # Positional pairing: the three UPDATE statements are identical strings. + updates = [(s, p) for s, p in zip(conn.sql, conn.params) + if isinstance(s, str) and s.startswith("UPDATE onboarding_samples")] + assert len(updates) == 3 + assert updates[0] == ( + "UPDATE onboarding_samples SET role = %s WHERE id = ANY(%s)", ("prompt", [1, 2])) + assert updates[1][1] == ("held_out", [3, 4, 5]) + assert updates[2][1] == ("unused", [6]) + # One txn around all three: the split lands atomically. + assert conn.txns_opened == 1 + assert conn.sql[0] == "TXN-START" and conn.sql[-1] == "TXN-COMMIT" + + +def test_mark_roles_forces_unused_update_even_when_empty(): + conn = FakeConn() + mark_roles(conn, {"prompt": [{"id": 1}], "held_out": [{"id": 2}], "unused": []}) + updates = [(s, p) for s, p in zip(conn.sql, conn.params) + if isinstance(s, str) and s.startswith("UPDATE onboarding_samples")] + assert len(updates) == 3 + assert updates[2][1] == ("unused", []) + + +# --- load_samples: ordered by (captured_at, id), role-blind, limited ---------- + + +def test_load_samples_selects_ordered_and_limited_regardless_of_role(): + conn = FakeConn(fetchall_results=[[(7, "line a"), (3, "line b")]]) + rows = load_samples(conn, "fp_a", 20) + + assert rows == [{"id": 7, "raw_text": "line a"}, {"id": 3, "raw_text": "line b"}] + sql = next(s for s in conn.sql if isinstance(s, str)) + assert "WHERE fingerprint_id = %s" in sql + assert "ORDER BY captured_at, id" in sql + assert "LIMIT %s" in sql + assert conn.params[conn.sql.index(sql)] == ("fp_a", 20) + + +# --- has_active_or_pending ----------------------------------------------------- + + +def test_has_active_or_pending_reports_both_flags(): + conn = FakeConn(fetchall_results=[[("active", 1)]]) + assert has_active_or_pending(conn, "fp_a") == (True, False) + + conn = FakeConn(fetchall_results=[[("pending_review", 2)]]) + assert has_active_or_pending(conn, "fp_a") == (False, True) + + conn = FakeConn(fetchall_results=[[]]) + assert has_active_or_pending(conn, "fp_a") == (False, False) + sql = next(s for s in conn.sql if isinstance(s, str) and "rules" in s) + assert "status IN ('active', 'pending_review')" in sql + + +# --- record_attempt + audit ---------------------------------------------------- + + +def test_record_attempt_upserts_the_attempt_row(): + conn = FakeConn() + record_attempt(conn, "fp_a", 20, "validation failed: boom") + sql = next(s for s in conn.sql if isinstance(s, str)) + assert sql.startswith("INSERT INTO onboarding_attempts") + assert "ON CONFLICT (fingerprint_id) DO UPDATE" in sql + assert conn.params[conn.sql.index(sql)] == ("fp_a", 20, "validation failed: boom") + + +def test_count_samples_counts_total_for_fingerprint(): + conn = FakeConn(fetchone_results=[[45]]) + assert count_samples(conn, "fp_a") == 45 + sql = next(s for s in conn.sql if isinstance(s, str)) + assert sql == "SELECT count(*) FROM onboarding_samples WHERE fingerprint_id = %s" + assert conn.params[conn.sql.index(sql)] == ("fp_a",) + + +def test_audit_inserts_row_with_actor_entity_detail(): + conn = FakeConn() + audit(conn, "samples_split", "fp_a", {"prompt": 5, "held_out": 15, "unused": 0}) + sql = next(s for s in conn.sql if isinstance(s, str)) + assert sql == "INSERT INTO audit_log (actor, action, entity, detail) VALUES (%s, %s, %s, %s)" + actor, action, entity, detail = (_unwrap(v) for v in conn.params[conn.sql.index(sql)]) + assert (actor, action, entity) == ("onboarding", "samples_split", "fp_a") + assert detail == {"prompt": 5, "held_out": 15, "unused": 0} diff --git a/services/pipeline/db.py b/services/pipeline/db.py index 73389fd..c5382c2 100644 --- a/services/pipeline/db.py +++ b/services/pipeline/db.py @@ -8,10 +8,14 @@ # raw_batches honestly records the re-delivery (Task 12 Assert D). # # psycopg is imported lazily inside the persist functions only, so unit tests -# exercise build_rows()/the dataclasses without psycopg installed (mirrors the -# collector's deferred confluent_kafka import). persist_* commit on success and -# leave the transaction aborted on failure — the worker rolls back and does NOT -# commit offsets, so failed batches are redelivered (at-least-once). +# exercise build_rows()/collect_samples() without psycopg installed (mirrors +# the collector's deferred confluent_kafka import). Connections run +# AUTOCOMMIT (T7): each persist_* wraps its work in exactly one +# `with conn.transaction():` block — the context manager commits on exit, +# rolls back on failure, and nothing idles inside an open transaction between +# batches. A failure propagates to the worker, which rolls back, seeks back +# and does NOT commit offsets, so failed batches are redelivered +# (at-least-once). from __future__ import annotations import dataclasses @@ -80,6 +84,16 @@ class NormalizedRow: ocsf: dict | None +@dataclass(frozen=True) +class SampleRow: + """One onboarding_samples row; captured_at/role default in the table + ('unused' until the M2 onboarding split promotes it).""" + + fingerprint_id: str + raw_id: uuid.UUID + raw_text: str + + def build_rows(msgs) -> tuple[list[RawRow], dict[int, PartitionMeta]]: """Assemble RawRows + per-partition PartitionMeta from Kafka messages. @@ -125,6 +139,44 @@ def build_rows(msgs) -> tuple[list[RawRow], dict[int, PartitionMeta]]: return rows, parts +def collect_samples(rows: list[RawRow], active_rules: dict, + existing_counts: dict[str, int], cap: int = 200) -> list[SampleRow]: + """Sample capture (R17): raw lines for fingerprints with NO active rule, + at most `cap` per fingerprint counting what onboarding_samples already + holds (existing_counts, from sample_counts) plus what this call takes. + + Pure: no connection, no psycopg — the SQL lives in sample_counts() and + persist_normalized(); this only applies the capture policy. + """ + samples: list[SampleRow] = [] + taken: dict[str, int] = {} + for row in rows: + fp = row.fingerprint_id + if fp in active_rules: + continue + if existing_counts.get(fp, 0) + taken.get(fp, 0) >= cap: + continue + taken[fp] = taken.get(fp, 0) + 1 + samples.append(SampleRow(fp, row.raw_id, row.raw_text)) + return samples + + +def sample_counts(conn: psycopg.Connection, + fingerprint_ids: list[str]) -> dict[str, int]: + """Existing onboarding_samples counts for this batch's rule-less + fingerprints — one grouped query for the whole batch (R17), so capture + caps against what earlier batches already stored.""" + if not fingerprint_ids: + return {} + with conn.cursor() as cur: + cur.execute( + "SELECT fingerprint_id, count(*) FROM onboarding_samples " + "WHERE fingerprint_id = ANY(%s) GROUP BY fingerprint_id", + (fingerprint_ids,), + ) + return {fp: count for fp, count in cur.fetchall()} + + def persist_batch(conn: psycopg.Connection, rows: list[RawRow], partitions: dict[int, PartitionMeta]) -> None: """Insert raw rows idempotently + append the per-partition chain, ONE txn. @@ -134,14 +186,15 @@ def persist_batch(conn: psycopg.Connection, rows: list[RawRow], one consumer per partition, spec §4), prev_hash = prior batch's merkle_root (GENESIS_PREV_HASH for the first), row_hashes = the batch's ordered content_hash list, and merkle_root computed from exactly that list. - Commits on success; on failure the caller rolls back and skips the offset - commit so Kafka redelivers the batch. + The `with conn.transaction():` block commits on success and rolls back on + failure; the exception still propagates so the caller seeks back and skips + the offset commit, and Kafka redelivers the batch. """ if not partitions: return from psycopg.types.json import Json # deferred: keeps psycopg off the test path - with conn.cursor() as cur: + with conn.transaction(), conn.cursor() as cur: cur.execute("CREATE TEMP TABLE staging_raw (LIKE raw_events INCLUDING DEFAULTS) ON COMMIT DROP") with cur.copy( f"COPY staging_raw ({', '.join(_RAW_COLUMNS)}) FROM STDIN" @@ -177,18 +230,22 @@ def persist_batch(conn: psycopg.Connection, rows: list[RawRow], meta.first_offset, meta.last_offset, ), ) - conn.commit() log.info("raw batch persisted: %d row(s) across %d partition(s)", len(rows), len(partitions)) -def persist_normalized(conn: psycopg.Connection, events: list[NormalizedRow]) -> None: +def persist_normalized(conn: psycopg.Connection, events: list[NormalizedRow], + samples: list[SampleRow] | None = None) -> None: """Insert normalized rows idempotently (staged ON CONFLICT on - (event_id, parsed_at)), ONE txn; commits on success. + (event_id, parsed_at)) and — in the SAME transaction — capture onboarding + samples (INSERT .. ON CONFLICT (raw_id) DO NOTHING: replay-safe, one raw + line -> at most one sample, migration 003); ONE `conn.transaction()` block + commits on success. - The FK to raw_events holds because persist_batch already committed the raw - rows; a replay collides on the PK and is skipped (spec §4 idempotency). + The raw rows were committed by persist_batch earlier in the same batch, so + the normalized FK target is durable; a replay collides on the PK and is + skipped (spec §4 idempotency). """ - if not events: + if not events and not samples: return from psycopg.types.json import Json # deferred: keeps psycopg off the test path @@ -196,7 +253,7 @@ def persist_normalized(conn: psycopg.Connection, events: list[NormalizedRow]) -> "event_id", "raw_id", "raw_received_at", "parsed_at", "fingerprint_id", "rule_id", "rule_version", "status", "ocsf", ) - with conn.cursor() as cur: + with conn.transaction(), conn.cursor() as cur: cur.execute("CREATE TEMP TABLE staging_ne (LIKE normalized_events INCLUDING DEFAULTS) ON COMMIT DROP") with cur.copy(f"COPY staging_ne ({', '.join(columns)}) FROM STDIN") as copy: for ev in events: @@ -210,5 +267,11 @@ def persist_normalized(conn: psycopg.Connection, events: list[NormalizedRow]) -> f"SELECT {', '.join(columns)} FROM staging_ne " "ON CONFLICT (event_id, parsed_at) DO NOTHING" ) - conn.commit() - log.info("normalized batch persisted: %d row(s)", len(events)) + if samples: + cur.executemany( + "INSERT INTO onboarding_samples (fingerprint_id, raw_id, raw_text) " + "VALUES (%s, %s, %s) ON CONFLICT (raw_id) DO NOTHING", + [(s.fingerprint_id, s.raw_id, s.raw_text) for s in samples], + ) + log.info("normalized batch persisted: %d row(s), %d sample(s)", + len(events), len(samples or [])) diff --git a/services/pipeline/tests/test_db_samples.py b/services/pipeline/tests/test_db_samples.py new file mode 100644 index 0000000..0b7e69a --- /dev/null +++ b/services/pipeline/tests/test_db_samples.py @@ -0,0 +1,230 @@ +# services/pipeline/tests/test_db_samples.py — sample capture (R17) + the T7 +# transaction refactor. +# +# collect_samples is pure (rows, active_rules, existing_counts, cap -> +# SampleRows); the SQL it feeds lands in persist_normalized's SAME transaction +# as the normalized rows. The fake conn records SQL in order and brackets each +# `with conn.transaction():` block with TXN-START / TXN-COMMIT markers (the +# COMMIT-equivalent of psycopg's context manager), so "the samples INSERT +# precedes COMMIT" is an assertion on the recorded sequence, the autocommit +# refactor is pinned by exactly-one-transaction per persist, and a zero-event +# batch must open no transaction at all. It also has no commit() method: a +# dropped explicit conn.commit() is the refactor's RED assertion. +import hashlib +import uuid +from datetime import UTC, datetime + +import pytest + +from pipeline.db import ( + NormalizedRow, + PartitionMeta, + RawRow, + SampleRow, + collect_samples, + persist_batch, + persist_normalized, +) + +_TS = datetime(2026, 9, 19, 12, 0, 0, tzinfo=UTC) + + +def _row(fp="auto_x", n=0): + return RawRow( + raw_id=uuid.uuid5(uuid.NAMESPACE_DNS, f"raw-{fp}-{n}"), + received_at=_TS, + source_id="s", + transport="http", + format_hint=None, + fingerprint_id=fp, + content_hash=hashlib.sha256(f"line {n}".encode()).hexdigest(), + raw_text=f"line {n}", + ) + + +def _norm(fp="auto_x", status="unparsed", seq=0): + return NormalizedRow( + event_id=uuid.uuid5(uuid.NAMESPACE_DNS, f"ev-{seq}"), + raw_id=uuid.uuid5(uuid.NAMESPACE_DNS, f"raw-{seq}"), + raw_received_at=_TS, + parsed_at=_TS, + fingerprint_id=fp, + rule_id=None, + rule_version=0, + status=status, + ocsf=None, + ) + + +# --- (b) collect_samples: pure capture policy -------------------------------- + + +def test_unknown_fingerprint_rows_yield_samples(): + rows = [_row(n=i) for i in range(3)] + samples = collect_samples(rows, {}, {"auto_x": 0}, cap=200) + assert samples == [SampleRow("auto_x", r.raw_id, r.raw_text) for r in rows] + + +def test_capture_capped_at_cap_minus_existing(): + rows = [_row(n=i) for i in range(5)] + samples = collect_samples(rows, {}, {}, cap=3) + assert len(samples) == 3 # in-call takes count toward the cap too + + +def test_existing_at_cap_yields_no_samples(): + rows = [_row(n=i) for i in range(3)] + assert collect_samples(rows, {}, {"auto_x": 200}) == [] + + +def test_fingerprint_with_active_rule_never_sampled(): + rows = [_row("auto_r", n=i) for i in range(3)] + assert collect_samples(rows, {"auto_r": object()}, {"auto_r": 0}, cap=200) == [] + + +def test_mixed_batch_samples_only_rule_less_fingerprints(): + rows = [_row("auto_a", 0), _row("known", 1), _row("auto_a", 2)] + samples = collect_samples(rows, {"known": object()}, {}, cap=200) + assert [s.raw_id for s in samples] == [rows[0].raw_id, rows[2].raw_id] + assert all(s.fingerprint_id == "auto_a" for s in samples) + + +# --- fake conn: records SQL, brackets transactions ---------------------------- + + +class RecordingConn: + """Fake psycopg connection for the txn refactor. + + transaction() returns a context manager that brackets the recorded SQL + with TXN-START and TXN-COMMIT (or TXN-ROLLBACK when the body raised) — + the COMMIT-equivalent of psycopg's autocommit transaction block. There is + deliberately no commit() method: persist_* must not call one anymore. + """ + + def __init__(self, fail_on: str | None = None): + self.sql: list = [] + self.params: list = [] + self.txns_opened = 0 + self.fail_on = fail_on + + def transaction(self): + conn = self + + class Txn: + def __enter__(self): + conn.txns_opened += 1 + conn.sql.append("TXN-START") + return self + + def __exit__(self, exc_type, exc, tb): + conn.sql.append("TXN-ROLLBACK" if exc_type is not None else "TXN-COMMIT") + return False + + return Txn() + + def cursor(self): + conn = self + + class Cur: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params=None): + if conn.fail_on and conn.fail_on in sql: + raise RuntimeError("db down") + conn.sql.append(sql) + conn.params.append(params) + + def executemany(self, sql, params_seq): + if conn.fail_on and conn.fail_on in sql: + raise RuntimeError("db down") + conn.sql.append(sql) + conn.params.extend(params_seq) + + def fetchone(self): + return [1] + + def fetchall(self): + return [] + + def copy(self, sql): + conn.sql.append(sql) + + class Copy: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def write_row(self, row): + conn.sql.append(("COPY-ROW", row)) + + return Copy() + + return Cur() + + def rollback(self): + self.sql.append("ROLLBACK") + + +# --- (c) samples ride the normalized transaction ------------------------------ + + +def test_samples_insert_inside_the_normalized_transaction(): + conn = RecordingConn() + events = [_norm(seq=0), _norm(seq=1)] + samples = [SampleRow("auto_x", events[0].raw_id, "line 0")] + persist_normalized(conn, events, samples) + + sql = [s for s in conn.sql if isinstance(s, str)] + sample_inserts = [s for s in sql if s.startswith("INSERT INTO onboarding_samples")] + assert len(sample_inserts) == 1 + assert "ON CONFLICT (raw_id) DO NOTHING" in sample_inserts[0] + # Same txn as the normalized rows: bracketed, INSERT before COMMIT. + assert conn.txns_opened == 1 + norm_insert = next(i for i, s in enumerate(sql) if "INSERT INTO normalized_events" in s) + assert sql.index("TXN-START") < norm_insert + assert sql.index(sample_inserts[0]) < sql.index("TXN-COMMIT") + # Columns fingerprint_id, raw_id, raw_text; captured_at left to default. + assert ("auto_x", events[0].raw_id, "line 0") in conn.params + + +def test_persist_normalized_without_samples_skips_the_insert(): + conn = RecordingConn() + persist_normalized(conn, [_norm()]) + assert not [s for s in conn.sql if isinstance(s, str) + and s.startswith("INSERT INTO onboarding_samples")] + + +def test_persist_batch_opens_exactly_one_transaction(): + conn = RecordingConn() + rows = [_row(n=0), _row(n=1)] + parts = {0: PartitionMeta(0, 0, 1, [r.content_hash for r in rows])} + persist_batch(conn, rows, parts) + assert conn.txns_opened == 1 + assert conn.sql[0] == "TXN-START" and conn.sql[-1] == "TXN-COMMIT" + + +def test_persist_normalized_failure_propagates_and_marks_rollback(): + # The T7 refactor must keep the failure path intact: the exception leaves + # the transaction block (which marks the rollback) and reaches the caller, + # whose rollback+seek-back handling redelivers the batch unchanged. + conn = RecordingConn(fail_on="INSERT INTO normalized_events") + with pytest.raises(RuntimeError): + persist_normalized(conn, [_norm()]) + assert "TXN-ROLLBACK" in conn.sql + + +# --- (d) txn hygiene: empty work opens nothing -------------------------------- + + +def test_zero_event_batch_opens_no_transaction(): + conn = RecordingConn() + persist_normalized(conn, []) + persist_normalized(conn, [], None) + persist_batch(conn, [], {}) + assert conn.txns_opened == 0 + assert conn.sql == [] diff --git a/services/pipeline/tests/test_worker.py b/services/pipeline/tests/test_worker.py index e22f2fe..9209d8e 100644 --- a/services/pipeline/tests/test_worker.py +++ b/services/pipeline/tests/test_worker.py @@ -131,7 +131,7 @@ def test_persist_normalized_failure_seeks_back_and_never_commits(monkeypatch): monkeypatch.setattr(worker_mod, "persist_batch", lambda conn, rows, parts: None) monkeypatch.setattr(worker_mod, "load_active_rules", lambda conn: {}) - def boom(conn, events): + def boom(conn, events, samples=None): raise RuntimeError("transient db drop") monkeypatch.setattr(worker_mod, "persist_normalized", boom) @@ -140,8 +140,163 @@ class RollbackOnlyConn: def rollback(self): pass + class _Cur: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params=None): + pass + + def fetchall(self): + return [] + + def cursor(self): + return self._Cur() # sample-count query finds no existing samples + msgs = [FakeMsg(0, 7, envelope_bytes()), FakeMsg(2, 41, envelope_bytes())] consumer = RecordingConsumer(msgs) worker = PipelineWorker(consumer, SilentProducer(), RollbackOnlyConn()) assert worker.process_batch(msgs) is False assert consumer.seeks == [{0: 7, 2: 41}] + + +def test_every_event_produced_as_envelope_keyed_by_fingerprint(monkeypatch): + # R17: the pipeline ships ONE envelope per event — parsed, unparsed AND + # parse_error — to normalized.events, keyed by fingerprint_id, shaped + # exactly like ulpf_core.models.EventEnvelope (M3's drift service imports + # it). The fake producer round-trips value through json.dumps/loads, so a + # non-JSON-safe field (e.g. a raw UUID) fails right here. + from ulpf_core.fingerprint import fingerprint_id + from ulpf_core.ids import event_id_for, raw_id_for + from ulpf_core.models import EventEnvelope, Mapping + + from pipeline.rules import ActiveRule + + monkeypatch.setattr(worker_mod, "persist_batch", lambda conn, rows, parts: None) + + seen: dict = {} + monkeypatch.setattr( + worker_mod, "persist_normalized", + lambda conn, events, samples=None: seen.update(events=events, samples=samples)) + + parsed_line = '{"a":1}' + error_line = "CEF:0|acme|gw|1|probe|nomatch" + unparsed_line = "some vendor nobody seeded yet" + fp_parsed, fp_error, fp_unparsed = ( + fingerprint_id(line) for line in (parsed_line, error_line, unparsed_line)) + rules = { + fp_parsed: ActiveRule(id=1, fingerprint_id=fp_parsed, version=3, + pattern="__JSON__", + mappings=[Mapping(source_field="a", ocsf_path="message")], + provenance="human"), + fp_error: ActiveRule(id=2, fingerprint_id=fp_error, version=5, + pattern="will-not-match-", mappings=[], + provenance="human"), + } + monkeypatch.setattr(worker_mod, "load_active_rules", lambda conn: rules) + + class CountCursor: + # The per-batch sample-count SELECT: no existing samples anywhere. + def __init__(self, calls): + self._calls = calls + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params=None): + self._calls.append((sql, params)) + + def fetchall(self): + return [] + + class CountConn: + def __init__(self): + self.calls: list = [] + + def cursor(self): + return CountCursor(self.calls) + + def rollback(self): + pass + + class RecordingProducer: + def __init__(self): + self.produced: list = [] + + def produce(self, topic, key, value): + self.produced.append((topic, key, json.loads(json.dumps(value)))) + + msgs = [FakeMsg(0, 1, envelope_bytes(parsed_line)), + FakeMsg(0, 2, envelope_bytes(error_line)), + FakeMsg(1, 9, envelope_bytes(unparsed_line))] + conn, producer = CountConn(), RecordingProducer() + worker = PipelineWorker(RecordingConsumer(msgs), producer, conn) + assert worker.process_batch(msgs) is True + + envelopes = {k: v for t, k, v in producer.produced if t == "normalized.events"} + assert set(envelopes) == {fp_parsed, fp_error, fp_unparsed} + # Produced-tuple COUNT per topic (the dict above dedupes by key, which + # would hide a double-production bug): one envelope per event, and the + # DLQ carries exactly the one parse_error. + produced_by_topic: dict[str, int] = {} + for topic, _, _ in producer.produced: + produced_by_topic[topic] = produced_by_topic.get(topic, 0) + 1 + assert produced_by_topic == {"normalized.events": 3, "pipeline.dlq": 1} + expected = { + fp_parsed: EventEnvelope( + event_id=str(event_id_for(raw_id_for("raw.logs", 0, 1), 3)), + raw_id=str(raw_id_for("raw.logs", 0, 1)), + fingerprint_id=fp_parsed, rule_version=3, status="parsed"), + fp_error: EventEnvelope( + event_id=str(event_id_for(raw_id_for("raw.logs", 0, 2), 5)), + raw_id=str(raw_id_for("raw.logs", 0, 2)), + fingerprint_id=fp_error, rule_version=5, status="parse_error"), + fp_unparsed: EventEnvelope( + event_id=str(event_id_for(raw_id_for("raw.logs", 1, 9), 0)), + raw_id=str(raw_id_for("raw.logs", 1, 9)), + fingerprint_id=fp_unparsed, rule_version=0, status="unparsed"), + } + for fp, env in expected.items(): + # parsed' ocsf is the full OCSF doc — compared by content below. + got = {k: v for k, v in envelopes[fp].items() if k != "ocsf"} + assert got == {k: v for k, v in env.model_dump().items() if k != "ocsf"} + assert envelopes[fp_parsed]["ocsf"]["message"] == "1" # parsed doc rides inside + assert envelopes[fp_error]["ocsf"] is None + assert envelopes[fp_unparsed]["ocsf"] is None + + # DLQ behavior unchanged: the parse_error still lands there (and only it). + dlq = [(k, v) for t, k, v in producer.produced if t == "pipeline.dlq"] + assert len(dlq) == 1 and dlq[0][0] == fp_error + + # Sample capture: ONE grouped count query for the batch's rule-less + # fingerprints; capture only takes samples for those. + assert len(conn.calls) == 1 + sql, params = conn.calls[0] + assert "FROM onboarding_samples" in sql and "count(*)" in sql + assert params[0] == [fp_unparsed] + assert [s.fingerprint_id for s in seen["samples"]] == [fp_unparsed] + + +def test_connect_db_requests_autocommit(monkeypatch): + # T7: the connection runs autocommit — each persist_* wraps its own + # `with conn.transaction():` block, so nothing idles inside an open txn. + import sys + import types + + seen: dict = {} + fake_psycopg = types.ModuleType("psycopg") + + def fake_connect(url, **kwargs): + seen.update(url=url, kwargs=kwargs) + return object() + + fake_psycopg.connect = fake_connect + monkeypatch.setitem(sys.modules, "psycopg", fake_psycopg) + worker_mod.connect_db("postgresql://u:p@h/db") + assert seen["kwargs"] == {"autocommit": True} diff --git a/services/pipeline/worker.py b/services/pipeline/worker.py index 742f07a..101cb4a 100644 --- a/services/pipeline/worker.py +++ b/services/pipeline/worker.py @@ -3,16 +3,20 @@ # Per batch of up to 500 messages (or 2s), the spec-critical ordering (§4): # 1. build_rows (fingerprint at build time, controller R3) # 2. persist_batch — raw_events + raw_batches chain, COMMITTED -# 3. refresh active rules +# 3. refresh active rules; one grouped count query for this batch's +# rule-less fingerprints' existing onboarding samples # 4. per event: apply active rule or 'unparsed'; parse failures -> parse_error -# + pipeline.dlq; parsed docs -> normalized.events (key=fingerprint_id) -# 5. persist_normalized — COMMITTED +# + pipeline.dlq; EVERY event (all three statuses) ships a compact +# normalized.events envelope keyed by fingerprint_id (R17) +# 5. persist_normalized — COMMITTED (sample capture rides the SAME txn, +# capped per fingerprint, ON CONFLICT (raw_id) DO NOTHING) # 6. only then consumer.commit() # Crash/rollback anywhere before step 6 and the worker seeks each affected # partition back to the failed batch's first offset (consume() had already # advanced the position; the next no-arg commit() would otherwise cover the # failed batch and its events would be silently lost). Redelivery is a no-op -# on the DB side: every write is deterministic and ON CONFLICT-guarded (§4). +# on the DB side: every write is deterministic and ON CONFLICT-guarded (§4), +# samples included (raw_id unique, so one line -> at most one sample). # # parsed_at is the raw event's received_at (not wall clock) so a replay mints # the same event_id and collides instead of duplicating. M1 never supersedes @@ -23,9 +27,17 @@ from ocsf_schema.validator import validate_document from ulpf_core.ids import event_id_for +from ulpf_core.models import EventEnvelope from ulpf_core.parsing import parse -from pipeline.db import NormalizedRow, build_rows, persist_batch, persist_normalized +from pipeline.db import ( + NormalizedRow, + build_rows, + collect_samples, + persist_batch, + persist_normalized, + sample_counts, +) from pipeline.rules import load_active_rules log = logging.getLogger("pipeline.worker") @@ -40,6 +52,11 @@ MAX_LINE_CHARS = 65536 # spec §10 line cap — re-checked before parse (collector checks at ingest) CANARY_EVERY = 100 # every 100th parsed doc through validate_document (spec §6.4) +# Sample capture cap (R17): at most N raw lines retained per rule-less +# fingerprint for M2's prompt/held-out split. Env-tunable so tests and ops +# can shrink it without a code change. +SAMPLE_CAPTURE_CAP = int(os.environ.get("ULPF_SAMPLE_CAPTURE_CAP", "200")) + # Sentinel rule version for events no active rule matched: deterministic, so a # replay of the same raw line yields the same event_id and is skipped. UNPARSED_RULE_VERSION = 0 @@ -91,9 +108,20 @@ def __init__(self, bootstrap: str): self._producer = confluent_kafka.Producer({"bootstrap.servers": bootstrap}) def produce(self, topic: str, key: str, value: dict) -> None: - self._producer.produce(topic, key=key.encode(), value=json.dumps(value).encode()) + self._producer.produce( + topic, key=key.encode(), value=json.dumps(value).encode(), + on_delivery=self._on_delivery) # T7: delivery failures must be visible self._producer.poll(0) # serve delivery callbacks without blocking + @staticmethod + def _on_delivery(err, msg) -> None: + """Delivery-report callback (T7): a lost normalized.events or DLQ + emission is a silent correctness hole until M3 consumes the topic — + surface it at ERROR the moment librdkafka reports it.""" + if err is not None: + topic = msg.topic() if msg is not None else "" + log.error("kafka delivery failed: topic=%s err=%s", topic, err) + def flush(self, timeout: float = 10.0) -> int: return self._producer.flush(timeout) @@ -176,12 +204,20 @@ def process_batch(self, msgs) -> bool: return False rules = load_active_rules(self.conn) # refreshed every batch + + # Sample capture (R17): only fingerprints with NO active rule, capped + # per fingerprint by what onboarding_samples already holds. One grouped + # count query for the whole batch; known fingerprints query nothing. + unknown = sorted({row.fingerprint_id for row in rows} - rules.keys()) + counts = sample_counts(self.conn, unknown) if unknown else {} + samples = collect_samples(rows, rules, counts, SAMPLE_CAPTURE_CAP) + events: list[NormalizedRow] = [] parsed = unparsed = parse_errors = 0 for row in rows: rule = rules.get(row.fingerprint_id) if rule is None: - events.append(NormalizedRow( + unparsed_row = NormalizedRow( event_id=event_id_for(row.raw_id, UNPARSED_RULE_VERSION), raw_id=row.raw_id, raw_received_at=row.received_at, @@ -191,21 +227,27 @@ def process_batch(self, msgs) -> bool: rule_version=UNPARSED_RULE_VERSION, status="unparsed", ocsf=None, - )) + ) + events.append(unparsed_row) unparsed += 1 + self._produce_envelope(unparsed_row) continue # Line cap re-checked here: parse() runs untrusted text on the hot path. if len(row.raw_text) > MAX_LINE_CHARS: - events.append(self._parse_error_row( - row, rule, f"line exceeds {MAX_LINE_CHARS} char cap")) + capped_row = self._parse_error_row( + row, rule, f"line exceeds {MAX_LINE_CHARS} char cap") + events.append(capped_row) parse_errors += 1 + self._produce_envelope(capped_row) continue doc, err = parse(row.raw_text, rule) if err is not None: - events.append(self._parse_error_row(row, rule, err)) + error_row = self._parse_error_row(row, rule, err) + events.append(error_row) parse_errors += 1 + self._produce_envelope(error_row) continue # parse() leaves time None when the rule maps no timestamp; the @@ -220,7 +262,7 @@ def process_batch(self, msgs) -> bool: self.canary_failures += 1 log.error("canary doc invalid for %s: %s", row.fingerprint_id, problems) - events.append(NormalizedRow( + parsed_row = NormalizedRow( event_id=event_id_for(row.raw_id, rule.version), raw_id=row.raw_id, raw_received_at=row.received_at, @@ -230,20 +272,16 @@ def process_batch(self, msgs) -> bool: rule_version=rule.version, status="parsed", ocsf=doc, - )) - try: - self.producer.produce(TOPIC_NORMALIZED, row.fingerprint_id, doc) - except Exception: - # The normalized row is durable in step 5 regardless; a lost - # emission is visible via the DB and must not stall the batch. - log.exception("produce to %s failed for %s", TOPIC_NORMALIZED, row.raw_id) + ) + events.append(parsed_row) + self._produce_envelope(parsed_row) parsed += 1 try: - persist_normalized(self.conn, events) + persist_normalized(self.conn, events, samples) except Exception: self.conn.rollback() - self._seek_back(msgs) # redeliver THIS batch (raw rows ON CONFLICT dedup) + self._seek_back(msgs) # redeliver THIS batch (raw rows + samples dedup on replay) log.exception("normalized persist failed; seeked back to first failed offsets; " "offsets NOT committed; batch redelivered") return False @@ -254,11 +292,33 @@ def process_batch(self, msgs) -> bool: # commit would silently cover it. self.consumer.commit() log.info("batch: %d consumed, %d parsed, %d unparsed, %d parse_error, " - "%d skipped-envelope, canary_failures=%d", + "%d skipped-raw, %d samples, canary_failures=%d", len(msgs), parsed, unparsed, parse_errors, len(msgs) - len(rows), - self.canary_failures) + len(samples), self.canary_failures) return True + def _produce_envelope(self, row: NormalizedRow) -> None: + """R17: ship the event's compact envelope to normalized.events, keyed + by fingerprint_id. M3's drift service consumes ALL THREE statuses, so + unparsed/parse_error carry ocsf=None (rule_version 0 / rule.version).""" + version = (row.rule_version if row.rule_version is not None + else UNPARSED_RULE_VERSION) + envelope = EventEnvelope( + event_id=str(row.event_id), + raw_id=str(row.raw_id), + fingerprint_id=row.fingerprint_id, + rule_version=version, + status=row.status, + ocsf=row.ocsf, + ) + try: + self.producer.produce(TOPIC_NORMALIZED, row.fingerprint_id, + envelope.model_dump()) + except Exception: + # The normalized row is durable in step 5 regardless; a lost + # emission is visible via the DB and must not stall the batch. + log.exception("produce to %s failed for %s", TOPIC_NORMALIZED, row.raw_id) + def _parse_error_row(self, row, rule, error: str) -> NormalizedRow: """DLQ produce + the parse_error row, which joins the same normalized txn as the rest of the batch.""" @@ -286,10 +346,14 @@ def _parse_error_row(self, row, rule, error: str) -> NormalizedRow: def connect_db(url: str): - """psycopg connection for pipeline_role (INSERT+SELECT only, no UPDATE/DELETE).""" + """psycopg connection for pipeline_role (INSERT+SELECT only, no UPDATE/DELETE). + + autocommit=True (T7): each persist_* wraps its own `with conn.transaction():` + block — nothing idles inside an open transaction between batches, and the + context manager still commits/rolls back atomically per write.""" import psycopg # deferred: unit tests never need psycopg - return psycopg.connect(url) + return psycopg.connect(url, autocommit=True) def main(): diff --git a/simulator/data/raw_logs_newapp.txt b/simulator/data/raw_logs_newapp.txt new file mode 100644 index 0000000..ae6dda7 --- /dev/null +++ b/simulator/data/raw_logs_newapp.txt @@ -0,0 +1,12 @@ +NEXUSFW-XE7 flowgate ts=2026-09-19T10:00:03Z src=198.51.100.4 srcport=52114 dst=203.0.113.9 dstport=8443 proto=TCP action=permit msg=Tunnel established sessionid=88123 +NEXUSFW-XE7 flowgate ts=2026-09-19T10:00:07Z src=198.51.100.9 srcport=49300 dst=203.0.113.4 dstport=443 proto=TCP action=deny msg=Blocked unauthorized relay sessionid=88124 +NEXUSFW-XE7 flowgate ts=2026-09-19T10:00:12Z src=198.51.100.23 srcport=40017 dst=203.0.113.77 dstport=22 proto=TCP action=deny msg=Login attempt rejected by policy sessionid=88125 +NEXUSFW-XE7 flowgate ts=2026-09-19T10:00:19Z src=198.51.100.55 srcport=61002 dst=203.0.113.12 dstport=53 proto=UDP action=permit msg=Recursive lookup granted sessionid=88126 +NEXUSFW-XE7 flowgate ts=2026-09-19T10:00:25Z src=198.51.100.77 srcport=51190 dst=203.0.113.200 dstport=8080 proto=TCP action=permit msg=Proxy session opened sessionid=88127 +NEXUSFW-XE7 flowgate ts=2026-09-19T10:00:31Z src=198.51.100.104 srcport=44521 dst=203.0.113.58 dstport=445 proto=TCP action=deny msg=SMB relay blocked sessionid=88128 +NEXUSFW-XE7 flowgate ts=2026-09-19T10:00:38Z src=198.51.100.140 srcport=52330 dst=203.0.113.3 dstport=123 proto=UDP action=permit msg=Time sync sessionid=88129 +NEXUSFW-XE7 flowgate ts=2026-09-19T10:00:44Z src=198.51.100.201 srcport=49155 dst=203.0.113.150 dstport=3389 proto=TCP action=deny msg=RDP gateway probe dropped sessionid=88130 +NEXUSFW-XE7 flowgate ts=2026-09-19T10:00:51Z src=198.51.100.16 srcport=50210 dst=203.0.113.90 dstport=514 proto=TCP action=permit msg=Log forwarder attached sessionid=88131 +NEXUSFW-XE7 flowgate ts=2026-09-19T10:00:58Z src=198.51.100.66 srcport=48002 dst=203.0.113.44 dstport=161 proto=UDP action=deny msg=SNMP community mismatch sessionid=88132 +NEXUSFW-XE7 flowgate ts=2026-09-19T10:01:04Z src=198.51.100.90 srcport=53001 dst=203.0.113.230 dstport=9200 proto=TCP action=permit msg=Indexer feeder accepted sessionid=88133 +NEXUSFW-XE7 flowgate ts=2026-09-19T10:01:10Z src=198.51.100.120 srcport=44417 dst=203.0.113.19 dstport=1521 proto=TCP action=deny msg=Database listener refused sessionid=88134 diff --git a/simulator/data/raw_logs_zenwall.txt b/simulator/data/raw_logs_zenwall.txt new file mode 100644 index 0000000..4259877 --- /dev/null +++ b/simulator/data/raw_logs_zenwall.txt @@ -0,0 +1,10 @@ +ZW1;2026-08-27T14:32:07Z;deny;src=198.51.100.7:4432;dst=10.0.0.9:22;rule=R7 +ZW1;2026-08-27T14:33:41Z;allow;src=192.168.1.60:53211;dst=8.8.8.8:53;rule=R2 +ZW1;2026-08-27T14:35:12Z;deny;src=203.0.113.90:51500;dst=10.0.0.9:3389;rule=R7 +ZW1;2026-08-27T14:36:47Z;alert;src=198.51.100.22:44521;dst=10.0.0.14:80;rule=R11 +ZW1;2026-08-27T14:38:30Z;deny;src=203.0.113.33:49213;dst=10.0.0.50:80;rule=R4 +ZW1;2026-08-27T14:40:05Z;allow;src=192.168.1.70:52890;dst=8.8.4.4:53;rule=R2 +ZW1;2026-08-27T14:41:41Z;deny;src=192.0.2.201:51900;dst=10.0.0.15:445;rule=R9 +ZW1;2026-08-27T14:43:02Z;alert;src=198.51.100.90:44012;dst=10.0.0.12:443;rule=R11 +ZW1;2026-08-27T14:45:15Z;deny;src=203.0.113.150:59012;dst=10.0.0.99:8080;rule=R7 +ZW1;2026-08-27T14:46:29Z;allow;src=192.168.1.18:60011;dst=8.8.8.8:53;rule=R2 diff --git a/simulator/simulate.py b/simulator/simulate.py index d39b0db..8bfcd09 100644 --- a/simulator/simulate.py +++ b/simulator/simulate.py @@ -1,7 +1,7 @@ # simulator/simulate.py — multi-vendor traffic simulator over the frozen golden corpora. # # Routes each corpus to the transport the collector expects (R7): -# cef/syslog/acmegw -> UDP datagrams, one log line each, no envelope (the +# cef/syslog/acmegw/newapp -> UDP datagrams, one log line each, no envelope (the # collector derives source_id from the peer address, so # there is nothing to tag on the wire); # json -> batched HTTP POSTs to /v1/ingest, source_id "ids01" @@ -10,7 +10,15 @@ # Pacing: --eps is the TOTAL event rate, distributed evenly across the corpora # found on disk; corpora are interleaved round-robin, one line per corpus per # tick. --loop repeats the corpora until --duration elapses; without it, each -# corpus is sent once through (still paced), then the run ends. +# corpus is sent once through (still paced), then the run ends. --duration is a +# hard budget in both modes: no send may start after the deadline, checked +# between passes AND mid-pass. +# +# Modes (T10): "steady" replays only the M1 golden corpora. "new-appliance" +# additionally loads the newapp corpus (an unknown proprietary format for the +# M2 onboarding loop) after --new-after seconds: until then its rotation slot +# is idle — no send, no cursor advance — so the already-running corpora are +# untouched while it waits. # # Stdout contract (parsed by the M1 smoke test): exactly one line per corpus, # sent= @@ -28,6 +36,7 @@ import time from dataclasses import dataclass from pathlib import Path +from typing import Protocol, runtime_checkable import httpx @@ -37,9 +46,10 @@ HTTP_BATCH_SIZE = 10 HTTP_TIMEOUT_S = 5.0 -UDP_KEYS = frozenset({"cef", "syslog", "acmegw"}) +UDP_KEYS = frozenset({"cef", "syslog", "acmegw", "newapp"}) HTTP_KEYS = frozenset({"json"}) CORPUS_KEYS = sorted(UDP_KEYS | HTTP_KEYS) +NEWAPP_KEY = "newapp" # loaded only in --mode new-appliance (see load_corpora) def transport_for(key: str) -> str: @@ -121,11 +131,26 @@ async def close(self) -> None: await self._client.aclose() +@runtime_checkable +class Sender(Protocol): + """Anything that can carry corpus lines (structural — no declaration needed). + + UdpSender, HttpSender and the tests' fakes all qualify by shape: + async send(line)->int, flush()->int. close() is intentionally not part of + the protocol: it is sync on UdpSender, async on HttpSender, and _close_all + already handles both. + """ + + async def send(self, line: str) -> int: ... + async def flush(self) -> int: ... + + @dataclass class Corpus: name: str lines: list[str] - sender: object # UdpSender / HttpSender / test double: async send(line)->int, flush()->int + sender: Sender + start_at: float = 0.0 # seconds from run start before this corpus joins the rotation def load_lines(path: Path) -> list[str]: @@ -136,10 +161,19 @@ def load_lines(path: Path) -> list[str]: return lines -def load_corpora(corpus_dir: Path | str, senders: dict) -> list[Corpus]: +def load_corpora(corpus_dir: Path | str, senders: dict, + include_newapp: bool = False) -> list[Corpus]: + """Load raw_logs_.txt per corpus key; default (steady) excludes newapp. + + include_newapp=True is --mode new-appliance: the unknown-format corpus + joins the load so onboarding can meet a format it has no rule for. The + default keeps the M1 smoke's stdout contract byte-identical. + """ corpus_dir = Path(corpus_dir) corpora = [] for key in CORPUS_KEYS: + if key == NEWAPP_KEY and not include_newapp: + continue # steady mode: the unknown appliance stays off the wire path = corpus_dir / f"raw_logs_{key}.txt" if not path.is_file(): print(f"warning: corpus file missing, skipping: {path}", file=sys.stderr) @@ -170,6 +204,9 @@ async def run(corpora: list[Corpus], *, eps: float, duration: float, loop: bool = False) -> tuple[dict, int]: """Send the corpora round-robin at eps total events/sec. + A corpus with start_at > 0 stays idle (no send, no cursor advance) until + that many seconds have elapsed, then joins the rotation in place. + Returns (sent_per_corpus, error_count). A send counts only when the transport accepted it (UDP sendto returned; HTTP got a 202). """ @@ -186,11 +223,17 @@ async def run(corpora: list[Corpus], *, eps: float, duration: float, pass_no = 0 while True: - if loop and time.monotonic() >= deadline: + if time.monotonic() >= deadline: # hard budget in loop AND one-shot mode break if not loop and all(cursors[i] >= len(corpora[i].lines) for i in range(n)): break + stopped = False for i, corpus in enumerate(corpora): + if time.monotonic() >= deadline: + stopped = True # mid-pass: a pass that would overrun stops here + break + if time.monotonic() - start < corpus.start_at: + continue # not yet due (start_at): its slot is idle this pass if cursors[i] >= len(corpus.lines): if not loop: continue @@ -202,6 +245,8 @@ async def run(corpora: list[Corpus], *, eps: float, duration: float, except Exception as exc: # noqa: BLE001 - one failed send counts and moves on errors += 1 print(f"error: {corpus.name}: {exc}", file=sys.stderr) + if stopped: + break pass_no += 1 delay = start + pass_no * tick - time.monotonic() if delay > 0: @@ -231,11 +276,18 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: p.add_argument("--eps", type=float, default=20.0, help="total events/sec across all corpora (default 20)") p.add_argument("--duration", type=float, default=60.0, - help="seconds to run when --loop is set (default 60)") + help="seconds before the run is cut off (hard budget in both " + "modes; --loop repeats until then) (default 60)") p.add_argument("--corpora", default=None, help="directory holding raw_logs_*.txt (default: simulator/data)") p.add_argument("--loop", action="store_true", help="repeat the corpora until --duration elapses") + p.add_argument("--mode", choices=("steady", "new-appliance"), default="steady", + help="steady: golden corpora only (default); new-appliance: " + "also replay the unknown-format newapp corpus (T10)") + p.add_argument("--new-after", type=float, default=0.0, dest="new_after", + help="new-appliance only: seconds before the newapp corpus " + "joins the rotation (default 0; steady ignores this)") return p.parse_args(argv) @@ -252,11 +304,15 @@ def main(argv: list[str] | None = None) -> int: print(f"error: {exc}", file=sys.stderr) return 1 - corpora = load_corpora(corpus_dir, senders) + include_newapp = args.mode == "new-appliance" + corpora = load_corpora(corpus_dir, senders, include_newapp=include_newapp) if not corpora: print(f"error: no corpora found in {corpus_dir}", file=sys.stderr) asyncio.run(_close_all(senders)) return 1 + for corpus in corpora: + if corpus.name == NEWAPP_KEY: + corpus.start_at = max(args.new_after, 0.0) async def drive(): try: diff --git a/simulator/tests/test_simulate.py b/simulator/tests/test_simulate.py index 595a8c7..4666cf8 100644 --- a/simulator/tests/test_simulate.py +++ b/simulator/tests/test_simulate.py @@ -12,6 +12,7 @@ HTTP_SOURCE_ID, Corpus, HttpSender, + Sender, UdpSender, default_senders, load_corpora, @@ -103,6 +104,7 @@ def test_transport_routing_table(): assert transport_for("cef") == "udp" assert transport_for("syslog") == "udp" assert transport_for("acmegw") == "udp" + assert transport_for("newapp") == "udp" assert transport_for("json") == "http" with pytest.raises(ValueError): transport_for("zenwall") @@ -211,6 +213,19 @@ def test_load_corpora_empty_dir_is_no_corpora(tmp_path): assert load_corpora(tmp_path, {}) == [] +# --- load_corpora: newapp corpus is gated behind include_newapp (T10) ------------ + +def test_load_corpora_steady_excludes_newapp_new_appliance_includes(tmp_path): + make_corpora(tmp_path, {"cef": "c1\n", "newapp": "n1\nn2\n"}) + senders = {"cef": FakeUdp("cef", []), "newapp": FakeUdp("newapp", [])} + steady = load_corpora(tmp_path, senders) # default: steady mode + assert [c.name for c in steady] == ["cef"] + with_newapp = load_corpora(tmp_path, senders, include_newapp=True) + assert [c.name for c in with_newapp] == ["cef", "newapp"] + assert with_newapp[1].lines == ["n1", "n2"] + assert with_newapp[1].start_at == 0.0 # default joins immediately + + # --- run(): pacing floor, round-robin, loop -------------------------------------- def udp_corpus(name, lines, log): @@ -285,6 +300,98 @@ def test_run_json_goes_to_http_sender_only(): assert [name for name, _ in log] == ["cef", "json", "json"] +# --- Sender protocol (T11): Corpus.sender is typed, not just "object" ----------- + +def test_sender_protocol_accepts_transport_and_test_doubles(): + assert isinstance(UdpSender(("localhost", 5514)), Sender) + assert isinstance(FakeUdp("x", []), Sender) + assert isinstance(FakeHttp("x", []), Sender) + assert not isinstance(object(), Sender) # no send/flush -> not a Sender + + +# --- start_at: a corpus joins the rotation only once it is due (T10-a) ---------- + +def test_newapp_joins_rotation_only_after_start_at(): + log = [] + corpora = [ + udp_corpus("cef", [f"c{i}" for i in range(20)], log), + udp_corpus("newapp", ["n1", "n2", "n3"], log), + ] + corpora[1].start_at = 0.1 # seconds from run start + sent, errors = do_run(corpora, eps=100, duration=0.5, loop=True) + assert errors == 0 + # tick = 2/100 = 0.02s; newapp must stay silent for ~5 passes. If start_at + # were ignored it would send on pass 0 and first_newapp would be 1. + first_newapp = log.index(("newapp", "n1")) + assert first_newapp >= 3 + assert all(name == "cef" for name, _ in log[:first_newapp]) # nothing advanced early + assert sent["newapp"] >= 1 # it did eventually join the rotation + + +def test_unstarted_corpus_does_not_block_nonloop_termination(): + # Non-loop: steady corpora exhaust while newapp waits; the run must keep + # ticking until start_at elapses, send newapp once, then end (not hang). + log = [] + corpora = [ + udp_corpus("cef", ["c1", "c2"], log), + udp_corpus("newapp", ["n1"], log), + ] + corpora[1].start_at = 0.15 + sent, errors = do_run(corpora, eps=20, duration=5, loop=False) + assert errors == 0 + assert sent == {"cef": 2, "newapp": 1} + + +# --- mid-pass deadline (T11): duration bounds sends even within one pass --------- + +class SlowUdp(FakeUdp): + """FakeUdp whose send takes real time, so a single pass can overrun.""" + + def __init__(self, name, log, delay): + super().__init__(name, log) + self.delay = delay + self.starts = [] # monotonic timestamp just before each send began + + async def send(self, line): + self.starts.append(time.monotonic()) + await asyncio.sleep(self.delay) + return await super().send(line) + + +def test_midpass_deadline_stops_sends_at_boundary(): + log = [] + senders = [SlowUdp(name, log, 0.03) for name in ("a", "b", "c")] + corpora = [Corpus(name, ["x1", "x2", "x3"], s) for name, s in zip(("a", "b", "c"), senders)] + # 3 sends/pass * 0.03s -> pass 0 ends ~0.09s; duration 0.1 elapses DURING + # pass 1. Only the mid-pass check stops pass 1 at the boundary (4 sends); + # a top-of-pass-only check would let the whole pass run (6 sends). + t0 = time.monotonic() + sent, errors = do_run(corpora, eps=10, duration=0.1, loop=False) + assert errors == 0 + total = sum(sent.values()) + assert 3 <= total <= 4 # boundary cut inside pass 1, not a full extra pass + for s in senders: + for start in s.starts: # no send may START after the deadline + assert start - t0 <= 0.12 + + +# --- pacing ceiling: eps*duration bounds the total from ABOVE too (T11) ---------- + +def test_pacing_ceiling_eps4_duration1_sends_do_not_overrun(): + log = [] + corpora = [ + udp_corpus("acmegw", ["a1", "a2", "a3"], log), + udp_corpus("cef", ["c1", "c2", "c3"], log), + udp_corpus("json", ["j1", "j2", "j3"], log), + udp_corpus("syslog", ["s1", "s2", "s3"], log), + ] + sent, errors = do_run(corpora, eps=4, duration=1, loop=True) + assert errors == 0 + # tick = 1s: pass 0 (4 sends) then sleep past the deadline. Jitter may add + # a partial pass, but unbounded (no pacing sleep) would send thousands. + assert sum(sent.values()) <= 12 + + # --- main(): stdout contract, exit codes, env wiring ------------------------------ def test_main_prints_one_summary_line_per_corpus_plus_total(tmp_path, capsys, monkeypatch): @@ -308,6 +415,65 @@ def test_main_exit_1_when_no_corpora(tmp_path, capsys): assert "no corpora" in capsys.readouterr().err +# --- main(): --mode wiring (T10) --------------------------------------------------- + +def test_main_steady_omits_newapp_line(tmp_path, capsys, monkeypatch): + make_corpora(tmp_path, {"cef": "c1\n", "newapp": "n1\n"}) + log = [] + monkeypatch.setattr( + "simulator.simulate.default_senders", + lambda: {"cef": FakeUdp("cef", log), "newapp": FakeUdp("newapp", log)}, + ) + rc = main(["--eps", "100", "--duration", "1", "--corpora", str(tmp_path)]) + out = capsys.readouterr().out.strip().splitlines() + assert rc == 0 + assert out == ["cef sent=1", "total sent=1"] # M1 contract: no newapp line + + +def test_main_steady_ignores_new_after_flag(tmp_path, capsys, monkeypatch): + make_corpora(tmp_path, {"cef": "c1\nc2\n"}) + log = [] + monkeypatch.setattr("simulator.simulate.default_senders", + lambda: {"cef": FakeUdp("cef", log)}) + rc = main(["--eps", "100", "--duration", "1", "--new-after", "999", + "--corpora", str(tmp_path)]) + out = capsys.readouterr().out.strip().splitlines() + assert rc == 0 + assert out == ["cef sent=2", "total sent=2"] + + +def test_main_new_appliance_prints_newapp_line(tmp_path, capsys, monkeypatch): + make_corpora(tmp_path, {"cef": "c1\nc2\n", "newapp": "n1\nn2\n"}) + log = [] + monkeypatch.setattr( + "simulator.simulate.default_senders", + lambda: {"cef": FakeUdp("cef", log), "newapp": FakeUdp("newapp", log)}, + ) + rc = main(["--eps", "100", "--duration", "1", "--mode", "new-appliance", + "--corpora", str(tmp_path)]) + out = capsys.readouterr().out.strip().splitlines() + assert rc == 0 + assert out == ["cef sent=2", "newapp sent=2", "total sent=4"] + + +def test_main_new_appliance_delays_newapp_until_new_after(tmp_path, capsys, monkeypatch): + make_corpora(tmp_path, {"cef": "c1\nc2\nc3\nc4\nc5\n", "newapp": "n1\n"}) + log = [] + monkeypatch.setattr( + "simulator.simulate.default_senders", + lambda: {"cef": FakeUdp("cef", log), "newapp": FakeUdp("newapp", log)}, + ) + rc = main(["--eps", "40", "--duration", "2", "--mode", "new-appliance", + "--new-after", "0.15", "--loop", "--corpora", str(tmp_path)]) + out = capsys.readouterr().out.strip().splitlines() + assert rc == 0 + assert "newapp sent=" in out[-2] # per-corpus line for newapp, then total + # tick = 2/40 = 0.05s; newapp must join only after ~3 ticks + first_newapp = log.index(("newapp", "n1")) + assert first_newapp >= 3 + assert all(name == "cef" for name, _ in log[:first_newapp]) + + def test_main_exit_1_on_total_send_failure_but_still_prints_summary(tmp_path, capsys, monkeypatch): make_corpora(tmp_path, {"cef": "c1\n"}) @@ -338,7 +504,7 @@ def test_default_senders_reads_env(monkeypatch): monkeypatch.setenv("COLLECTOR_UDP", "collector-host:5999") monkeypatch.setenv("COLLECTOR_HTTP", "http://collector-host:9999") senders = default_senders() - assert set(senders) == {"cef", "syslog", "acmegw", "json"} + assert set(senders) == {"cef", "newapp", "syslog", "acmegw", "json"} assert senders["cef"].addr == ("collector-host", 5999) assert senders["json"].url == "http://collector-host:9999/v1/ingest" for s in senders.values(): diff --git a/web/src/App.css b/web/src/App.css index 843da95..75c4faa 100644 --- a/web/src/App.css +++ b/web/src/App.css @@ -1,4 +1,4 @@ -/* web/src/App.css — Overview page styles. +/* web/src/App.css — Overview + shell + Review Queue styles. Air-gap rules: system font stack only, no external fonts/icons/CDN. */ :root { @@ -242,3 +242,348 @@ body { margin: 0; word-break: break-all; } + +/* --- tab nav shell ------------------------------------------------------ */ + +.tabs { + display: flex; + gap: 4px; + border-bottom: 1px solid var(--border); + margin-bottom: 16px; +} + +.tab { + background: none; + border: 1px solid transparent; + border-bottom: none; + border-radius: 6px 6px 0 0; + color: var(--muted); + padding: 8px 16px; + font-size: 14px; + cursor: pointer; +} + +.tab:hover { + color: var(--text); +} + +.tab-active, +.tab[aria-selected='true'] { + color: var(--text); + background: var(--panel); + border-color: var(--border); +} + +/* --- review queue candidate cards ---------------------------------------- */ + +.candidate { + margin-bottom: 20px; +} + +.candidate h3, +.previews h3, +.checks h3 { + font-size: 13px; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--muted); + margin: 18px 0 6px; +} + +.candidate-head { + display: flex; + align-items: center; + gap: 10px; + flex-wrap: wrap; +} + +.candidate-head h3 { + margin: 0; + font-size: 16px; + text-transform: none; + letter-spacing: 0; + color: var(--text); +} + +.candidate textarea.code { + width: 100%; + box-sizing: border-box; + resize: vertical; +} + +/* checks grid: verbatim gate-check key names with a ✓/✗ mark */ +.checks-grid { + list-style: none; + margin: 0; + padding: 0; + display: grid; + grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); + gap: 4px 16px; +} + +.check { + display: flex; + align-items: baseline; + gap: 8px; + font-size: 13px; +} + +.check-mark { + width: 14px; + text-align: center; + font-weight: 700; +} + +.check-ok { + color: var(--ok); +} + +.check-fail { + color: var(--err); +} + +.notes { + margin: 8px 0 0; + padding-left: 20px; + color: var(--warn); + font-size: 13px; +} + +/* before/after preview pairs */ +.preview { + margin-bottom: 12px; +} + +.preview .badge { + margin: 4px 0; +} + +.badge-error, +.badge-rejected { + color: var(--err); +} + +.badge-parsed, +.badge-active, +.badge-human { + color: var(--ok); +} + +.badge-pending_review, +.badge-deactivated { + color: var(--warn); +} + +.badge-superseded { + color: var(--muted); +} + +.badge-slm { + color: var(--accent); +} + +.badge-slm-edited { + color: var(--quar); +} + +/* mapping editor */ +.mapping-head, +.mapping-row { + display: grid; + grid-template-columns: 1fr 1fr auto; + gap: 8px; + align-items: center; +} + +.mapping-head { + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.05em; + margin-bottom: 4px; +} + +.mapping-row { + margin-bottom: 6px; +} + +.mapping-row input { + background: var(--bg); + border: 1px solid var(--border); + border-radius: 6px; + color: var(--text); + padding: 6px 8px; + width: 100%; + box-sizing: border-box; +} + +.mapping-row input:focus-visible, +.review-actions input:focus-visible, +.override-pattern:focus-visible { + outline: none; + border-color: var(--accent); +} + +.mapping-editor button, +.review-actions button { + background: var(--panel-2); + color: var(--text); + border: 1px solid var(--border); + border-radius: 6px; + padding: 5px 10px; + font-size: 12px; + cursor: pointer; +} + +.mapping-editor button:hover, +.review-actions button:hover { + border-color: var(--accent); +} + +.add-mapping { + margin-top: 4px; +} + +/* actor / reason inputs + action buttons */ +.review-actions { + display: grid; + grid-template-columns: minmax(140px, 220px) 1fr; + gap: 12px; + margin-top: 16px; +} + +.review-actions label { + display: flex; + flex-direction: column; + gap: 4px; + font-size: 12px; +} + +.review-actions input { + background: var(--bg); + border: 1px solid var(--border); + border-radius: 6px; + color: var(--text); + padding: 6px 8px; +} + +.review-buttons { + display: flex; + gap: 8px; + margin-top: 12px; +} + +.review-buttons button { + border-radius: 6px; + padding: 7px 18px; + font-size: 14px; + cursor: pointer; + border: 1px solid var(--border); +} + +.review-buttons button:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.review-buttons .approve { + background: rgba(63, 185, 80, 0.15); + border-color: var(--ok); + color: var(--ok); +} + +.review-buttons .reject { + background: rgba(248, 81, 73, 0.12); + border-color: var(--err); + color: var(--err); +} + +.review-buttons .override-toggle { + background: var(--panel-2); + color: var(--text); +} + +.review-buttons .override-toggle:hover { + border-color: var(--accent); +} + +@media (max-width: 560px) { + .review-actions { + grid-template-columns: 1fr; + } +} + +/* --- rules registry (Task 9) --------------------------------------------- + The registry reuses .feed-table (borders, sticky header, row hover) plus + the badge-* tokens; the additions below only cover what no existing class + expresses: the full-width detail row, the timeline/audit lists and the + summary row's deactivate button. */ + +/* the expanded detail spans the whole table row — no row-pointer there */ +.rules-detail-row td, +.rules-error-row td { + cursor: default; + background: var(--panel-2); +} + +.rules-detail h3 { + font-size: 13px; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--muted); + margin: 14px 0 6px; +} + +.rules-detail h3:first-child { + margin-top: 4px; +} + +/* version timeline: one line per stored version, newest first */ +.version-timeline { + list-style: none; + margin: 0; + padding: 0; + display: grid; + gap: 4px; + font-size: 13px; +} + +.version-timeline li { + display: flex; + align-items: baseline; + gap: 10px; + flex-wrap: wrap; +} + +/* audit feed: `ts · actor · action` lines, action strings verbatim */ +.audit-feed { + list-style: none; + margin: 0; + padding: 0; + display: grid; + gap: 2px; + font-size: 13px; +} + +.audit-sep { + color: var(--muted); +} + +/* the summary row's only action button */ +.rules-table .deactivate { + background: rgba(248, 81, 73, 0.12); + border: 1px solid var(--err); + border-radius: 6px; + color: var(--err); + padding: 3px 10px; + font-size: 12px; + cursor: pointer; +} + +.rules-table .deactivate:hover { + border-color: var(--text); +} + +.rules-table .deactivate:disabled { + opacity: 0.5; + cursor: not-allowed; +} diff --git a/web/src/App.tsx b/web/src/App.tsx index 7e553c0..0c91aca 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,15 +1,27 @@ -// web/src/App.tsx — the M1 Overview page: 4 stat cards, the SSE live feed -// table, and a raw-traceability drawer. Same-origin /api only (dev: Vite -// proxy, prod: Caddy) — the gateway ships no CORS, and the dashboard is -// air-gapped: system fonts, zero external requests. +// web/src/App.tsx — the three-tab M2 shell: Overview (the M1 dashboard), +// Review Queue (Task 8 human review loop) and the Rules registry + audit +// trail (Task 9). +// Tab state is plain useState — no router (air-gapped dashboard: system +// fonts, zero external requests). Same-origin /api only (dev: Vite proxy, +// prod: Caddy) — the gateway ships no CORS. import { useCallback, useEffect, useState } from 'react' import { getRaw, getStats } from './api' import type { EventRow, OcsfEndpoint, RawTrace, Stats } from './types' import { useEventStream } from './useEventStream' +import ReviewQueue from './ReviewQueue' +import Rules from './Rules' import './App.css' const STATS_REFRESH_MS = 5000 +type Tab = 'overview' | 'review' | 'rules' + +const TABS: { id: Tab; label: string }[] = [ + { id: 'overview', label: 'Overview' }, + { id: 'review', label: 'Review Queue' }, + { id: 'rules', label: 'Rules' }, +] + /** parsed / (parsed + unparsed + parse_error + quarantined); "—" before any * traffic has been seen (divide-by-zero guard). */ function parsedPercent(byStatus: Stats['by_status']): string { @@ -51,23 +63,22 @@ function statCards(stats: Stats | null): StatCard[] { } /** Side drawer: full OCSF document plus the raw line behind the event - * (fetched by event_id). Fetch is cancelled if the drawer closes first. */ + * (fetched by event_id; the fetch is aborted via AbortController if the + * drawer closes first). */ function TraceDrawer({ row, onClose }: { row: EventRow; onClose: () => void }) { const [trace, setTrace] = useState(null) const [error, setError] = useState(null) useEffect(() => { - let cancelled = false - getRaw(row.event_id) - .then((t) => { - if (!cancelled) setTrace(t) - }) + const controller = new AbortController() + getRaw(row.event_id, controller.signal) + .then((t) => setTrace(t)) .catch((err: unknown) => { - if (!cancelled) setError(err instanceof Error ? err.message : String(err)) + if (!controller.signal.aborted) { + setError(err instanceof Error ? err.message : String(err)) + } }) - return () => { - cancelled = true - } + return () => controller.abort() }, [row.event_id]) useEffect(() => { @@ -131,7 +142,9 @@ function TraceDrawer({ row, onClose }: { row: EventRow; onClose: () => void }) { ) } -export default function App() { +/** The M1 Overview page: 4 stat cards, the SSE live-feed table, and the + * raw-traceability drawer. Unchanged behavior. */ +function Overview() { const [stats, setStats] = useState(null) const [statsError, setStatsError] = useState(null) const [selected, setSelected] = useState(null) @@ -155,7 +168,7 @@ export default function App() { }, [refreshStats]) return ( -
+

ULPF Overview

live feed · {rows.length} rows shown (cap 200) @@ -219,6 +232,32 @@ export default function App() { onClose={() => setSelected(null)} /> )} +
+ ) +} + +export default function App() { + const [tab, setTab] = useState('overview') + + return ( +
+ + {tab === 'overview' && } + {tab === 'review' && } + {tab === 'rules' && }
) } diff --git a/web/src/ReviewQueue.test.tsx b/web/src/ReviewQueue.test.tsx new file mode 100644 index 0000000..0b13ffb --- /dev/null +++ b/web/src/ReviewQueue.test.tsx @@ -0,0 +1,344 @@ +// @vitest-environment jsdom +// web/src/ReviewQueue.test.tsx — the Review Queue + three-tab shell tests +// (Task 8). The './api' module is mocked (house rule: tests never touch the +// network); the shell test additionally stubs EventSource, which jsdom does +// not ship, because Overview mounts the SSE hook. +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import App from './App' +import ReviewQueue from './ReviewQueue' +import { getRules, postJson } from './api' +import type { RuleRow, ValidationReport } from './types' + +vi.mock('./api', () => ({ + getStats: vi.fn().mockResolvedValue(null), + getEvents: vi.fn(), + getRaw: vi.fn().mockResolvedValue(null), + getRules: vi.fn(), + getRuleHistory: vi.fn(), + getAudit: vi.fn(), + getSamples: vi.fn(), + postJson: vi.fn(), +})) + +const mockGetRules = vi.mocked(getRules) +const mockPostJson = vi.mocked(postJson) + +/** Minimal EventSource double: jsdom has none, and the Overview feed only + * needs the constructor + close() to exist. */ +class StubEventSource { + static readonly CONNECTING = 0 + static readonly OPEN = 1 + static readonly CLOSED = 2 + readyState = StubEventSource.OPEN + onmessage: ((ev: MessageEvent) => void) | null = null + onerror: ((ev: Event) => void) | null = null + close(): void { + this.readyState = StubEventSource.CLOSED + } +} + +/** A pending_review candidate whose report has one failing check, a note, and + * one parsed preview — exercises the ✓/✗ grid and the preview pair. */ +function pendingRule(overrides: Partial = {}): RuleRow { + const validation: ValidationReport = { + passed: false, + checks: { + caps_and_allowlist: true, + samples_parse: true, + held_out_match_all: true, + ip_fields_valid: true, + port_fields_valid: true, + no_orphan_mappings: false, + adversarial_probe: true, + no_hardcoded_literals: true, + }, + held_out_match_rate: 0.75, + notes: ['held-out line 3: boom'], + previews: [ + { + line: 'Sep 19 09:05:11 fw1 kernel: DROP tcp 10.0.0.1:445 -> 10.0.0.2:445', + status: 'parsed', + ocsf: { + class_uid: 4001, + src_endpoint: { ip: '10.0.0.1', port: 445 }, + dst_endpoint: { ip: '10.0.0.2', port: 445 }, + }, + }, + ], + prompt_count: 5, + held_out_count: 4, + } + return { + id: 42, + fingerprint_id: 'fp-acme-fw', + version: 3, + pattern: '^kernel: (?PDROP|ACCEPT)\\s+.*src=(?P\\S+)', + mappings: [ + { source_field: 'src_ip', ocsf_path: 'src_endpoint.ip' }, + { source_field: 'action', ocsf_path: 'action' }, + ], + provenance: 'slm', + confidence: 0.87, + status: 'pending_review', + created_by: 'onboarding', + created_at: '2026-09-19T09:05:00Z', + activated_at: null, + deactivated_at: null, + validation, + ...overrides, + } +} + +beforeEach(() => { + vi.stubGlobal('EventSource', StubEventSource as unknown as typeof EventSource) +}) + +afterEach(() => { + cleanup() // vitest without globals does not auto-cleanup RTL renders + vi.unstubAllGlobals() +}) + +describe('ReviewQueue', () => { + it('renders a pending candidate: fingerprint, provenance, confidence, ✓/✗ checks grid', async () => { + mockGetRules.mockResolvedValue({ rules: [pendingRule()] }) + render() + // mount fetch is scoped to pending_review + await screen.findByText('fp-acme-fw') + expect(mockGetRules).toHaveBeenCalledWith({ status: 'pending_review' }) + expect(screen.getByText('slm')).not.toBeNull() + // T5-f: the honest number is held_out_match_rate, surfaced next to the + // optimistic prompt-only confidence + expect(screen.getByText('0.87')).not.toBeNull() + expect(screen.getByText('75%')).not.toBeNull() + // checks grid: verbatim key names, one ✓ per passing check, ✗ for the + // single failing one + expect(screen.getByText('caps_and_allowlist')).not.toBeNull() + expect(screen.getByText('no_orphan_mappings')).not.toBeNull() + expect(screen.getAllByText('✓')).toHaveLength(7) + expect(screen.getAllByText('✗')).toHaveLength(1) + // the report's notes are visible + expect(screen.getByText(/held-out line 3: boom/)).not.toBeNull() + }) + + it('renders a before/after preview pair: held-out line then pretty-printed OCSF', async () => { + mockGetRules.mockResolvedValue({ rules: [pendingRule()] }) + render() + await screen.findByText( + 'Sep 19 09:05:11 fw1 kernel: DROP tcp 10.0.0.1:445 -> 10.0.0.2:445', + ) + expect(screen.getByText(/"src_endpoint"/)).not.toBeNull() + expect(screen.getByText(/"dst_endpoint"/)).not.toBeNull() + }) + + it('Approve POSTs {actor, reason} from the inputs to the candidate URL, then refreshes', async () => { + const rule = pendingRule() + mockGetRules.mockResolvedValueOnce({ rules: [rule] }) // mount + mockGetRules.mockResolvedValueOnce({ rules: [] }) // post-approve refresh + mockPostJson.mockResolvedValueOnce({ rule_id: 42, version: 3, status: 'active' }) + render() + await screen.findByText('fp-acme-fw') + + fireEvent.change(screen.getByLabelText('actor'), { + target: { value: 'karthik' }, + }) + fireEvent.change(screen.getByLabelText('reason'), { + target: { value: 'matches the ACME DROP format' }, + }) + fireEvent.click(screen.getByRole('button', { name: 'Approve' })) + + await waitFor(() => { + expect(mockPostJson).toHaveBeenCalledWith( + '/api/rules/fp-acme-fw/candidates/42/approve', + { actor: 'karthik', reason: 'matches the ACME DROP format' }, + ) + }) + // the list is refetched after the action; the approved card is gone + await screen.findByText(/no pending candidates/i) + expect(mockGetRules).toHaveBeenCalledTimes(2) + }) + + it('Approve defaults the actor to anonymous when the input is untouched', async () => { + mockGetRules.mockResolvedValueOnce({ rules: [pendingRule()] }) + mockGetRules.mockResolvedValueOnce({ rules: [] }) + mockPostJson.mockResolvedValueOnce({ rule_id: 42, version: 3, status: 'active' }) + render() + await screen.findByText('fp-acme-fw') + fireEvent.click(screen.getByRole('button', { name: 'Approve' })) + await waitFor(() => { + expect(mockPostJson).toHaveBeenCalledWith( + '/api/rules/fp-acme-fw/candidates/42/approve', + { actor: 'anonymous' }, + ) + }) + }) + + it('shows an error banner when the approve POST fails (non-2xx)', async () => { + mockGetRules.mockResolvedValue({ rules: [pendingRule()] }) + mockPostJson.mockRejectedValueOnce( + new Error( + 'POST /api/rules/fp-acme-fw/candidates/42/approve failed: 409 Conflict', + ), + ) + render() + await screen.findByText('fp-acme-fw') + fireEvent.click(screen.getByRole('button', { name: 'Approve' })) + await screen.findByRole('alert') + expect(screen.getByRole('alert').textContent).toMatch(/409/) + // a failed action must not wipe the card — the reviewer can retry + expect(screen.getByText('fp-acme-fw')).not.toBeNull() + }) + + it('mapping editor edits a row and approve sends edited_mappings', async () => { + mockGetRules.mockResolvedValueOnce({ rules: [pendingRule()] }) + mockGetRules.mockResolvedValueOnce({ rules: [] }) + mockPostJson.mockResolvedValueOnce({ rule_id: 43, version: 4, status: 'active' }) + render() + await screen.findByText('fp-acme-fw') + + fireEvent.change(screen.getByLabelText('source_field row 1'), { + target: { value: 'src_addr' }, + }) + fireEvent.click(screen.getByRole('button', { name: 'Approve' })) + + await waitFor(() => { + expect(mockPostJson).toHaveBeenCalledWith( + '/api/rules/fp-acme-fw/candidates/42/approve', + { + actor: 'anonymous', + edited_mappings: [ + { source_field: 'src_addr', ocsf_path: 'src_endpoint.ip' }, + { source_field: 'action', ocsf_path: 'action' }, + ], + }, + ) + }) + }) + + it('mapping editor adds and removes rows before approve', async () => { + mockGetRules.mockResolvedValueOnce({ rules: [pendingRule()] }) + mockGetRules.mockResolvedValueOnce({ rules: [] }) + mockPostJson.mockResolvedValueOnce({ rule_id: 43, version: 4, status: 'active' }) + render() + await screen.findByText('fp-acme-fw') + + fireEvent.click(screen.getByRole('button', { name: 'Add mapping' })) + fireEvent.change(screen.getByLabelText('source_field row 3'), { + target: { value: 'msg' }, + }) + fireEvent.change(screen.getByLabelText('ocsf_path row 3'), { + target: { value: 'message' }, + }) + fireEvent.click(screen.getByRole('button', { name: 'Remove row 2' })) + fireEvent.click(screen.getByRole('button', { name: 'Approve' })) + + await waitFor(() => { + expect(mockPostJson).toHaveBeenCalledWith( + '/api/rules/fp-acme-fw/candidates/42/approve', + { + actor: 'anonymous', + edited_mappings: [ + { source_field: 'src_ip', ocsf_path: 'src_endpoint.ip' }, + { source_field: 'msg', ocsf_path: 'message' }, + ], + }, + ) + }) + }) + + it('override textarea populated -> approve sends override {pattern, mappings}', async () => { + mockGetRules.mockResolvedValueOnce({ rules: [pendingRule()] }) + mockGetRules.mockResolvedValueOnce({ rules: [] }) + mockPostJson.mockResolvedValueOnce({ rule_id: 44, version: 4, status: 'active' }) + render() + await screen.findByText('fp-acme-fw') + + fireEvent.click(screen.getByRole('button', { name: /override/i })) + fireEvent.change(screen.getByLabelText('override pattern'), { + target: { value: '^hand-written: (?P\\S+)' }, + }) + fireEvent.click(screen.getByRole('button', { name: 'Approve' })) + + await waitFor(() => { + expect(mockPostJson).toHaveBeenCalledWith( + '/api/rules/fp-acme-fw/candidates/42/approve', + { + actor: 'anonymous', + override: { + pattern: '^hand-written: (?P\\S+)', + mappings: [ + { source_field: 'src_ip', ocsf_path: 'src_endpoint.ip' }, + { source_field: 'action', ocsf_path: 'action' }, + ], + }, + }, + ) + }) + }) + + it('Reject POSTs {actor} and the card disappears after the refresh', async () => { + mockGetRules.mockResolvedValueOnce({ rules: [pendingRule()] }) + mockGetRules.mockResolvedValueOnce({ rules: [] }) + mockPostJson.mockResolvedValueOnce({ status: 'rejected' }) + render() + await screen.findByText('fp-acme-fw') + + fireEvent.change(screen.getByLabelText('actor'), { + target: { value: 'reviewer1' }, + }) + fireEvent.click(screen.getByRole('button', { name: 'Reject' })) + + await waitFor(() => { + expect(mockPostJson).toHaveBeenCalledWith( + '/api/rules/fp-acme-fw/candidates/42/reject', + { actor: 'reviewer1' }, + ) + }) + await screen.findByText(/no pending candidates/i) + expect(screen.queryByText('fp-acme-fw')).toBeNull() + }) + + it('shows the empty state when no candidates are pending', async () => { + mockGetRules.mockResolvedValue({ rules: [] }) + render() + await screen.findByText(/no pending candidates/i) + }) + + it('shows an error banner when the initial load fails', async () => { + mockGetRules.mockRejectedValue( + new Error('GET /api/rules failed: 500 Server Error'), + ) + render() + await screen.findByRole('alert') + expect(screen.getByRole('alert').textContent).toMatch(/500/) + }) +}) + +describe('three-tab shell', () => { + it('switches between Overview, Review Queue, and the Rules registry', async () => { + mockGetRules.mockResolvedValue({ rules: [pendingRule()] }) + render() + + // Overview is the landing tab: the live-feed page heading + expect(screen.getByRole('heading', { name: 'ULPF Overview' })).not.toBeNull() + expect(screen.queryByText('fp-acme-fw')).toBeNull() + + fireEvent.click(screen.getByRole('tab', { name: 'Review Queue' })) + expect(await screen.findByText('fp-acme-fw')).not.toBeNull() + expect( + screen.queryByRole('heading', { name: 'ULPF Overview' }), + ).toBeNull() + + // Task 9: the Rules tab is the real registry now — the fingerprint row + // renders (pending_review shows no Deactivate button) + fireEvent.click(screen.getByRole('tab', { name: 'Rules' })) + expect(await screen.findByText('fp-acme-fw')).not.toBeNull() + expect(screen.queryByRole('button', { name: 'Deactivate' })).toBeNull() + expect( + screen.queryByRole('heading', { name: 'ULPF Overview' }), + ).toBeNull() + + fireEvent.click(screen.getByRole('tab', { name: 'Overview' })) + expect(screen.getByRole('heading', { name: 'ULPF Overview' })).not.toBeNull() + }) +}) diff --git a/web/src/ReviewQueue.tsx b/web/src/ReviewQueue.tsx new file mode 100644 index 0000000..1a4ae0f --- /dev/null +++ b/web/src/ReviewQueue.tsx @@ -0,0 +1,337 @@ +// web/src/ReviewQueue.tsx — the human rule-review loop (Task 8 hero page). +// +// Fetches every pending_review candidate (GET /api/rules?status=pending_review) +// on mount and after every action, and renders one card per candidate: +// - the Task-3 gate's checks grid — check keys rendered VERBATIM (Task 3 +// ruling: `caps_and_allowlist` may be false for cap, allow-list OR +// per-sample schema failures, so the key itself is the contract); +// - before/after preview pairs (held-out line + parsed OCSF JSON); +// - confidence AND validation.held_out_match_rate (T5-f: SLM confidence is +// prompt-only and optimistic — the held-out rate is the honest number); +// - a mapping editor whose edits travel as `edited_mappings`; +// - an override pattern textarea whose contents travel as `override`; +// - actor/reason inputs and Approve/Reject with disabled-while-submitting +// and an error banner on non-2xx. +// +// Air-gap rules: same-origin /api only, system fonts, zero external requests. +import { useCallback, useEffect, useState } from 'react' +import { getRules, postJson } from './api' +import type { ApproveBody, RejectBody, RuleMapping, RuleRow } from './types' + +const PENDING_STATUS = 'pending_review' + +function pct(rate: number): string { + return `${Math.round(rate * 100)}%` +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err) +} + +/** The gate's checks grid: verbatim key names, one ✓ or ✗ per check. */ +function ChecksGrid({ validation }: { validation: NonNullable }) { + return ( +
+

Validation checks

+
    + {Object.entries(validation.checks).map(([key, ok]) => ( +
  • + + {ok ? '✓' : '✗'} + + {key} +
    • + ))} +
+ {validation.notes.length > 0 && ( +
    + {validation.notes.map((note, i) => ( +
  • {note}
    • + ))} +
+ )} +
+ ) +} + +/** Before/after pairs: the held-out sample line and what the candidate's + * parser produced for it (pretty-printed OCSF, or a parse-error badge). */ +function Previews({ validation }: { validation: NonNullable }) { + if (validation.previews.length === 0) return null + return ( +
+

Previews (held-out line → OCSF)

+ {validation.previews.map((p, i) => ( +
+ 
{p.line}
+ {p.status} + 
+            {p.ocsf !== null ? JSON.stringify(p.ocsf, null, 2) : '(line failed to parse)'}
+
+
+ ))} +
+ ) +} + +/** Editable source_field/ocsf_path rows: add, edit, remove before approving. + * Changes only travel to the gateway when they differ from the candidate. */ +function MappingEditor({ + mappings, + onChange, +}: { + mappings: RuleMapping[] + onChange: (next: RuleMapping[]) => void +}) { + const update = (index: number, key: keyof RuleMapping, value: string) => { + onChange(mappings.map((m, i) => (i === index ? { ...m, [key]: value } : m))) + } + return ( +
+
+ source_field + ocsf_path + +
+ {mappings.map((m, i) => ( +
+ update(i, 'source_field', e.target.value)} + /> + update(i, 'ocsf_path', e.target.value)} + /> + +
+ ))} + +
+ ) +} + +/** One pending candidate with its review controls. */ +function CandidateCard({ rule, onDone }: { rule: RuleRow; onDone: () => void }) { + const [mappings, setMappings] = useState(() => + rule.mappings.map((m) => ({ ...m })), + ) + const [overrideOn, setOverrideOn] = useState(false) + const [overridePattern, setOverridePattern] = useState(rule.pattern) + const [actor, setActor] = useState('anonymous') + const [reason, setReason] = useState('') + const [submitting, setSubmitting] = useState(false) + const [error, setError] = useState(null) + + const actorValue = actor.trim() === '' ? 'anonymous' : actor.trim() + const reasonValue = reason.trim() + const mappingsChanged = + JSON.stringify(mappings) !== JSON.stringify(rule.mappings) + + const approveBody = (): ApproveBody => { + const body: ApproveBody = { actor: actorValue } + if (reasonValue !== '') body.reason = reasonValue + if (overrideOn) { + // override wins wholesale: pattern textarea + current editor rows + body.override = { pattern: overridePattern, mappings } + } else if (mappingsChanged) { + body.edited_mappings = mappings + } + return body + } + + const rejectBody = (): RejectBody => { + const body: RejectBody = { actor: actorValue } + if (reasonValue !== '') body.reason = reasonValue + return body + } + + const act = (action: 'approve' | 'reject') => { + setSubmitting(true) + setError(null) + const url = `/api/rules/${encodeURIComponent(rule.fingerprint_id)}/candidates/${rule.id}/${action}` + postJson(url, action === 'approve' ? approveBody() : rejectBody()) + .then(() => onDone()) // refetch — the card leaves the queue + .catch((err: unknown) => setError(errorMessage(err))) + .finally(() => setSubmitting(false)) + } + + return ( +
+
+

{rule.fingerprint_id}

+ {rule.provenance} + v{rule.version} +
+ +
+
Confidence
+
+ {rule.confidence ?? '—'} + {rule.provenance === 'slm' && ( + (prompt-only, optimistic — see held-out rate) + )} +
+ {rule.validation !== null && ( + <> +
Held-out match
+
{pct(rule.validation.held_out_match_rate)}
+ + )} +
Created by
+
{rule.created_by}
+
+ +

Pattern

+ {overrideOn ? ( +