From 23e79e1b830156661d5c727a3a283430d9cb53a1 Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 11:50:15 +0530 Subject: [PATCH 01/18] chore: restructure repo for MVP; demo moves to legacy-demo/ Co-Authored-By: Claude Code --- .gitignore | 12 +- README.md | 111 ++---------------- {drift_monitor => bench}/.gitkeep | 0 {ingestion => deploy}/.gitkeep | 0 {onboarding => docs}/.gitkeep | 0 app.py => legacy-demo/app.py | 0 {docs => legacy-demo/docs}/drift-monitor.md | 0 {docs => legacy-demo/docs}/ingestion.md | 0 {docs => legacy-demo/docs}/onboarding.md | 0 {docs => legacy-demo/docs}/parser-engine.md | 0 {docs => legacy-demo/docs}/review-ui.md | 0 {docs => legacy-demo/docs}/testing.md | 0 .../drift_monitor}/.gitkeep | 0 .../drift_monitor}/__init__.py | 0 .../drift_monitor}/demo_replay.py | 0 .../drift_monitor}/monitor.py | 0 .../drift_monitor}/plot_drift.py | 0 .../drift_monitor}/simulate_drift.py | 0 .../drift_monitor}/test_day1.py | 0 .../drift_monitor}/test_simulate_drift.py | 0 {review_ui => legacy-demo/ingestion}/.gitkeep | 0 .../ingestion}/__init__.py | 0 .../ingestion}/ingest.py | 0 .../ingestion}/validate_rule.py | 0 .../onboarding/.gitkeep | 0 .../onboarding}/__init__.py | 0 .../onboarding}/generate_rule.py | 0 .../onboarding}/test_generate_rule.py | 0 .../onboarding}/test_multi_format.py | 0 .../parser_engine/.gitkeep | 0 .../parser_engine}/__init__.py | 0 .../parser_engine}/apply_rule.py | 0 .../parser_engine}/fingerprint.py | 0 .../parser_engine}/rule_store.py | 0 .../parser_engine}/test_parser_engine.py | 0 .../review_ui/.gitkeep | 0 legacy-demo/review_ui/__init__.py | 0 {review_ui => legacy-demo/review_ui}/app.py | 0 .../review_ui}/data_access.py | 0 .../review_ui}/pages/1_Rule_Review.py | 0 .../pages/2_Normalized_Event_Browser.py | 0 legacy-demo/schemas/__init__.py | 0 {schemas => legacy-demo/schemas}/schemas.py | 0 .../test_generate_rule_compat.py | 0 .../test_validate_rule.py | 0 legacy-demo/testdata/__init__.py | 0 .../testdata}/check_fixtures.py | 0 .../testdata}/fixtures.py | 0 .../fixtures/bad_rule_fixture_syslog.json | 0 .../testdata}/fixtures/drift_sequence.json | 0 .../fixtures/sample_normalized_event.json | 0 .../testdata}/fixtures/sample_raw_event.json | 0 .../testdata}/fixtures/sample_rule.json | 0 .../testdata}/fixtures/sample_rule_bad.json | 0 .../fixtures/sample_rule_extension_style.json | 0 .../fixtures/sample_validation_result.json | 0 .../testdata}/integration_runner.py | 0 .../testdata}/raw_logs_acmegw.txt | 0 .../testdata}/raw_logs_cef.txt | 0 .../testdata}/raw_logs_json.txt | 0 .../testdata}/raw_logs_syslog.txt | 0 libs/.gitkeep | 0 pyproject.toml | 7 ++ scripts/.gitkeep | 0 services/.gitkeep | 0 simulator/.gitkeep | 0 web/.gitkeep | 0 67 files changed, 27 insertions(+), 103 deletions(-) rename {drift_monitor => bench}/.gitkeep (100%) rename {ingestion => deploy}/.gitkeep (100%) rename {onboarding => docs}/.gitkeep (100%) rename app.py => legacy-demo/app.py (100%) rename {docs => legacy-demo/docs}/drift-monitor.md (100%) rename {docs => legacy-demo/docs}/ingestion.md (100%) rename {docs => legacy-demo/docs}/onboarding.md (100%) rename {docs => legacy-demo/docs}/parser-engine.md (100%) rename {docs => legacy-demo/docs}/review-ui.md (100%) rename {docs => legacy-demo/docs}/testing.md (100%) rename {parser_engine => legacy-demo/drift_monitor}/.gitkeep (100%) rename {drift_monitor => legacy-demo/drift_monitor}/__init__.py (100%) rename {drift_monitor => legacy-demo/drift_monitor}/demo_replay.py (100%) rename {drift_monitor => legacy-demo/drift_monitor}/monitor.py (100%) rename {drift_monitor => legacy-demo/drift_monitor}/plot_drift.py (100%) rename {drift_monitor => legacy-demo/drift_monitor}/simulate_drift.py (100%) rename {drift_monitor => legacy-demo/drift_monitor}/test_day1.py (100%) rename {drift_monitor => legacy-demo/drift_monitor}/test_simulate_drift.py (100%) rename {review_ui => legacy-demo/ingestion}/.gitkeep (100%) rename {ingestion => legacy-demo/ingestion}/__init__.py (100%) rename {ingestion => legacy-demo/ingestion}/ingest.py (100%) rename {ingestion => legacy-demo/ingestion}/validate_rule.py (100%) rename onboarding/__init__.py => legacy-demo/onboarding/.gitkeep (100%) rename {parser_engine => legacy-demo/onboarding}/__init__.py (100%) rename {onboarding => legacy-demo/onboarding}/generate_rule.py (100%) rename {onboarding => legacy-demo/onboarding}/test_generate_rule.py (100%) rename {onboarding => legacy-demo/onboarding}/test_multi_format.py (100%) rename review_ui/__init__.py => legacy-demo/parser_engine/.gitkeep (100%) rename {schemas => legacy-demo/parser_engine}/__init__.py (100%) rename {parser_engine => legacy-demo/parser_engine}/apply_rule.py (100%) rename {parser_engine => legacy-demo/parser_engine}/fingerprint.py (100%) rename {parser_engine => legacy-demo/parser_engine}/rule_store.py (100%) rename {parser_engine => legacy-demo/parser_engine}/test_parser_engine.py (100%) rename testdata/__init__.py => legacy-demo/review_ui/.gitkeep (100%) create mode 100644 legacy-demo/review_ui/__init__.py rename {review_ui => legacy-demo/review_ui}/app.py (100%) rename {review_ui => legacy-demo/review_ui}/data_access.py (100%) rename {review_ui => legacy-demo/review_ui}/pages/1_Rule_Review.py (100%) rename {review_ui => legacy-demo/review_ui}/pages/2_Normalized_Event_Browser.py (100%) create mode 100644 legacy-demo/schemas/__init__.py rename {schemas => legacy-demo/schemas}/schemas.py (100%) rename test_generate_rule_compat.py => legacy-demo/test_generate_rule_compat.py (100%) rename test_validate_rule.py => legacy-demo/test_validate_rule.py (100%) create mode 100644 legacy-demo/testdata/__init__.py rename {testdata => legacy-demo/testdata}/check_fixtures.py (100%) rename {testdata => legacy-demo/testdata}/fixtures.py (100%) rename {testdata => legacy-demo/testdata}/fixtures/bad_rule_fixture_syslog.json (100%) rename {testdata => legacy-demo/testdata}/fixtures/drift_sequence.json (100%) rename {testdata => legacy-demo/testdata}/fixtures/sample_normalized_event.json (100%) rename {testdata => legacy-demo/testdata}/fixtures/sample_raw_event.json (100%) rename {testdata => legacy-demo/testdata}/fixtures/sample_rule.json (100%) rename {testdata => legacy-demo/testdata}/fixtures/sample_rule_bad.json (100%) rename {testdata => legacy-demo/testdata}/fixtures/sample_rule_extension_style.json (100%) rename {testdata => legacy-demo/testdata}/fixtures/sample_validation_result.json (100%) rename {testdata => legacy-demo/testdata}/integration_runner.py (100%) rename {testdata => legacy-demo/testdata}/raw_logs_acmegw.txt (100%) rename {testdata => legacy-demo/testdata}/raw_logs_cef.txt (100%) rename {testdata => legacy-demo/testdata}/raw_logs_json.txt (100%) rename {testdata => legacy-demo/testdata}/raw_logs_syslog.txt (100%) create mode 100644 libs/.gitkeep create mode 100644 pyproject.toml create mode 100644 scripts/.gitkeep create mode 100644 services/.gitkeep create mode 100644 simulator/.gitkeep create mode 100644 web/.gitkeep diff --git a/.gitignore b/.gitignore index e17f188..aeb1679 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,14 @@ Thumbs.db .ollama #zip files *.zip -testdata/registered_rules.json \ No newline at end of file +testdata/registered_rules.json +#references +references/ + +# MVP build artifacts +node_modules/ +web/dist/ +.venv/ +*.egg-info/ +__pycache__/ +.pytest_cache/ \ No newline at end of file diff --git a/README.md b/README.md index 416cac4..ed8ff9d 100644 --- a/README.md +++ b/README.md @@ -1,104 +1,11 @@ # ULPF — Universal Log Pre-processing Framework -A prototype built for Smart India Hackathon 2026, Problem Statement 26156 -(NTRO, Blockchain & Cybersecurity theme). - -## What this is - -ULPF ingests security logs from perimeter network devices (firewalls, -IDS, VPN gateways) in heterogeneous vendor formats — Syslog, CEF, LEEF, -JSON, CSV, and undocumented proprietary formats — and normalizes them -into a common schema (OCSF) for SIEM and analytics use. - -The core design idea: when a genuinely new, unrecognized log format -appears, a small local language model (SLM) proposes a parsing rule -(a regex + field mapping) from a handful of sample lines. A human -reviews, edits, or overrides that candidate before it's trusted on real -traffic. From that point on, parsing is fully deterministic — the model -is never called again for that format unless it later drifts. - -## Architecture - -``` -Perimeter device logs - | - Ingestion + raw storage (immutable, before any processing) - | - Format fingerprinting (deterministic, no model calls) - | - Known format? ----no----> SLM generates candidate rule - | | - yes Human review (approve/edit/override) - | | - Deterministic parser <---------+ - | - Normalized OCSF event (linked to raw log for traceability) - | - Drift monitoring (continuous; degraded rules fail closed) - | - SIEM / data lake -``` - -Full technical rationale, design decisions, and references are in -`docs/`. - -## Repo structure - -``` -schemas/schemas.py — canonical data contracts used by every module -onboarding/ — SLM-based rule generation for new formats -parser_engine/ — format fingerprinting + deterministic parsing -ingestion/ — raw log storage, traceability, rule validation -review_ui/ — Streamlit review-gate UI + event browser -drift_monitor/ — rule health tracking and drift response -testdata/ — sample logs, fixtures, integration test runner -app.py — wires every module into the end-to-end pipeline -docs/ — module-level documentation -``` - -## Running it - -**One-time setup:** -``` -pip install ollama streamlit -ollama pull ibm/granite4.1:3b # or a larger tag if available -``` - -**Run the full pipeline once, to onboard and register rules:** -``` -python testdata/integration_runner.py -``` - -**Launch the review UI (uses the rules the runner just registered):** -``` -streamlit run review_ui/app.py -``` - -Run the integration runner at least once before the UI — the UI reads -from the same persisted rule store, and falls back to fixture data if -no real rules are registered yet. - -## Module documentation - -- [`docs/onboarding.md`](docs/onboarding.md) — SLM rule generation -- [`docs/parser-engine.md`](docs/parser-engine.md) — fingerprinting + deterministic parsing -- [`docs/ingestion.md`](docs/ingestion.md) — raw storage + rule validation -- [`docs/review-ui.md`](docs/review-ui.md) — human review gate + event browser -- [`docs/drift-monitor.md`](docs/drift-monitor.md) — rule health monitoring -- [`docs/testing.md`](docs/testing.md) — test data and the integration runner - -## Scope notes - -Built as a hackathon prototype in a short timeframe. A few deliberate -scope decisions, documented for transparency rather than hidden: - -- **Kafka, Postgres**, and horizontally-scaled deployment are part of - the intended production architecture (see the idea presentation) but - are not implemented here — SQLite and in-process function calls are - sufficient to demonstrate the design at prototype scale. -- **Drift detection** uses simple threshold-based severity - classification, not full statistical process control — documented as - the honest scope for this stage. -- One deliberately difficult, invented vendor format is intentionally - left un-auto-onboarded in test data, to demonstrate the manual - override path when the model can't fully resolve a format on its own. \ No newline at end of file +SIH 2026 PS 26156 MVP. Status: M1 walking skeleton in development. + +- `services/` — collector, pipeline, gateway (+ onboarding, drift in M2/M3) +- `libs/ulpf-core` — fingerprinting, parsing, IDs, hash chain (single implementation) +- `libs/ocsf-schema` — curated OCSF subset schema + validator +- `web/` — React dashboard +- `simulator/` — multi-vendor traffic player +- `deploy/` — docker-compose, migrations +- `legacy-demo/` — the original hackathon demo, frozen. See its README. diff --git a/drift_monitor/.gitkeep b/bench/.gitkeep similarity index 100% rename from drift_monitor/.gitkeep rename to bench/.gitkeep diff --git a/ingestion/.gitkeep b/deploy/.gitkeep similarity index 100% rename from ingestion/.gitkeep rename to deploy/.gitkeep diff --git a/onboarding/.gitkeep b/docs/.gitkeep similarity index 100% rename from onboarding/.gitkeep rename to docs/.gitkeep diff --git a/app.py b/legacy-demo/app.py similarity index 100% rename from app.py rename to legacy-demo/app.py diff --git a/docs/drift-monitor.md b/legacy-demo/docs/drift-monitor.md similarity index 100% rename from docs/drift-monitor.md rename to legacy-demo/docs/drift-monitor.md diff --git a/docs/ingestion.md b/legacy-demo/docs/ingestion.md similarity index 100% rename from docs/ingestion.md rename to legacy-demo/docs/ingestion.md diff --git a/docs/onboarding.md b/legacy-demo/docs/onboarding.md similarity index 100% rename from docs/onboarding.md rename to legacy-demo/docs/onboarding.md diff --git a/docs/parser-engine.md b/legacy-demo/docs/parser-engine.md similarity index 100% rename from docs/parser-engine.md rename to legacy-demo/docs/parser-engine.md diff --git a/docs/review-ui.md b/legacy-demo/docs/review-ui.md similarity index 100% rename from docs/review-ui.md rename to legacy-demo/docs/review-ui.md diff --git a/docs/testing.md b/legacy-demo/docs/testing.md similarity index 100% rename from docs/testing.md rename to legacy-demo/docs/testing.md diff --git a/parser_engine/.gitkeep b/legacy-demo/drift_monitor/.gitkeep similarity index 100% rename from parser_engine/.gitkeep rename to legacy-demo/drift_monitor/.gitkeep diff --git a/drift_monitor/__init__.py b/legacy-demo/drift_monitor/__init__.py similarity index 100% rename from drift_monitor/__init__.py rename to legacy-demo/drift_monitor/__init__.py diff --git a/drift_monitor/demo_replay.py b/legacy-demo/drift_monitor/demo_replay.py similarity index 100% rename from drift_monitor/demo_replay.py rename to legacy-demo/drift_monitor/demo_replay.py diff --git a/drift_monitor/monitor.py b/legacy-demo/drift_monitor/monitor.py similarity index 100% rename from drift_monitor/monitor.py rename to legacy-demo/drift_monitor/monitor.py diff --git a/drift_monitor/plot_drift.py b/legacy-demo/drift_monitor/plot_drift.py similarity index 100% rename from drift_monitor/plot_drift.py rename to legacy-demo/drift_monitor/plot_drift.py diff --git a/drift_monitor/simulate_drift.py b/legacy-demo/drift_monitor/simulate_drift.py similarity index 100% rename from drift_monitor/simulate_drift.py rename to legacy-demo/drift_monitor/simulate_drift.py diff --git a/drift_monitor/test_day1.py b/legacy-demo/drift_monitor/test_day1.py similarity index 100% rename from drift_monitor/test_day1.py rename to legacy-demo/drift_monitor/test_day1.py diff --git a/drift_monitor/test_simulate_drift.py b/legacy-demo/drift_monitor/test_simulate_drift.py similarity index 100% rename from drift_monitor/test_simulate_drift.py rename to legacy-demo/drift_monitor/test_simulate_drift.py diff --git a/review_ui/.gitkeep b/legacy-demo/ingestion/.gitkeep similarity index 100% rename from review_ui/.gitkeep rename to legacy-demo/ingestion/.gitkeep diff --git a/ingestion/__init__.py b/legacy-demo/ingestion/__init__.py similarity index 100% rename from ingestion/__init__.py rename to legacy-demo/ingestion/__init__.py diff --git a/ingestion/ingest.py b/legacy-demo/ingestion/ingest.py similarity index 100% rename from ingestion/ingest.py rename to legacy-demo/ingestion/ingest.py diff --git a/ingestion/validate_rule.py b/legacy-demo/ingestion/validate_rule.py similarity index 100% rename from ingestion/validate_rule.py rename to legacy-demo/ingestion/validate_rule.py diff --git a/onboarding/__init__.py b/legacy-demo/onboarding/.gitkeep similarity index 100% rename from onboarding/__init__.py rename to legacy-demo/onboarding/.gitkeep diff --git a/parser_engine/__init__.py b/legacy-demo/onboarding/__init__.py similarity index 100% rename from parser_engine/__init__.py rename to legacy-demo/onboarding/__init__.py diff --git a/onboarding/generate_rule.py b/legacy-demo/onboarding/generate_rule.py similarity index 100% rename from onboarding/generate_rule.py rename to legacy-demo/onboarding/generate_rule.py diff --git a/onboarding/test_generate_rule.py b/legacy-demo/onboarding/test_generate_rule.py similarity index 100% rename from onboarding/test_generate_rule.py rename to legacy-demo/onboarding/test_generate_rule.py diff --git a/onboarding/test_multi_format.py b/legacy-demo/onboarding/test_multi_format.py similarity index 100% rename from onboarding/test_multi_format.py rename to legacy-demo/onboarding/test_multi_format.py diff --git a/review_ui/__init__.py b/legacy-demo/parser_engine/.gitkeep similarity index 100% rename from review_ui/__init__.py rename to legacy-demo/parser_engine/.gitkeep diff --git a/schemas/__init__.py b/legacy-demo/parser_engine/__init__.py similarity index 100% rename from schemas/__init__.py rename to legacy-demo/parser_engine/__init__.py diff --git a/parser_engine/apply_rule.py b/legacy-demo/parser_engine/apply_rule.py similarity index 100% rename from parser_engine/apply_rule.py rename to legacy-demo/parser_engine/apply_rule.py diff --git a/parser_engine/fingerprint.py b/legacy-demo/parser_engine/fingerprint.py similarity index 100% rename from parser_engine/fingerprint.py rename to legacy-demo/parser_engine/fingerprint.py diff --git a/parser_engine/rule_store.py b/legacy-demo/parser_engine/rule_store.py similarity index 100% rename from parser_engine/rule_store.py rename to legacy-demo/parser_engine/rule_store.py diff --git a/parser_engine/test_parser_engine.py b/legacy-demo/parser_engine/test_parser_engine.py similarity index 100% rename from parser_engine/test_parser_engine.py rename to legacy-demo/parser_engine/test_parser_engine.py diff --git a/testdata/__init__.py b/legacy-demo/review_ui/.gitkeep similarity index 100% rename from testdata/__init__.py rename to legacy-demo/review_ui/.gitkeep diff --git a/legacy-demo/review_ui/__init__.py b/legacy-demo/review_ui/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/review_ui/app.py b/legacy-demo/review_ui/app.py similarity index 100% rename from review_ui/app.py rename to legacy-demo/review_ui/app.py diff --git a/review_ui/data_access.py b/legacy-demo/review_ui/data_access.py similarity index 100% rename from review_ui/data_access.py rename to legacy-demo/review_ui/data_access.py diff --git a/review_ui/pages/1_Rule_Review.py b/legacy-demo/review_ui/pages/1_Rule_Review.py similarity index 100% rename from review_ui/pages/1_Rule_Review.py rename to legacy-demo/review_ui/pages/1_Rule_Review.py diff --git a/review_ui/pages/2_Normalized_Event_Browser.py b/legacy-demo/review_ui/pages/2_Normalized_Event_Browser.py similarity index 100% rename from review_ui/pages/2_Normalized_Event_Browser.py rename to legacy-demo/review_ui/pages/2_Normalized_Event_Browser.py diff --git a/legacy-demo/schemas/__init__.py b/legacy-demo/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/schemas/schemas.py b/legacy-demo/schemas/schemas.py similarity index 100% rename from schemas/schemas.py rename to legacy-demo/schemas/schemas.py diff --git a/test_generate_rule_compat.py b/legacy-demo/test_generate_rule_compat.py similarity index 100% rename from test_generate_rule_compat.py rename to legacy-demo/test_generate_rule_compat.py diff --git a/test_validate_rule.py b/legacy-demo/test_validate_rule.py similarity index 100% rename from test_validate_rule.py rename to legacy-demo/test_validate_rule.py diff --git a/legacy-demo/testdata/__init__.py b/legacy-demo/testdata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/testdata/check_fixtures.py b/legacy-demo/testdata/check_fixtures.py similarity index 100% rename from testdata/check_fixtures.py rename to legacy-demo/testdata/check_fixtures.py diff --git a/testdata/fixtures.py b/legacy-demo/testdata/fixtures.py similarity index 100% rename from testdata/fixtures.py rename to legacy-demo/testdata/fixtures.py diff --git a/testdata/fixtures/bad_rule_fixture_syslog.json b/legacy-demo/testdata/fixtures/bad_rule_fixture_syslog.json similarity index 100% rename from testdata/fixtures/bad_rule_fixture_syslog.json rename to legacy-demo/testdata/fixtures/bad_rule_fixture_syslog.json diff --git a/testdata/fixtures/drift_sequence.json b/legacy-demo/testdata/fixtures/drift_sequence.json similarity index 100% rename from testdata/fixtures/drift_sequence.json rename to legacy-demo/testdata/fixtures/drift_sequence.json diff --git a/testdata/fixtures/sample_normalized_event.json b/legacy-demo/testdata/fixtures/sample_normalized_event.json similarity index 100% rename from testdata/fixtures/sample_normalized_event.json rename to legacy-demo/testdata/fixtures/sample_normalized_event.json diff --git a/testdata/fixtures/sample_raw_event.json b/legacy-demo/testdata/fixtures/sample_raw_event.json similarity index 100% rename from testdata/fixtures/sample_raw_event.json rename to legacy-demo/testdata/fixtures/sample_raw_event.json diff --git a/testdata/fixtures/sample_rule.json b/legacy-demo/testdata/fixtures/sample_rule.json similarity index 100% rename from testdata/fixtures/sample_rule.json rename to legacy-demo/testdata/fixtures/sample_rule.json diff --git a/testdata/fixtures/sample_rule_bad.json b/legacy-demo/testdata/fixtures/sample_rule_bad.json similarity index 100% rename from testdata/fixtures/sample_rule_bad.json rename to legacy-demo/testdata/fixtures/sample_rule_bad.json diff --git a/testdata/fixtures/sample_rule_extension_style.json b/legacy-demo/testdata/fixtures/sample_rule_extension_style.json similarity index 100% rename from testdata/fixtures/sample_rule_extension_style.json rename to legacy-demo/testdata/fixtures/sample_rule_extension_style.json diff --git a/testdata/fixtures/sample_validation_result.json b/legacy-demo/testdata/fixtures/sample_validation_result.json similarity index 100% rename from testdata/fixtures/sample_validation_result.json rename to legacy-demo/testdata/fixtures/sample_validation_result.json diff --git a/testdata/integration_runner.py b/legacy-demo/testdata/integration_runner.py similarity index 100% rename from testdata/integration_runner.py rename to legacy-demo/testdata/integration_runner.py diff --git a/testdata/raw_logs_acmegw.txt b/legacy-demo/testdata/raw_logs_acmegw.txt similarity index 100% rename from testdata/raw_logs_acmegw.txt rename to legacy-demo/testdata/raw_logs_acmegw.txt diff --git a/testdata/raw_logs_cef.txt b/legacy-demo/testdata/raw_logs_cef.txt similarity index 100% rename from testdata/raw_logs_cef.txt rename to legacy-demo/testdata/raw_logs_cef.txt diff --git a/testdata/raw_logs_json.txt b/legacy-demo/testdata/raw_logs_json.txt similarity index 100% rename from testdata/raw_logs_json.txt rename to legacy-demo/testdata/raw_logs_json.txt diff --git a/testdata/raw_logs_syslog.txt b/legacy-demo/testdata/raw_logs_syslog.txt similarity index 100% rename from testdata/raw_logs_syslog.txt rename to legacy-demo/testdata/raw_logs_syslog.txt diff --git a/libs/.gitkeep b/libs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..17be3ab --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[tool.pytest.ini_options] +testpaths = ["libs", "services"] +pythonpath = ["."] + +[tool.ruff] +line-length = 100 +target-version = "py312" diff --git a/scripts/.gitkeep b/scripts/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/services/.gitkeep b/services/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/simulator/.gitkeep b/simulator/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/web/.gitkeep b/web/.gitkeep new file mode 100644 index 0000000..e69de29 From da5c181f537ccb202a8aae062c115dad2f8f3c8a Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 12:01:14 +0530 Subject: [PATCH 02/18] feat: ulpf-core package with envelope models and deterministic IDs Co-Authored-By: Claude Code --- libs/ulpf-core/pyproject.toml | 12 ++++++++++++ libs/ulpf-core/tests/test_ids.py | 14 ++++++++++++++ libs/ulpf-core/tests/test_models.py | 13 +++++++++++++ libs/ulpf-core/ulpf_core/__init__.py | 0 libs/ulpf-core/ulpf_core/ids.py | 13 +++++++++++++ libs/ulpf-core/ulpf_core/models.py | 19 +++++++++++++++++++ 6 files changed, 71 insertions(+) create mode 100644 libs/ulpf-core/pyproject.toml create mode 100644 libs/ulpf-core/tests/test_ids.py create mode 100644 libs/ulpf-core/tests/test_models.py create mode 100644 libs/ulpf-core/ulpf_core/__init__.py create mode 100644 libs/ulpf-core/ulpf_core/ids.py create mode 100644 libs/ulpf-core/ulpf_core/models.py diff --git a/libs/ulpf-core/pyproject.toml b/libs/ulpf-core/pyproject.toml new file mode 100644 index 0000000..c548e1a --- /dev/null +++ b/libs/ulpf-core/pyproject.toml @@ -0,0 +1,12 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "ulpf-core" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = ["pydantic~=2.7"] + +[tool.setuptools] +packages = ["ulpf_core"] diff --git a/libs/ulpf-core/tests/test_ids.py b/libs/ulpf-core/tests/test_ids.py new file mode 100644 index 0000000..cdf2843 --- /dev/null +++ b/libs/ulpf-core/tests/test_ids.py @@ -0,0 +1,14 @@ +import uuid +from ulpf_core.ids import raw_id_for, event_id_for + +def test_raw_id_deterministic(): + assert raw_id_for("raw.logs", 3, 9182) == raw_id_for("raw.logs", 3, 9182) + +def test_raw_id_varies_by_position(): + a, b, c = raw_id_for("raw.logs", 3, 1), raw_id_for("raw.logs", 4, 1), raw_id_for("raw.logs", 3, 2) + assert len({a, b, c}) == 3 + +def test_event_id_versioned(): + r = uuid.UUID("00000000-0000-0000-0000-000000000001") + assert event_id_for(r, 1) != event_id_for(r, 2) + assert event_id_for(r, 1) == event_id_for(r, 1) diff --git a/libs/ulpf-core/tests/test_models.py b/libs/ulpf-core/tests/test_models.py new file mode 100644 index 0000000..510b2ca --- /dev/null +++ b/libs/ulpf-core/tests/test_models.py @@ -0,0 +1,13 @@ +import pytest +from ulpf_core.models import RawEnvelope + +def test_envelope_roundtrip(): + e = RawEnvelope(source_id="fw01", transport="syslog-udp", + received_at="2026-09-19T10:00:00+00:00", raw="Aug 27 14:32:07 fw01 kernel: X") + assert RawEnvelope.model_validate_json(e.model_dump_json()) == e + +def test_envelope_rejects_empty_and_oversized(): + with pytest.raises(Exception): + RawEnvelope(source_id="s", transport="http", received_at="t", raw="") + with pytest.raises(Exception): + RawEnvelope(source_id="s", transport="http", received_at="t", raw="x" * 70000) diff --git a/libs/ulpf-core/ulpf_core/__init__.py b/libs/ulpf-core/ulpf_core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/libs/ulpf-core/ulpf_core/ids.py b/libs/ulpf-core/ulpf_core/ids.py new file mode 100644 index 0000000..6f4cc14 --- /dev/null +++ b/libs/ulpf-core/ulpf_core/ids.py @@ -0,0 +1,13 @@ +import uuid + +# Fixed forever — changing it invalidates every stored raw_id/event_id. +NAMESPACE = uuid.UUID("7308095f-9773-5212-acef-a4c9f422b66c") + +def raw_id_for(topic: str, partition: int, offset: int) -> uuid.UUID: + """Stable across replays: the same message always maps to the same id.""" + return uuid.uuid5(NAMESPACE, f"{topic}:{partition}:{offset}") + +def event_id_for(raw_id: uuid.UUID, rule_version: int) -> uuid.UUID: + """Re-parse under a new version mints a new id; replay of the same + version collides and is skipped (idempotent append, spec §4).""" + return uuid.uuid5(NAMESPACE, f"{raw_id}:{rule_version}") diff --git a/libs/ulpf-core/ulpf_core/models.py b/libs/ulpf-core/ulpf_core/models.py new file mode 100644 index 0000000..6b0e135 --- /dev/null +++ b/libs/ulpf-core/ulpf_core/models.py @@ -0,0 +1,19 @@ +from pydantic import BaseModel, Field + +class RawEnvelope(BaseModel): + source_id: str + transport: str # "syslog-udp" | "syslog-tcp" | "http" + received_at: str # ISO8601 UTC, set by collector + raw: str = Field(min_length=1, max_length=65536) # line cap, spec §10 + format_hint: str | None = None + +class Mapping(BaseModel): + source_field: str + ocsf_path: str + +class Rule(BaseModel): + fingerprint_id: str + version: int + pattern: str + mappings: list[Mapping] + provenance: str # "slm" | "slm-edited" | "human" From 92169eef0d20a2971c2ca9f4320cc0391636072d Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 12:25:33 +0530 Subject: [PATCH 03/18] feat: per-line fingerprinting with shape-hash unknowns + fragmentation bench MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Known-format ids (cef_, leef_, json, syslog, csv, xml) port the legacy-demo heuristics and add csv/xml branches; unknown lines get a provisional auto_<8hex> id from their shape hash so no shared unknown bucket exists (spec §9). Shape hashing collapses numerics, empty key=value values and free-text runs so same-format events hash identically; the bench measures this with 500 seeded mutants across 5 golden corpora (100% stable) and fails on known-format fragmentation. Co-Authored-By: Claude Code --- bench/fingerprint_fragmentation.py | 270 ++++++++++++++++++ docs/m1-fragmentation.md | 43 +++ .../tests/data/golden/raw_logs_acmegw.txt | 10 + .../tests/data/golden/raw_logs_cef.txt | 16 ++ .../tests/data/golden/raw_logs_json.txt | 15 + .../tests/data/golden/raw_logs_syslog.txt | 15 + .../tests/data/golden/raw_logs_zenwall.txt | 10 + libs/ulpf-core/tests/test_fingerprint.py | 40 +++ libs/ulpf-core/ulpf_core/fingerprint.py | 185 ++++++++++++ 9 files changed, 604 insertions(+) create mode 100644 bench/fingerprint_fragmentation.py create mode 100644 docs/m1-fragmentation.md create mode 100644 libs/ulpf-core/tests/data/golden/raw_logs_acmegw.txt create mode 100644 libs/ulpf-core/tests/data/golden/raw_logs_cef.txt create mode 100644 libs/ulpf-core/tests/data/golden/raw_logs_json.txt create mode 100644 libs/ulpf-core/tests/data/golden/raw_logs_syslog.txt create mode 100644 libs/ulpf-core/tests/data/golden/raw_logs_zenwall.txt create mode 100644 libs/ulpf-core/tests/test_fingerprint.py create mode 100644 libs/ulpf-core/ulpf_core/fingerprint.py diff --git a/bench/fingerprint_fragmentation.py b/bench/fingerprint_fragmentation.py new file mode 100644 index 0000000..f0be2ad --- /dev/null +++ b/bench/fingerprint_fragmentation.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Fragmentation bench for ulpf-core fingerprinting (spec §9, M1). + +For every golden corpus (libs/ulpf-core/tests/data/golden/raw_logs_*.txt): + +1. Fingerprint every line and group by ``fingerprint_id``. A known-format + group whose lines do not all share one shape hash is a fragmentation. +2. Generate 100 mutated lines per corpus. Mutations randomize timestamps, + IPs, ports (``IP:port``, ``key=value`` and JSON numeric fields), hostnames + (``h####``) and free-text message word counts — exactly the values that + vary between real events of the same format. +3. A mutant is stable iff it keeps its base line's fingerprint (known formats) + and its base line's shape hash. Report per-corpus distinct shape hashes + and the mutation-stability ratio. + +Exit status: 0 if every known-format group is fragmentation-free and fully +mutation-stable, 1 otherwise. (``auto_`` groups cannot fragment by +construction: the provisional id is derived from the shape hash itself.) + +Run from the repo root: python bench/fingerprint_fragmentation.py +Writes: docs/m1-fragmentation.md +""" + +from __future__ import annotations + +import random +import re +import sys +from collections.abc import Callable +from pathlib import Path + +from ulpf_core.fingerprint import _classify, fingerprint_id, shape_hash + +ROOT = Path(__file__).resolve().parents[1] +GOLDEN = ROOT / "libs" / "ulpf-core" / "tests" / "data" / "golden" +DOC = ROOT / "docs" / "m1-fragmentation.md" + +SEED = 20260919 +MUTATIONS = 100 + +# --- mutation machinery ------------------------------------------------------ +# Timestamps mutate first and park behind \x00 sentinels so later numeric +# passes cannot chew freshly generated values into shape-breaking forms. +# IP:port mutates before the IP pass so the ":port" tail does not lose the +# digit its lookbehind needs. + +_ISO_TS_RE = re.compile( + r"(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2}):(\d{2}))?([+-]\d{2}:?\d{2}|Z)?" +) +_SYSLOG_TS_RE = re.compile( + r"\b((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec))\s+(\d{1,2})\s+(\d{2}):(\d{2}):(\d{2})\b" +) +_PORT_RE = re.compile(r"(?<=\d):\d{1,5}\b") # port tail of IP:port +_IP_RE = re.compile(r"\d{1,3}(?:\.\d{1,3}){3}") +_KV_NUM_RE = re.compile(r"(?<==)\d{1,5}\b") # key=12345 values +_JSON_NUM_RE = re.compile(r'(?<=":)\d{1,5}\b') # "key":12345 values +_HOSTNAME_RE = re.compile(r"\b[a-z][a-z0-9-]*\d{1,4}\b") # fw01, vpn-gw01, eth0 +_SENTINEL_RE = re.compile(r"\x00A(\d+)\x00") +_MONTH_RE = re.compile(r"^(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)$") +_MARKUP_CHARS = set('{}"<>') + + +def _rand_iso_ts(match: re.Match[str], rng: random.Random) -> str: + text = f"{rng.randint(2020, 2030):04d}-{rng.randint(1, 12):02d}-{rng.randint(1, 28):02d}" + if match.group(4): + text += f"T{rng.randint(0, 23):02d}:{rng.randint(0, 59):02d}:{rng.randint(0, 59):02d}" + return text + (match.group(7) or "") + + +def _rand_syslog_ts(match: re.Match[str], rng: random.Random) -> str: + # Keep the month word so the mutant still detects as syslog. + return ( + f"{match.group(1)} {rng.randint(1, 28)} " + f"{rng.randint(0, 23):02d}:{rng.randint(0, 59):02d}:{rng.randint(0, 59):02d}" + ) + + +def _rand_ip(rng: random.Random) -> str: + return ".".join(str(rng.randint(1, 254)) for _ in range(4)) + + +def mutate_line(line: str, rng: random.Random) -> str: + """Return a same-format mutant of *line* with randomized values.""" + stash: list[str] = [] + + def _swap(make: Callable[[re.Match[str]], str]) -> Callable[[re.Match[str]], str]: + def _sub(match: re.Match[str]) -> str: + stash.append(make(match)) + return f"\x00A{len(stash) - 1}\x00" + + return _sub + + line = _ISO_TS_RE.sub(_swap(lambda m: _rand_iso_ts(m, rng)), line) + line = _SYSLOG_TS_RE.sub(_swap(lambda m: _rand_syslog_ts(m, rng)), line) + line = _PORT_RE.sub(_swap(lambda m: f":{rng.randint(1, 65535)}"), line) + line = _IP_RE.sub(_swap(lambda m: _rand_ip(rng)), line) + line = _KV_NUM_RE.sub(_swap(lambda m: str(rng.randint(1, 65535))), line) + line = _JSON_NUM_RE.sub(_swap(lambda m: str(rng.randint(1, 65535))), line) + line = _HOSTNAME_RE.sub(_swap(lambda m: f"h{rng.randint(0, 9999):04d}"), line) + + # Free-text runs (message bodies) get a random word count. Structural + # tokens (markup, sentinels, syslog month, anything classified) are kept. + if " " in line: + kept: list[str] = [] + run: list[str] = [] + + def _flush() -> None: + if run: + kept.extend(rng.choice(WORDS) for _ in range(rng.randint(1, 6))) + run.clear() + + for token in line.split(): + if ( + _SENTINEL_RE.fullmatch(token) + or _MONTH_RE.fullmatch(token) + or _MARKUP_CHARS.intersection(token) + or _classify(token) != "W" + ): + _flush() + kept.append(token) + else: + run.append(token) + _flush() + line = " ".join(kept) + + return _SENTINEL_RE.sub(lambda m: stash[int(m.group(1))], line) + + +WORDS = [ + "alpha", "bravo", "delta", "echo", "fox", "golf", + "hotel", "india", "juliet", "kilo", "lima", "mike", +] + + +def run_bench(mutations: int = MUTATIONS, seed: int = SEED) -> dict: + rng = random.Random(seed) + corpora = sorted(GOLDEN.glob("raw_logs_*.txt")) + if not corpora: + raise SystemExit(f"no golden corpora found under {GOLDEN}") + + report: dict = {"seed": seed, "mutations": mutations, "corpora": [], "passed": True} + for path in corpora: + lines = [l for l in path.read_text(encoding="utf-8").splitlines() if l.strip()] + groups: dict[str, list[str]] = {} + for line in lines: + groups.setdefault(fingerprint_id(line), []).append(line) + + group_rows = [] + for fp in sorted(groups): + hashes = {shape_hash(l) for l in groups[fp]} + known = not fp.startswith("auto_") + fragmented = known and len(hashes) > 1 + group_rows.append({ + "fingerprint": fp, + "lines": len(groups[fp]), + "distinct_hashes": len(hashes), + "fragmented": fragmented, + }) + + canonical = {fp: shape_hash(ls[0]) for fp, ls in groups.items()} + mutant_hashes: set[str] = set() + stable = 0 + failures: list[str] = [] + for _ in range(mutations): + base = rng.choice(lines) + fp = fingerprint_id(base) + mutant = mutate_line(base, rng) + mutant_hashes.add(shape_hash(mutant)) + ok = shape_hash(mutant) == canonical[fp] + if not fp.startswith("auto_"): + ok = ok and fingerprint_id(mutant) == fp + if ok: + stable += 1 + elif not fp.startswith("auto_"): + failures.append( + f"[{path.name}] {fp} mutant unstable:\n" + f" base: {base}\n mutant: {mutant}" + ) + + known_failure = any(r["fragmented"] for r in group_rows) or bool(failures) + report["passed"] &= not known_failure + report["corpora"].append({ + "name": path.name, + "lines": len(lines), + "groups": group_rows, + "group_count": len(groups), + "distinct_line_hashes": len({shape_hash(l) for l in lines}), + "distinct_mutant_hashes": len(mutant_hashes), + "stability": stable / mutations, + "failed": known_failure, + "failures": failures, + }) + return report + + +def write_doc(report: dict) -> None: + out: list[str] = [ + "# M1 fragmentation bench (spec §9)", + "", + f"Generated by `python bench/fingerprint_fragmentation.py` " + f"(seed {report['seed']}, {report['mutations']} mutants per corpus). " + "Do not edit by hand; re-run to regenerate.", + "", + "## Method", + "", + "- Golden corpora live in `libs/ulpf-core/tests/data/golden/raw_logs_*.txt`.", + "- Lines are grouped by `fingerprint_id`; every line in a group must share", + " one shape hash (the CEF corpus intentionally holds 3 vendor groups).", + "- Each corpus also gets 100 mutants with randomized timestamps, IPs, ports,", + " hostnames (`h####`) and message word counts. A mutant is stable iff it", + " keeps its base line's fingerprint (known formats) and shape hash.", + "- Exit is non-zero if any known-format group fragments or loses a mutant", + " (`auto_` groups cannot fragment by construction: id = shape hash).", + "", + "## Results", + "", + "| corpus | lines | fingerprint groups | distinct line shape hashes | distinct mutant shape hashes | mutation stability | result |", + "|---|---:|---:|---:|---:|---:|---|", + ] + for corpus in report["corpora"]: + out.append( + f"| {corpus['name']} | {corpus['lines']} | {corpus['group_count']} " + f"| {corpus['distinct_line_hashes']} | {corpus['distinct_mutant_hashes']} " + f"| {corpus['stability']:.0%} | {'FAIL' if corpus['failed'] else 'PASS'} |" + ) + out += ["", "## Per fingerprint group", "", + "| corpus | fingerprint_id | lines | distinct shape hashes | result |", + "|---|---|---:|---:|---|"] + for corpus in report["corpora"]: + for row in corpus["groups"]: + out.append( + f"| {corpus['name']} | `{row['fingerprint']}` | {row['lines']} " + f"| {row['distinct_hashes']} | {'FAIL' if row['fragmented'] else 'PASS'} |" + ) + out += [ + "", + "## Reading the numbers", + "", + "- `distinct line shape hashes == fingerprint groups` means the golden lines", + " do not fragment within any format.", + "- `mutation stability 100%` means no realistic value mutation changes a", + " line's shape — the property drift detection (later milestones) relies on.", + "", + ] + DOC.write_text("\n".join(out), encoding="utf-8") + + +def main() -> int: + report = run_bench() + write_doc(report) + + print(f"seed={report['seed']} mutants/corpus={report['mutations']}") + for corpus in report["corpora"]: + status = "FAIL" if corpus["failed"] else "PASS" + print( + f"{corpus['name']:<28} lines={corpus['lines']:<3} " + f"groups={corpus['group_count']} " + f"line_hashes={corpus['distinct_line_hashes']} " + f"mutant_hashes={corpus['distinct_mutant_hashes']} " + f"stability={corpus['stability']:.0%} {status}" + ) + for failure in corpus["failures"]: + print(failure) + print(f"wrote {DOC.relative_to(ROOT)}") + print("RESULT:", "PASS" if report["passed"] else "FAIL") + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/m1-fragmentation.md b/docs/m1-fragmentation.md new file mode 100644 index 0000000..8bc570a --- /dev/null +++ b/docs/m1-fragmentation.md @@ -0,0 +1,43 @@ +# M1 fragmentation bench (spec §9) + +Generated by `python bench/fingerprint_fragmentation.py` (seed 20260919, 100 mutants per corpus). Do not edit by hand; re-run to regenerate. + +## Method + +- Golden corpora live in `libs/ulpf-core/tests/data/golden/raw_logs_*.txt`. +- Lines are grouped by `fingerprint_id`; every line in a group must share + one shape hash (the CEF corpus intentionally holds 3 vendor groups). +- Each corpus also gets 100 mutants with randomized timestamps, IPs, ports, + hostnames (`h####`) and message word counts. A mutant is stable iff it + keeps its base line's fingerprint (known formats) and shape hash. +- Exit is non-zero if any known-format group fragments or loses a mutant + (`auto_` groups cannot fragment by construction: id = shape hash). + +## Results + +| corpus | lines | fingerprint groups | distinct line shape hashes | distinct mutant shape hashes | mutation stability | result | +|---|---:|---:|---:|---:|---:|---| +| raw_logs_acmegw.txt | 10 | 1 | 1 | 1 | 100% | PASS | +| raw_logs_cef.txt | 16 | 3 | 2 | 2 | 100% | PASS | +| raw_logs_json.txt | 15 | 1 | 1 | 1 | 100% | PASS | +| raw_logs_syslog.txt | 15 | 1 | 1 | 1 | 100% | PASS | +| raw_logs_zenwall.txt | 10 | 1 | 1 | 1 | 100% | PASS | + +## Per fingerprint group + +| corpus | fingerprint_id | lines | distinct shape hashes | result | +|---|---|---:|---:|---| +| raw_logs_acmegw.txt | `auto_9d3d5688` | 10 | 1 | PASS | +| raw_logs_cef.txt | `cef_ciscoasa` | 5 | 1 | PASS | +| raw_logs_cef.txt | `cef_fortigate` | 4 | 1 | PASS | +| raw_logs_cef.txt | `cef_paloalto` | 7 | 1 | PASS | +| raw_logs_json.txt | `json` | 15 | 1 | PASS | +| raw_logs_syslog.txt | `syslog` | 15 | 1 | PASS | +| raw_logs_zenwall.txt | `auto_8274266a` | 10 | 1 | PASS | + +## Reading the numbers + +- `distinct line shape hashes == fingerprint groups` means the golden lines + do not fragment within any format. +- `mutation stability 100%` means no realistic value mutation changes a + line's shape — the property drift detection (later milestones) relies on. diff --git a/libs/ulpf-core/tests/data/golden/raw_logs_acmegw.txt b/libs/ulpf-core/tests/data/golden/raw_logs_acmegw.txt new file mode 100644 index 0000000..650b74a --- /dev/null +++ b/libs/ulpf-core/tests/data/golden/raw_logs_acmegw.txt @@ -0,0 +1,10 @@ +ACMEGW|v3|2026-08-27T14:32:07+05:30|type:conn-deny|source=203.0.113.45:51422|target=192.168.1.10:22|proto=tcp|reason=blacklist-match|sev=high +ACMEGW|v3|2026-08-27T14:34:12+05:30|type:conn-allow|source=192.168.1.60:53211|target=8.8.8.8:53|proto=udp|reason=policy-ok|sev=info +ACMEGW|v3|2026-08-27T14:36:47+05:30|type:conn-deny|source=198.51.100.22:44521|target=192.168.1.14:80|proto=tcp|reason=known-bad-ip|sev=high +ACMEGW|v3|2026-08-27T14:38:30+05:30|type:conn-deny|source=203.0.113.90:51500|target=192.168.1.20:3389|proto=tcp|reason=brute-force|sev=critical +ACMEGW|v3|2026-08-27T14:40:05+05:30|type:conn-allow|source=192.168.1.70:52890|target=203.0.113.220:443|proto=tcp|reason=policy-ok|sev=info +ACMEGW|v3|2026-08-27T14:41:41+05:30|type:conn-deny|source=192.0.2.201:51900|target=192.168.1.15:3389|proto=tcp|reason=brute-force|sev=critical +ACMEGW|v3|2026-08-27T14:43:02+05:30|type:conn-deny|source=203.0.113.33:49213|target=192.168.1.50:80|proto=tcp|reason=malware-signature|sev=high +ACMEGW|v3|2026-08-27T14:45:15+05:30|type:conn-deny|source=198.51.100.90:44012|target=192.168.1.12:445|proto=tcp|reason=exploit-attempt|sev=critical +ACMEGW|v3|2026-08-27T14:46:29+05:30|type:conn-allow|source=192.168.1.18:60011|target=8.8.4.4:53|proto=udp|reason=policy-ok|sev=info +ACMEGW|v3|2026-08-27T14:47:03+05:30|type:conn-deny|source=203.0.113.150:59012|target=192.168.1.99:8080|proto=tcp|reason=proxy-abuse|sev=medium diff --git a/libs/ulpf-core/tests/data/golden/raw_logs_cef.txt b/libs/ulpf-core/tests/data/golden/raw_logs_cef.txt new file mode 100644 index 0000000..094a783 --- /dev/null +++ b/libs/ulpf-core/tests/data/golden/raw_logs_cef.txt @@ -0,0 +1,16 @@ +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|spyware|5|src=203.0.113.45 dst=192.168.1.10 spt=51422 dpt=443 act=block cat=spyware msg=Suspicious DNS Query +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|malware|7|src=198.51.100.22 dst=192.168.1.14 spt=44521 dpt=80 act=alert cat=malware msg=Known bad hash +CEF:0|CiscoASA|ASA|9.1|106023|Deny|4|src=203.0.113.90 dst=192.168.1.20 spt=51500 dpt=22 act=deny +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|spyware|3|src=192.0.2.15 dst=192.168.1.22 spt=52011 dpt=443 act=allow cat=spyware msg=Low confidence match +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|exploit|9|src=203.0.113.77 dst=192.168.1.10 spt=33210 dpt=3389 act=block cat=exploit msg=RDP brute force attempt +CEF:0|CiscoASA|ASA|9.1|106023|Deny|4|src=198.51.100.5 dst=192.168.1.30 spt=41022 dpt=25 act=deny +CEF:0|CiscoASA|ASA|9.1|106100|Permit|2|src=203.0.113.12 dst=192.168.1.40 spt=60011 dpt=443 act=allow +CEF:0|CiscoASA|ASA|9.1|106023|Deny|6|src=192.0.2.201 dst=192.168.1.15 spt=51900 dpt=3389 act=deny +CEF:0|FortiGate|FortiOS|7.2|virus|eicar|8|src=203.0.113.33 dst=192.168.1.50 spt=49213 dpt=80 act=block cat=virus msg=EICAR test file detected +CEF:0|FortiGate|FortiOS|7.2|traffic|forward|1|src=192.168.1.60 dst=8.8.8.8 spt=53211 dpt=53 act=allow cat=dns msg=Normal DNS lookup +CEF:0|FortiGate|FortiOS|7.2|ips|signature|7|src=198.51.100.90 dst=192.168.1.12 spt=44012 dpt=445 act=block cat=exploit msg=SMB exploit attempt CVE-2020-0796 +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|c2|10|src=192.168.1.18 dst=203.0.113.201 spt=51555 dpt=8443 act=block cat=command-and-control msg=Beaconing to known C2 domain +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|scan|5|src=203.0.113.44 dst=192.168.1.0 spt=40000 dpt=21 act=alert cat=scan msg=Port scan detected +CEF:0|CiscoASA|ASA|9.1|106023|Deny|3|src=203.0.113.150 dst=192.168.1.99 spt=59012 dpt=8080 act=deny +CEF:0|FortiGate|FortiOS|7.2|webfilter|blocked|4|src=192.168.1.70 dst=203.0.113.220 spt=52890 dpt=443 act=block cat=web-filter msg=Blocked category gambling +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|vulnerability|6|src=198.51.100.61 dst=192.168.1.5 spt=48822 dpt=443 act=alert cat=vulnerability msg=SSL certificate anomaly detected diff --git a/libs/ulpf-core/tests/data/golden/raw_logs_json.txt b/libs/ulpf-core/tests/data/golden/raw_logs_json.txt new file mode 100644 index 0000000..26bfaca --- /dev/null +++ b/libs/ulpf-core/tests/data/golden/raw_logs_json.txt @@ -0,0 +1,15 @@ +{"timestamp":"2026-08-27T14:32:07+0530","event_type":"alert","src_ip":"203.0.113.45","src_port":51422,"dest_ip":"192.168.1.10","dest_port":22,"proto":"TCP","alert":{"signature":"ET SCAN Potential SSH Scan","severity":2}} +{"timestamp":"2026-08-27T14:34:00+0530","event_type":"alert","src_ip":"198.51.100.22","src_port":44521,"dest_ip":"192.168.1.14","dest_port":443,"proto":"TCP","alert":{"signature":"ET TROJAN Suspicious C2 Beacon","severity":1}} +{"timestamp":"2026-08-27T14:35:12+0530","event_type":"alert","src_ip":"203.0.113.90","src_port":51500,"dest_ip":"192.168.1.20","dest_port":3389,"proto":"TCP","alert":{"signature":"ET SCAN RDP Brute Force","severity":2}} +{"timestamp":"2026-08-27T14:36:47+0530","event_type":"flow","src_ip":"192.168.1.60","src_port":53211,"dest_ip":"8.8.8.8","dest_port":53,"proto":"UDP","alert":{"signature":"Normal DNS Lookup","severity":3}} +{"timestamp":"2026-08-27T14:37:59+0530","event_type":"alert","src_ip":"192.0.2.15","src_port":52011,"dest_ip":"192.168.1.22","dest_port":443,"proto":"TCP","alert":{"signature":"ET POLICY Low Confidence Spyware Match","severity":3}} +{"timestamp":"2026-08-27T14:38:34+0530","event_type":"alert","src_ip":"203.0.113.77","src_port":33210,"dest_ip":"192.168.1.10","dest_port":3389,"proto":"TCP","alert":{"signature":"ET EXPLOIT RDP Brute Force Attempt","severity":1}} +{"timestamp":"2026-08-27T14:39:20+0530","event_type":"alert","src_ip":"192.168.1.18","src_port":51555,"dest_ip":"203.0.113.201","dest_port":8443,"proto":"TCP","alert":{"signature":"ET TROJAN Known C2 Beacon Domain","severity":1}} +{"timestamp":"2026-08-27T14:40:05+0530","event_type":"alert","src_ip":"198.51.100.5","src_port":41022,"dest_ip":"192.168.1.30","dest_port":25,"proto":"TCP","alert":{"signature":"ET SCAN SMTP Probe","severity":2}} +{"timestamp":"2026-08-27T14:41:41+0530","event_type":"alert","src_ip":"203.0.113.150","src_port":59012,"dest_ip":"192.168.1.99","dest_port":8080,"proto":"TCP","alert":{"signature":"ET WEB_SERVER Suspicious Proxy Access","severity":2}} +{"timestamp":"2026-08-27T14:42:18+0530","event_type":"alert","src_ip":"192.0.2.201","src_port":51900,"dest_ip":"192.168.1.15","dest_port":3389,"proto":"TCP","alert":{"signature":"ET SCAN RDP Brute Force","severity":2}} +{"timestamp":"2026-08-27T14:43:02+0530","event_type":"alert","src_ip":"203.0.113.33","src_port":49213,"dest_ip":"192.168.1.50","dest_port":80,"proto":"TCP","alert":{"signature":"ET MALWARE EICAR Test File Detected","severity":1}} +{"timestamp":"2026-08-27T14:44:37+0530","event_type":"flow","src_ip":"192.168.1.70","src_port":52890,"dest_ip":"203.0.113.220","dest_port":443,"proto":"TCP","alert":{"signature":"Normal HTTPS Session","severity":3}} +{"timestamp":"2026-08-27T14:45:15+0530","event_type":"alert","src_ip":"198.51.100.90","src_port":44012,"dest_ip":"192.168.1.12","dest_port":445,"proto":"TCP","alert":{"signature":"ET EXPLOIT SMB CVE-2020-0796","severity":1}} +{"timestamp":"2026-08-27T14:45:53+0530","event_type":"alert","src_ip":"203.0.113.44","src_port":40000,"dest_ip":"192.168.1.1","dest_port":21,"proto":"TCP","alert":{"signature":"ET SCAN FTP Port Scan","severity":2}} +{"timestamp":"2026-08-27T14:46:29+0530","event_type":"alert","src_ip":"198.51.100.61","src_port":48822,"dest_ip":"192.168.1.5","dest_port":443,"proto":"TCP","alert":{"signature":"ET POLICY SSL Certificate Anomaly","severity":2}} diff --git a/libs/ulpf-core/tests/data/golden/raw_logs_syslog.txt b/libs/ulpf-core/tests/data/golden/raw_logs_syslog.txt new file mode 100644 index 0000000..0ee65fb --- /dev/null +++ b/libs/ulpf-core/tests/data/golden/raw_logs_syslog.txt @@ -0,0 +1,15 @@ +Aug 27 14:32:07 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=203.0.113.45 DST=192.168.1.10 LEN=60 PROTO=TCP SPT=51422 DPT=22 FLAGS=SYN +Aug 27 14:33:12 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=198.51.100.22 DST=192.168.1.14 LEN=52 PROTO=UDP SPT=33211 DPT=53 FLAGS= +Aug 27 14:34:01 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=203.0.113.90 DST=192.168.1.20 LEN=60 PROTO=TCP SPT=51500 DPT=3389 FLAGS=SYN +Aug 27 14:35:44 fw01 kernel: IPTABLES-ACCEPT: IN=eth0 OUT=eth1 SRC=192.168.1.60 DST=8.8.8.8 LEN=68 PROTO=UDP SPT=53211 DPT=53 FLAGS= +Aug 27 14:36:19 vpn-gw01 kernel: IPTABLES-DROP: IN=eth1 OUT= SRC=192.0.2.15 DST=192.168.1.22 LEN=60 PROTO=TCP SPT=52011 DPT=443 FLAGS=SYN +Aug 27 14:37:52 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=203.0.113.77 DST=192.168.1.10 LEN=60 PROTO=TCP SPT=33210 DPT=3389 FLAGS=SYN +Aug 27 14:38:30 vpn-gw01 kernel: IPTABLES-ACCEPT: IN=eth1 OUT=eth0 SRC=192.168.1.18 DST=203.0.113.201 LEN=1420 PROTO=TCP SPT=51555 DPT=8443 FLAGS=ACK +Aug 27 14:39:07 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=198.51.100.5 DST=192.168.1.30 LEN=60 PROTO=TCP SPT=41022 DPT=25 FLAGS=SYN +Aug 27 14:40:21 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=203.0.113.150 DST=192.168.1.99 LEN=60 PROTO=TCP SPT=59012 DPT=8080 FLAGS=SYN +Aug 27 14:41:55 vpn-gw01 kernel: IPTABLES-DROP: IN=eth1 OUT= SRC=192.0.2.201 DST=192.168.1.15 LEN=60 PROTO=TCP SPT=51900 DPT=3389 FLAGS=SYN +Aug 27 14:42:38 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=203.0.113.33 DST=192.168.1.50 LEN=60 PROTO=TCP SPT=49213 DPT=80 FLAGS=SYN +Aug 27 14:43:12 fw01 kernel: IPTABLES-ACCEPT: IN=eth0 OUT=eth1 SRC=192.168.1.70 DST=203.0.113.220 LEN=580 PROTO=TCP SPT=52890 DPT=443 FLAGS=ACK +Aug 27 14:44:49 vpn-gw01 kernel: IPTABLES-DROP: IN=eth1 OUT= SRC=198.51.100.90 DST=192.168.1.12 LEN=60 PROTO=TCP SPT=44012 DPT=445 FLAGS=SYN +Aug 27 14:45:26 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=203.0.113.44 DST=192.168.1.1 LEN=40 PROTO=TCP SPT=40000 DPT=21 FLAGS=SYN +Aug 27 14:46:03 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=198.51.100.61 DST=192.168.1.5 LEN=60 PROTO=TCP SPT=48822 DPT=443 FLAGS=SYN diff --git a/libs/ulpf-core/tests/data/golden/raw_logs_zenwall.txt b/libs/ulpf-core/tests/data/golden/raw_logs_zenwall.txt new file mode 100644 index 0000000..4259877 --- /dev/null +++ b/libs/ulpf-core/tests/data/golden/raw_logs_zenwall.txt @@ -0,0 +1,10 @@ +ZW1;2026-08-27T14:32:07Z;deny;src=198.51.100.7:4432;dst=10.0.0.9:22;rule=R7 +ZW1;2026-08-27T14:33:41Z;allow;src=192.168.1.60:53211;dst=8.8.8.8:53;rule=R2 +ZW1;2026-08-27T14:35:12Z;deny;src=203.0.113.90:51500;dst=10.0.0.9:3389;rule=R7 +ZW1;2026-08-27T14:36:47Z;alert;src=198.51.100.22:44521;dst=10.0.0.14:80;rule=R11 +ZW1;2026-08-27T14:38:30Z;deny;src=203.0.113.33:49213;dst=10.0.0.50:80;rule=R4 +ZW1;2026-08-27T14:40:05Z;allow;src=192.168.1.70:52890;dst=8.8.4.4:53;rule=R2 +ZW1;2026-08-27T14:41:41Z;deny;src=192.0.2.201:51900;dst=10.0.0.15:445;rule=R9 +ZW1;2026-08-27T14:43:02Z;alert;src=198.51.100.90:44012;dst=10.0.0.12:443;rule=R11 +ZW1;2026-08-27T14:45:15Z;deny;src=203.0.113.150:59012;dst=10.0.0.99:8080;rule=R7 +ZW1;2026-08-27T14:46:29Z;allow;src=192.168.1.18:60011;dst=8.8.8.8:53;rule=R2 diff --git a/libs/ulpf-core/tests/test_fingerprint.py b/libs/ulpf-core/tests/test_fingerprint.py new file mode 100644 index 0000000..a553b1b --- /dev/null +++ b/libs/ulpf-core/tests/test_fingerprint.py @@ -0,0 +1,40 @@ +# libs/ulpf-core/tests/test_fingerprint.py +import re +from pathlib import Path +from ulpf_core.fingerprint import fingerprint_id, shape_hash + +GOLDEN = Path(__file__).parent / "data" / "golden" +LINES = {p.stem: [l for l in p.read_text().splitlines() if l.strip()] for p in GOLDEN.glob("raw_logs_*.txt")} + +def test_known_formats(): + assert fingerprint_id(LINES["raw_logs_cef"][0]) == "cef_paloalto" + assert fingerprint_id(LINES["raw_logs_syslog"][0]) == "syslog" + assert fingerprint_id(LINES["raw_logs_json"][0]) == "json" + +def test_unknown_gets_stable_auto_id(): + fp = fingerprint_id(LINES["raw_logs_acmegw"][0]) + assert re.fullmatch(r"auto_[0-9a-f]{8}", fp) + +def test_two_unknown_vendors_differ(): # spec §9: no shared "unknown" bucket + assert fingerprint_id(LINES["raw_logs_acmegw"][0]) != fingerprint_id(LINES["raw_logs_zenwall"][0]) + +def test_mutation_stability_within_format_group(): + # Invariant (spec §9): all lines of ONE format share one shape hash. + # Group by fingerprint first — raw_logs_cef.txt mixes 3 CEF vendors, + # and different vendors legitimately hash differently. + groups: dict[str, set[str]] = {} + for lines in LINES.values(): + for l in lines: + groups.setdefault(fingerprint_id(l), set()).add(shape_hash(l)) + for fp, hashes in groups.items(): + assert len(hashes) == 1, f"{fp} fragmented into {len(hashes)} shapes: {hashes}" + +def test_mutated_hostnames_and_numbers_stable(): + base = LINES["raw_logs_syslog"][0] + mutated = re.sub(r"SRC=[\d.]+", "SRC=1.2.3.4", base) + mutated = re.sub(r"DPT=\d+", "DPT=9999", mutated) + assert shape_hash(mutated) == shape_hash(base) + +def test_csv_and_xml_detected(): + assert fingerprint_id("a,b,c,1,2,3") == "csv" + assert fingerprint_id("1.2.3.4") == "xml" diff --git a/libs/ulpf-core/ulpf_core/fingerprint.py b/libs/ulpf-core/ulpf_core/fingerprint.py new file mode 100644 index 0000000..09d5a88 --- /dev/null +++ b/libs/ulpf-core/ulpf_core/fingerprint.py @@ -0,0 +1,185 @@ +"""Per-line format fingerprinting and shape hashing (spec §9, M1 version). + +Two layers, both deterministic and cheap (no model calls): + +- ``fingerprint_id(line)``: known-format ids (``cef_``, ``leef_``, + ``json``, ``syslog``, ``csv``, ``xml``) ported from the battle-tested + legacy-demo/parser_engine/fingerprint.py heuristics, plus new csv/xml branches. + Anything else gets a provisional ``auto_<8hex>`` id derived from its shape + hash, so unknown vendors never share an "unknown" bucket (spec §9). + +- ``shape_sequence(line)`` / ``shape_hash(line)``: a coarse structural class + per token. Two events of the same format must hash identically even when + IPs, ports, timestamps, hostnames and message wording differ — that is the + fragmentation guarantee the bench (bench/fingerprint_fragmentation.py) + measures. Tunings that make this hold: + * numeric collapse — every pure number is the same class (``N``); + * empty key=value values (``OUT=``, ``FLAGS=``) classify like word values, + so present-but-empty fields match present-with-value fields; + * runs of free-text tokens (``W``) collapse to a single ``W`` so message + word count does not change the shape. + +M1 scope: per-line assignment only; sessionization comes later. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import xml.etree.ElementTree as ET + +# --- Known-format detection (ported from legacy-demo/parser_engine/fingerprint.py) --- + +# Standard 3-letter syslog month abbreviations +SYSLOG_MONTH_RE = re.compile( + r"^?\s*(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}", + re.IGNORECASE, +) +SYSLOG_PRI_RE = re.compile(r"^<\d{1,3}>") + +_XML_TAG_RE = re.compile(r"<[^>]*>") + + +def _is_csv(text: str) -> bool: + """CSV heuristic: at least 2 commas and no whitespace outside quotes.""" + if text.count(",") < 2: + return False + in_quotes = False + for ch in text: + if ch == '"': + in_quotes = not in_quotes + elif not in_quotes and ch.isspace(): + return False + return True + + +def _first_element_end(text: str) -> int | None: + """Index just past the first complete top-level XML element, or None.""" + depth = 0 + for match in _XML_TAG_RE.finditer(text): + body = match.group(0)[1:-1] + if body.startswith(("!", "?")): # , + continue + if body.startswith("/"): + depth -= 1 + if depth == 0: + return match.end() + elif body.endswith("/"): # self-closing + if depth == 0: + return match.end() + else: + depth += 1 + return None + + +def _is_xml(text: str) -> bool: + """XML heuristic: parses as XML, or its truncated first element does.""" + try: + ET.fromstring(text) + return True + except ET.ParseError: + pass + end = _first_element_end(text) + if end is None: + return False + try: + ET.fromstring(text[:end]) + return True + except ET.ParseError: + return False + + +def _known_format(line: str) -> str | None: + """Return the known-format id for this line, or None if unknown.""" + text = line.strip() + if not text: + return None + + # 1. CEF Header + if text.startswith("CEF:"): + parts = text.split("|") + if len(parts) >= 3: + vendor = re.sub(r"[^a-zA-Z0-9]", "", parts[1].lower()) + return f"cef_{vendor}" if vendor else "cef" + return "cef" + + # 2. LEEF Header + if text.startswith("LEEF:"): + parts = text.split("|") + if len(parts) >= 3: + vendor = re.sub(r"[^a-zA-Z0-9]", "", parts[1].lower()) + return f"leef_{vendor}" if vendor else "leef" + return "leef" + + # 3. JSON Format + if text.startswith("{"): + try: + json.loads(text) + return "json" + except json.JSONDecodeError: + pass + + # 4. Syslog Format + if SYSLOG_MONTH_RE.match(text) or SYSLOG_PRI_RE.match(text): + return "syslog" + + # 5. XML before CSV: compact single-line XML (1,23,4) has no + # whitespace and can carry commas, so the csv heuristic would misread it. + if text.startswith("<") and _is_xml(text): + return "xml" + + # 6. CSV Format + if _is_csv(text): + return "csv" + + return None + + +# --- Shape classification --- + +_IP = r"\d{1,3}(?:\.\d{1,3}){3}" + +_SHAPE_RULES = [ + (re.compile(rf"{_IP}:\d+$"), "IPPORT"), + (re.compile(rf"{_IP}$"), "IP"), + (re.compile(r"^[0-9a-fA-F:]{6,}$"), "HEXISH"), # MAC / IPv6-ish + (re.compile(r"^\d{4}-\d{2}-\d{2}[T ].+"), "ISOTS"), + (re.compile(r"^[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}$"), "SYSLOGTS"), + (re.compile(r"^\d+$"), "N"), # numeric collapse +] + + +def _classify(token: str) -> str: + if "=" in token: + key, _, value = token.partition("=") + # An empty value ("OUT=", "FLAGS=") classifies like a word value so a + # field that is present-but-empty hashes the same as one with a value. + return f"KV:{_classify(value)}" + for rx, name in _SHAPE_RULES: + if rx.match(token): + return name + return "W" + + +def shape_sequence(line: str) -> list[str]: + """Structural class per token; free-text runs collapse to a single W.""" + text = line.strip() + tokens = text.split() if " " in text else re.split(r"[|;,]", text) + sequence: list[str] = [] + for cls in (_classify(token) for token in tokens): + # Message bodies vary in word count between events of the same format, + # so a run of free-text tokens hashes as one W (spec §9 stability). + if cls == "W" and sequence and sequence[-1] == "W": + continue + sequence.append(cls) + return sequence + + +def shape_hash(line: str) -> str: + return hashlib.sha256("|".join(shape_sequence(line)).encode()).hexdigest()[:8] + + +def fingerprint_id(line: str) -> str: + known = _known_format(line) # returns e.g. "cef_paloalto" or None + return known if known else f"auto_{shape_hash(line)}" From 0d4ffe7f73d752609c2d4a93a88541f27eb37104 Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 13:01:15 +0530 Subject: [PATCH 04/18] feat: OCSF subset schema + single parsing implementation with regex timeouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libs/ocsf-schema ships the curated Network Activity subset (draft 2020-12, root additionalProperties:false) and validate_document() for activation-time and canary use only (spec §6.4); time/severity_id/action/message are nullable so timestamp-less rules validate pre-pipeline, and jsonschema rides as the optional `activation` extra. ulpf_core.parsing is the one implementation everything imports: legacy apply_rule ported onto the regex module with a per-match ms timeout (spec §10), allow-listed mapping targets enforced at document build, safe severity/port casts (fixes the demo hot-path crash), full OCSF document shape, and EXTENSION_KV as the single shared grammar for M2 onboarding. validate_rule_output adds pattern length/group/nesting caps, compile, allow-list, sample-parse and document-validation checks for the human activation gate. Co-Authored-By: Claude Code --- libs/ocsf-schema/ocsf_schema/__init__.py | 5 + libs/ocsf-schema/ocsf_schema/schema.json | 43 +++ libs/ocsf-schema/ocsf_schema/validator.py | 30 ++ libs/ocsf-schema/pyproject.toml | 19 ++ libs/ocsf-schema/tests/test_validator.py | 87 ++++++ libs/ulpf-core/pyproject.toml | 2 +- libs/ulpf-core/tests/test_parsing.py | 82 ++++++ libs/ulpf-core/ulpf_core/parsing.py | 322 ++++++++++++++++++++++ 8 files changed, 589 insertions(+), 1 deletion(-) create mode 100644 libs/ocsf-schema/ocsf_schema/__init__.py create mode 100644 libs/ocsf-schema/ocsf_schema/schema.json create mode 100644 libs/ocsf-schema/ocsf_schema/validator.py create mode 100644 libs/ocsf-schema/pyproject.toml create mode 100644 libs/ocsf-schema/tests/test_validator.py create mode 100644 libs/ulpf-core/tests/test_parsing.py create mode 100644 libs/ulpf-core/ulpf_core/parsing.py diff --git a/libs/ocsf-schema/ocsf_schema/__init__.py b/libs/ocsf-schema/ocsf_schema/__init__.py new file mode 100644 index 0000000..3054bf7 --- /dev/null +++ b/libs/ocsf-schema/ocsf_schema/__init__.py @@ -0,0 +1,5 @@ +"""ULPF curated OCSF subset schema (spec §6.4).""" + +from ocsf_schema.validator import validate_document + +__all__ = ["validate_document"] diff --git a/libs/ocsf-schema/ocsf_schema/schema.json b/libs/ocsf-schema/ocsf_schema/schema.json new file mode 100644 index 0000000..691373f --- /dev/null +++ b/libs/ocsf-schema/ocsf_schema/schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ULPF Network Activity (curated OCSF subset)", + "description": "Curated subset of the OCSF Network Activity class (spec §6.4). Validated at rule activation and by the runtime canary only — never per event.", + "type": "object", + "additionalProperties": false, + "required": ["class_uid", "activity_id", "time", "metadata", "unmapped"], + "properties": { + "class_uid": {"const": 4001}, + "class_name": {"const": "Network Activity"}, + "activity_id": {"type": "integer", "minimum": 0, "maximum": 99}, + "severity_id": {"type": ["integer", "null"], "minimum": 0, "maximum": 99}, + "time": {"type": ["string", "null"], "description": "Event time; null when the rule mapped no timestamp — the pipeline fills ingestion time before emission."}, + "action": {"type": ["string", "null"]}, + "message": {"type": ["string", "null"]}, + "src_endpoint": { + "type": ["object", "null"], + "properties": { + "ip": {"type": "string"}, + "port": {"type": ["integer", "string"]} + } + }, + "dst_endpoint": { + "type": ["object", "null"], + "properties": { + "ip": {"type": "string"}, + "port": {"type": ["integer", "string"]} + } + }, + "metadata": { + "type": "object", + "properties": { + "product": {"const": "ULPF"} + }, + "required": ["product"] + }, + "unmapped": { + "type": "object", + "additionalProperties": true, + "description": "OCSF standard unmapped object — captured fields with no OCSF home are never dropped (spec §6.4)." + } + } +} diff --git a/libs/ocsf-schema/ocsf_schema/validator.py b/libs/ocsf-schema/ocsf_schema/validator.py new file mode 100644 index 0000000..19bc2d2 --- /dev/null +++ b/libs/ocsf-schema/ocsf_schema/validator.py @@ -0,0 +1,30 @@ +"""Validation of OCSF documents against the curated subset schema (spec §6.4). + +Used at rule activation + runtime canary only — NEVER per event (per-event +JSON Schema validation is far too slow for the hot path; hot-path structure is +guaranteed by construction in ulpf_core.parsing). +""" + +from __future__ import annotations + +import json +import pathlib + +_SCHEMA = json.loads((pathlib.Path(__file__).parent / "schema.json").read_text()) + + +def validate_document(doc: dict) -> list[str]: + """Return human-readable schema errors for one document; empty list = valid. + + jsonschema is optional at runtime: if it is not installed, validation + degrades to no-op ([]). Activation tests pin jsonschema so the real + validator is what tests exercise. + """ + try: + from jsonschema import Draft202012Validator + except ImportError: + return [] # jsonschema optional at runtime; activation tests pin it + return [ + f"schema: {e.message} at {list(e.absolute_path)}" + for e in Draft202012Validator(_SCHEMA).iter_errors(doc) + ] diff --git a/libs/ocsf-schema/pyproject.toml b/libs/ocsf-schema/pyproject.toml new file mode 100644 index 0000000..c4f196f --- /dev/null +++ b/libs/ocsf-schema/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "ocsf-schema" +version = "0.1.0" +description = "ULPF curated OCSF subset schema (Network Activity) + activation-time validator" +requires-python = ">=3.12" + +[project.optional-dependencies] +# jsonschema is only needed at rule activation / canary (spec §6.4), never per event. +activation = ["jsonschema~=4.23"] + +[tool.setuptools] +packages = ["ocsf_schema"] + +[tool.setuptools.package-data] +ocsf_schema = ["schema.json"] diff --git a/libs/ocsf-schema/tests/test_validator.py b/libs/ocsf-schema/tests/test_validator.py new file mode 100644 index 0000000..e2970de --- /dev/null +++ b/libs/ocsf-schema/tests/test_validator.py @@ -0,0 +1,87 @@ +# libs/ocsf-schema/tests/test_validator.py +import sys + +from ocsf_schema.validator import validate_document + + +def _valid_doc() -> dict: + return { + "class_uid": 4001, + "class_name": "Network Activity", + "activity_id": 99, + "severity_id": None, + "time": None, + "src_endpoint": {"ip": "203.0.113.45", "port": 51422}, + "dst_endpoint": {"ip": "192.168.1.10", "port": 443}, + "action": "block", + "message": "Suspicious DNS Query", + "metadata": {"product": "ULPF"}, + "unmapped": {"cat": "spyware"}, + } + + +def test_valid_document_passes(): + assert validate_document(_valid_doc()) == [] + + +def test_null_time_and_string_time_both_legal(): + # time is nullable: parse() runs pre-pipeline, where ingestion time is not + # filled yet; a mapped ISO timestamp is the other legal shape. + doc = _valid_doc() + doc["time"] = "2026-08-27T14:32:07Z" + assert validate_document(doc) == [] + doc["time"] = None + assert validate_document(doc) == [] + + +def test_severity_int_and_null_both_legal(): + doc = _valid_doc() + doc["severity_id"] = 5 + assert validate_document(doc) == [] + doc["severity_id"] = None + assert validate_document(doc) == [] + + +def test_missing_required_field_rejected(): + doc = _valid_doc() + del doc["unmapped"] + assert validate_document(doc) + + +def test_unknown_class_uid_rejected(): + doc = _valid_doc() + doc["class_uid"] = 4002 + assert validate_document(doc) + + +def test_activity_id_out_of_range_rejected(): + doc = _valid_doc() + doc["activity_id"] = 100 + assert validate_document(doc) + + +def test_undeclared_property_rejected(): + doc = _valid_doc() + doc["evil_path"] = {"password": "hunter2"} # root additionalProperties: false + assert validate_document(doc) + + +def test_metadata_wrong_product_rejected(): + doc = _valid_doc() + doc["metadata"] = {"product": "NotULPF"} + assert validate_document(doc) + + +def test_port_must_be_integer_or_string(): + doc = _valid_doc() + doc["src_endpoint"]["port"] = [51422] # neither integer nor string + assert validate_document(doc) + doc["src_endpoint"]["port"] = "51422" # string is legal per schema + assert validate_document(doc) == [] + + +def test_jsonschema_missing_falls_back_to_no_errors(monkeypatch): + # Contract: jsonschema optional at runtime — ImportError degrades to []. + # (None in sys.modules makes `from jsonschema import ...` raise ImportError.) + monkeypatch.setitem(sys.modules, "jsonschema", None) + assert validate_document(_valid_doc()) == [] diff --git a/libs/ulpf-core/pyproject.toml b/libs/ulpf-core/pyproject.toml index c548e1a..afb2c97 100644 --- a/libs/ulpf-core/pyproject.toml +++ b/libs/ulpf-core/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "ulpf-core" version = "0.1.0" requires-python = ">=3.12" -dependencies = ["pydantic~=2.7"] +dependencies = ["pydantic~=2.7", "regex~=2024.7"] [tool.setuptools] packages = ["ulpf_core"] diff --git a/libs/ulpf-core/tests/test_parsing.py b/libs/ulpf-core/tests/test_parsing.py new file mode 100644 index 0000000..02fe523 --- /dev/null +++ b/libs/ulpf-core/tests/test_parsing.py @@ -0,0 +1,82 @@ +# libs/ulpf-core/tests/test_parsing.py — key cases, all with concrete data: +from ulpf_core.parsing import parse, validate_rule_output, OCSF_TARGETS +from ulpf_core.models import Rule, Mapping + +CEF_RULE = Rule(fingerprint_id="cef_paloalto", version=1, + pattern=r"^.*\|(?P.*)$", + mappings=[Mapping(source_field="src", ocsf_path="src_endpoint.ip"), + Mapping(source_field="dst", ocsf_path="dst_endpoint.ip"), + Mapping(source_field="spt", ocsf_path="src_endpoint.port"), + Mapping(source_field="dpt", ocsf_path="dst_endpoint.port"), + Mapping(source_field="act", ocsf_path="action"), + Mapping(source_field="msg", ocsf_path="message")], + provenance="human") +CEF_LINE = ("CEF:0|PaloAlto|PAN-OS|10.1|THREAT|spyware|5|src=203.0.113.45 " + "dst=192.168.1.10 spt=51422 dpt=443 act=block cat=spyware msg=Suspicious DNS Query") + +JSON_RULE = Rule(fingerprint_id="json", version=1, pattern="__JSON__", + mappings=[Mapping(source_field="src_ip", ocsf_path="src_endpoint.ip"), + Mapping(source_field="src_port", ocsf_path="src_endpoint.port"), + Mapping(source_field="dest_ip", ocsf_path="dst_endpoint.ip"), + Mapping(source_field="dest_port", ocsf_path="dst_endpoint.port"), + Mapping(source_field="timestamp", ocsf_path="time"), + Mapping(source_field="alert.severity", ocsf_path="severity_id"), + Mapping(source_field="event_type", ocsf_path="action"), + Mapping(source_field="alert.signature", ocsf_path="message")], + provenance="human") + +def test_parse_cef_core_fields(): + doc, err = parse(CEF_LINE, CEF_RULE) + assert err is None + assert doc["src_endpoint"] == {"ip": "203.0.113.45", "port": 51422} + assert doc["dst_endpoint"]["port"] == 443 + assert doc["action"] == "block" + assert doc["unmapped"]["cat"] == "spyware" # never dropped (spec §6.4) + assert doc["message"] == "Suspicious DNS Query" + +def test_parse_no_match_is_error_not_raise(): + doc, err = parse("COMPLETELY UNRELATED", CEF_RULE) + assert doc is None and "did not match" in err + +def test_timeout_treated_as_error(): # spec §10 + evil = Rule(fingerprint_id="x", version=1, + pattern=r"(a+)+$", mappings=[], provenance="human") + doc, err = parse("a" * 60 + "!", evil, timeout_ms=30) + assert doc is None and ("timeout" in err or "did not match" in err) + +def test_timeout_path_maps_timeouterror_to_error(): # spec §10 plumbing + # (a|aa)+b cannot match (no 'b') and backtracks exponentially, so the + # regex module's per-match timeout genuinely fires here (30ms ≪ full search). + evil = Rule(fingerprint_id="x", version=1, + pattern=r"(a|aa)+b", mappings=[], provenance="human") + doc, err = parse("a" * 40, evil, timeout_ms=30) + assert doc is None and "timeout" in err + +def test_json_sentinel_path(): # __JSON__ rule on golden JSON corpus line + import json as _json + from pathlib import Path + line = (Path(__file__).parent / "data/golden/raw_logs_json.txt").read_text().splitlines()[0] + src = _json.loads(line)["src_ip"] # expectation derived from the line itself + doc, err = parse(line, JSON_RULE) + assert err is None + assert doc["src_endpoint"]["ip"] == src + assert doc["metadata"] == {"product": "ULPF"} + assert doc["unmapped"] # unmapped keys preserved, spec §6.4 + +def test_severity_text_does_not_crash(): # demo hot-path crash bug stays fixed + import json as _json + line = _json.dumps({"timestamp": "2026-08-27T14:32:07Z", "event_type": "alert", + "src_ip": "203.0.113.45", "dest_ip": "10.0.0.9", + "alert": {"severity": "high", "signature": "text severity probe"}}) + doc, err = parse(line, JSON_RULE) + assert err is None and doc is not None and doc["severity_id"] is None + +def test_must_be_allowlisted(): + rule = Rule(fingerprint_id="x", version=1, pattern=r"(?P.*)", + mappings=[Mapping(source_field="a", ocsf_path="evil.path")], provenance="human") + errs = validate_rule_output(rule, ["sample"]); assert any("allow" in e for e in errs) + +def test_activation_validation_passes_for_golden_cef(): + from pathlib import Path + lines = (Path(__file__).parent / "data/golden/raw_logs_cef.txt").read_text().splitlines() + assert validate_rule_output(CEF_RULE, lines) == [] diff --git a/libs/ulpf-core/ulpf_core/parsing.py b/libs/ulpf-core/ulpf_core/parsing.py new file mode 100644 index 0000000..a258f0f --- /dev/null +++ b/libs/ulpf-core/ulpf_core/parsing.py @@ -0,0 +1,322 @@ +"""Single parsing implementation: raw line + Rule -> OCSF document (spec §6.1, §6.4). + +Ported from legacy-demo/parser_engine/apply_rule.py (the proven demo logic) +with the spec fixes M1 requires: + +- rule-supplied regex runs through the ``regex`` module under a per-match + timeout (spec §10) — timeout / compile error / no match are returned as + errors, never a hot-path hang or raise; +- mapping targets are allow-listed (``OCSF_TARGETS``) at document build — + defense-in-depth behind activation-time validation, so a rule can never + write outside the curated subset; +- severity/port integer casts are wrapped: a bad value becomes ``None`` + (severity) or keeps the string (port) instead of crashing the hot path + (the demo's crash bug, fixed); +- the full OCSF document is built (class_uid / class_name / activity_id / + metadata), not the demo's partial shape. + +``time``: parse() takes the line only, so the document carries the rule-mapped +event time when the rule maps one, else ``None`` — the pipeline fills +ingestion time (received_at) before emission. +""" + +from __future__ import annotations + +import json +from typing import Any + +import regex + +from ulpf_core.models import Rule + +# The only OCSF paths a rule mapping may target (activation allow-list, spec §6.4). +OCSF_TARGETS: frozenset[str] = frozenset({ + "src_endpoint.ip", + "src_endpoint.port", + "dst_endpoint.ip", + "dst_endpoint.port", + "time", + "severity_id", + "action", + "message", +}) + +# Extension-blob key=value grammar — the ONE definition; onboarding (M2) imports +# this instead of recompiling. Matches key="value" OR key=value_up_to_next_key_or_end. +EXTENSION_KV = regex.compile(r'(\w+)=(?:"([^"]*)"|([^=]+?)(?=\s+\w+=|$))') + +# Sentinel pattern: parse the line as JSON and walk dotted paths, no regex. +JSON_SENTINEL = "__JSON__" + +# CEF-style header groups describe the envelope, not payload fields — they +# never land in unmapped (legacy structural exclusion, carried over). +_STRUCTURAL_KEYS = frozenset( + {"extension", "vendor", "product", "version", "sig_id", "name", "severity"} +) + +# Pattern caps checked at activation (spec §10: length/group-count/nesting). +MAX_PATTERN_CHARS = 1000 +MAX_PATTERN_GROUPS = 32 +MAX_PATTERN_NESTING = 5 + +_CLASS_UID = 4001 +_CLASS_NAME = "Network Activity" +_ACTIVITY_ID = 99 # "other" — curated subset carries no per-activity taxonomy yet + + +def parse(line: str, rule: Rule, *, timeout_ms: int = 50) -> tuple[dict | None, str | None]: + """Apply one rule to one raw line; returns ``(ocsf_doc, None)`` or ``(None, error)``. + + Never raises on bad input — bad pattern, timeout, no match and invalid JSON + are all errors — raising is reserved for programmer error (e.g. a Rule that + violates its own model). The document's ``time`` is the rule-mapped event + time, or ``None`` when the rule maps no timestamp; the pipeline fills + ingestion time (received_at) before emission. + """ + if rule.pattern == JSON_SENTINEL: + return _parse_json(line, rule) + return _parse_regex(line, rule, timeout_ms) + + +def validate_rule_output(rule: Rule, samples: list[str]) -> list[str]: + """Activation-time check (spec §6.4): pattern caps, compiles, allow-listed + mappings, every sample parses, output documents validate. + + Returns the list of problems; empty means the rule may activate. Never + raises on rule or sample content — a broken rule is exactly what this + exists to catch. + """ + errors = _pattern_cap_errors(rule.pattern) + if errors: + return errors + + try: + regex.compile(rule.pattern) + except regex.error as exc: + return [f"compile error: {exc}"] + + for mapping in rule.mappings: + if mapping.ocsf_path not in OCSF_TARGETS: + errors.append( + f"mapping {mapping.source_field!r} -> {mapping.ocsf_path!r} " + "is not in the OCSF allow-list" + ) + + for i, line in enumerate(samples): + doc, err = parse(line, rule) + if err is not None: + errors.append(f"sample {i}: {err}") + continue + errors.extend(f"sample {i}: {e}" for e in _document_errors(doc)) + return errors + + +# --- regex path ------------------------------------------------------------- + +def _parse_regex(line: str, rule: Rule, timeout_ms: int) -> tuple[dict | None, str | None]: + try: + compiled = regex.compile(rule.pattern) + except regex.error as exc: + return None, f"compile error: {exc}" + try: + # `regex` takes the timeout per match, in seconds (spec §10: ms-scale). + match = compiled.search(line, timeout=timeout_ms / 1000.0) + except TimeoutError: + return None, "timeout" + if match is None: + return None, f"rule pattern {rule.pattern!r} did not match line" + + all_fields: dict[str, Any] = dict(match.groupdict()) + extension_text = all_fields.get("extension") + if extension_text: + all_fields.update(_parse_extension_kv(extension_text)) + + parts = _new_parts() + mapped: set[str] = set() + lookup = {m.source_field: m.ocsf_path for m in rule.mappings} + for source_field, value in all_fields.items(): + ocsf_path = lookup.get(source_field) + if ocsf_path is None: + continue + mapped.add(source_field) + _set_path(parts, ocsf_path, value) + return _build_document(parts, _unmapped_regex(all_fields, mapped)), None + + +def _parse_extension_kv(extension_str: str) -> dict[str, str]: + """Parse key=value pairs from extension text (e.g. src=1.2.3.4 msg=Suspicious DNS Query).""" + pairs: dict[str, str] = {} + if not extension_str: + return pairs + for match in EXTENSION_KV.finditer(extension_str): + key = match.group(1) + value = match.group(2) if match.group(2) is not None else match.group(3) + pairs[key] = value.strip() + return pairs + + +# --- JSON sentinel path ------------------------------------------------------ + +def _parse_json(line: str, rule: Rule) -> tuple[dict | None, str | None]: + """``__JSON__`` rule: parse the line as JSON and walk dotted-path mappings.""" + try: + obj = json.loads(line) + except (json.JSONDecodeError, RecursionError) as exc: + # RecursionError: pathologically deep nesting is bad input too, not a crash. + return None, f"invalid JSON: {exc}" + if not isinstance(obj, dict): + return None, "invalid JSON: expected an object" + + parts = _new_parts() + mapped: set[str] = set() + for mapping in rule.mappings: + value = _resolve_json_path(obj, mapping.source_field) + if value is None: + continue + mapped.add(mapping.source_field) + _set_path(parts, mapping.ocsf_path, value) + return _build_document(parts, _unmapped_json(obj, mapped)), None + + +def _resolve_json_path(obj: dict, dotted_path: str) -> Any: + """Walk a nested dict using a dotted path like 'alert.severity'.""" + current: Any = obj + for part in dotted_path.split("."): + if not isinstance(current, dict) or part not in current: + return None + current = current[part] + return current + + +def _flatten(obj: dict, prefix: str = "") -> dict[str, Any]: + flat: dict[str, Any] = {} + for key, value in obj.items(): + path = f"{prefix}.{key}" if prefix else key + if isinstance(value, dict): + flat.update(_flatten(value, path)) + else: + flat[path] = value + return flat + + +# --- document construction (shared) ------------------------------------------- + +def _new_parts() -> dict[str, Any]: + return { + "src_endpoint": {}, + "dst_endpoint": {}, + "severity_id": None, + "time": None, + "action": None, + "message": None, + } + + +def _set_path(parts: dict[str, Any], ocsf_path: str, value: Any) -> None: + """Write one mapped value into the document accumulator. + + Targets outside OCSF_TARGETS are rejected here (defense-in-depth — + activation-time validation rejects first), so a rule can never write + outside the curated subset, e.g. override metadata.product or class_uid. + """ + if ocsf_path not in OCSF_TARGETS: + return + if ocsf_path == "src_endpoint.ip": + parts["src_endpoint"]["ip"] = str(value) + elif ocsf_path == "src_endpoint.port": + parts["src_endpoint"]["port"] = _as_int(value, keep=True) + elif ocsf_path == "dst_endpoint.ip": + parts["dst_endpoint"]["ip"] = str(value) + elif ocsf_path == "dst_endpoint.port": + parts["dst_endpoint"]["port"] = _as_int(value, keep=True) + elif ocsf_path == "action": + parts["action"] = str(value) + elif ocsf_path == "message": + parts["message"] = str(value) + elif ocsf_path == "time": + parts["time"] = str(value) + elif ocsf_path == "severity_id": + parts["severity_id"] = _as_int(value, keep=False) + + +def _as_int(value: Any, *, keep: bool) -> Any: + """int cast with the hot-path fix: a bad value keeps the string (ports) + or becomes None (severity) instead of crashing the parse.""" + try: + return int(value) + except (TypeError, ValueError): + return value if keep else None + + +def _unmapped_regex(all_fields: dict[str, Any], mapped: set[str]) -> dict[str, Any]: + """Captured fields with no OCSF mapping — never dropped (spec §6.4).""" + out: dict[str, Any] = {} + for source_field, value in all_fields.items(): + if source_field in mapped or source_field in _STRUCTURAL_KEYS or value is None: + continue + out[source_field] = int(value) if isinstance(value, str) and value.isdigit() else value + return out + + +def _unmapped_json(obj: dict, mapped: set[str]) -> dict[str, Any]: + return {k: v for k, v in _flatten(obj).items() if k not in mapped} + + +def _build_document(parts: dict[str, Any], unmapped: dict[str, Any]) -> dict[str, Any]: + return { + "class_uid": _CLASS_UID, + "class_name": _CLASS_NAME, + "activity_id": _ACTIVITY_ID, + "severity_id": parts["severity_id"], + "time": parts["time"], + "src_endpoint": parts["src_endpoint"] or None, + "dst_endpoint": parts["dst_endpoint"] or None, + "action": parts["action"], + "message": parts["message"], + "metadata": {"product": "ULPF"}, + "unmapped": unmapped, + } + + +# --- activation helpers -------------------------------------------------------- + +def _pattern_cap_errors(pattern: str) -> list[str]: + """Spec §10 pattern caps: length / group-count / nesting depth.""" + errors: list[str] = [] + if len(pattern) > MAX_PATTERN_CHARS: + errors.append( + f"pattern length {len(pattern)} exceeds cap {MAX_PATTERN_CHARS}" + ) + groups = 0 + depth = 0 + max_depth = 0 + i = 0 + while i < len(pattern): + ch = pattern[i] + if ch == "\\": # escaped character — never a group delimiter + i += 2 + continue + if ch == "(": + groups += 1 + depth += 1 + max_depth = max(max_depth, depth) + elif ch == ")": + depth -= 1 + i += 1 + if groups > MAX_PATTERN_GROUPS: + errors.append(f"pattern group count {groups} exceeds cap {MAX_PATTERN_GROUPS}") + if max_depth > MAX_PATTERN_NESTING: + errors.append(f"pattern nesting depth {max_depth} exceeds cap {MAX_PATTERN_NESTING}") + return errors + + +def _document_errors(doc: dict) -> list[str]: + """ocsf_schema.validator.validate_document, or [] when ocsf-schema is not + installed. The two libs are always installed together in practice (CI, dev + setup); the fallback mirrors the validator's own jsonschema-optional + contract and keeps ulpf-core importable standalone.""" + try: + from ocsf_schema.validator import validate_document + except ImportError: + return [] + return validate_document(doc) From bd8b6a5e39bde341f3ef8e947581742907cebce7 Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 13:17:00 +0530 Subject: [PATCH 05/18] fix: bound extension-KV scan with regex timeout; fail closed at activation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings on 0d4ffe7: the extension key=value scan ran un-timeouted on untrusted line text — a crafted 32KB CEF extension stalled parse() ~3s under the default 50ms timeout (quadratic backtracking of (\w+)= against an a-wall); the scan now runs under the same ms-scale timeout and a timeout returns the standard error (spec §10). Activation validation now fails closed when ocsf-schema is absent (schema validator unavailable -> error, never a silent pass); the validator's internal jsonschema fallback is unchanged. Port casts keep only string values (dict/list -> None), and the exact OCSF_TARGETS contract is pinned by a committed test. Co-Authored-By: Claude Code --- libs/ulpf-core/tests/test_parsing.py | 34 ++++++++++++++++++++ libs/ulpf-core/ulpf_core/parsing.py | 47 ++++++++++++++++++---------- 2 files changed, 65 insertions(+), 16 deletions(-) diff --git a/libs/ulpf-core/tests/test_parsing.py b/libs/ulpf-core/tests/test_parsing.py index 02fe523..f182c59 100644 --- a/libs/ulpf-core/tests/test_parsing.py +++ b/libs/ulpf-core/tests/test_parsing.py @@ -1,4 +1,6 @@ # libs/ulpf-core/tests/test_parsing.py — key cases, all with concrete data: +import sys + from ulpf_core.parsing import parse, validate_rule_output, OCSF_TARGETS from ulpf_core.models import Rule, Mapping @@ -80,3 +82,35 @@ def test_activation_validation_passes_for_golden_cef(): from pathlib import Path lines = (Path(__file__).parent / "data/golden/raw_logs_cef.txt").read_text().splitlines() assert validate_rule_output(CEF_RULE, lines) == [] + +def test_ocsf_targets_contract(): + assert OCSF_TARGETS == frozenset({"src_endpoint.ip","src_endpoint.port","dst_endpoint.ip","dst_endpoint.port","time","severity_id","action","message"}) + +def test_non_string_bad_port_becomes_none(): # keep only strings (spec §6.4 fix) + import json as _json + line = _json.dumps({"src_ip": "203.0.113.45", "src_port": {"oops": 1}}) + doc, err = parse(line, JSON_RULE) + assert err is None and doc is not None + assert doc["src_endpoint"] == {"ip": "203.0.113.45", "port": None} + +def test_extension_kv_scan_timeout_is_error(): # spec §10: untrusted blob too + # An 'a'-wall ending in '=' sends the extension KV regex into quadratic + # backtracking; at 32KB it used to stall parse() ~3s at the DEFAULT + # 50ms timeout because only the rule-pattern search was covered. + import time + evil_line = "CEF:0|v|p|1|s|n|5|" + "a" * 32000 + "=" + t0 = time.perf_counter() + doc, err = parse(evil_line, CEF_RULE) + elapsed = time.perf_counter() - t0 + assert doc is None and "timeout" in err + assert elapsed < 1.0 + +def test_activation_fails_closed_without_schema_validator(monkeypatch): + # Ruling: activation validation fails CLOSED — a missing ocsf-schema is an + # error, never a silent pass (the jsonschema fallback inside the validator + # itself is separate and runtime-canary-scoped). Poison both cache keys so + # the from-import hits ImportError even when earlier tests imported it. + monkeypatch.setitem(sys.modules, "ocsf_schema", None) + monkeypatch.setitem(sys.modules, "ocsf_schema.validator", None) + errs = validate_rule_output(CEF_RULE, [CEF_LINE]) + assert errs and any("schema validator unavailable" in e for e in errs) diff --git a/libs/ulpf-core/ulpf_core/parsing.py b/libs/ulpf-core/ulpf_core/parsing.py index a258f0f..97e2ef5 100644 --- a/libs/ulpf-core/ulpf_core/parsing.py +++ b/libs/ulpf-core/ulpf_core/parsing.py @@ -3,9 +3,10 @@ Ported from legacy-demo/parser_engine/apply_rule.py (the proven demo logic) with the spec fixes M1 requires: -- rule-supplied regex runs through the ``regex`` module under a per-match - timeout (spec §10) — timeout / compile error / no match are returned as - errors, never a hot-path hang or raise; +- rule-supplied regex and the extension key=value scan (both untrusted line + text) run through the ``regex`` module under a per-match ms timeout + (spec §10) — timeout / compile error / no match are returned as errors, + never a hot-path hang or raise; - mapping targets are allow-listed (``OCSF_TARGETS``) at document build — defense-in-depth behind activation-time validation, so a rule can never write outside the curated subset; @@ -129,7 +130,12 @@ def _parse_regex(line: str, rule: Rule, timeout_ms: int) -> tuple[dict | None, s all_fields: dict[str, Any] = dict(match.groupdict()) extension_text = all_fields.get("extension") if extension_text: - all_fields.update(_parse_extension_kv(extension_text)) + try: + all_fields.update(_parse_extension_kv(extension_text, timeout_ms)) + except TimeoutError: + # The rule-pattern timeout above does not cover the KV scan; the + # extension blob is untrusted line text too (spec §10). + return None, "timeout" parts = _new_parts() mapped: set[str] = set() @@ -143,12 +149,17 @@ def _parse_regex(line: str, rule: Rule, timeout_ms: int) -> tuple[dict | None, s return _build_document(parts, _unmapped_regex(all_fields, mapped)), None -def _parse_extension_kv(extension_str: str) -> dict[str, str]: - """Parse key=value pairs from extension text (e.g. src=1.2.3.4 msg=Suspicious DNS Query).""" +def _parse_extension_kv(extension_str: str, timeout_ms: int) -> dict[str, str]: + """Parse key=value pairs from extension text (e.g. src=1.2.3.4 msg=Suspicious DNS Query). + + The extension blob is untrusted line text, so the scan runs under the same + ms-scale timeout as the rule pattern (spec §10) — quadratic backtracking + on a crafted blob becomes a TimeoutError, not a hot-path stall. + """ pairs: dict[str, str] = {} if not extension_str: return pairs - for match in EXTENSION_KV.finditer(extension_str): + for match in EXTENSION_KV.finditer(extension_str, timeout=timeout_ms / 1000.0): key = match.group(1) value = match.group(2) if match.group(2) is not None else match.group(3) pairs[key] = value.strip() @@ -240,12 +251,14 @@ def _set_path(parts: dict[str, Any], ocsf_path: str, value: Any) -> None: def _as_int(value: Any, *, keep: bool) -> Any: - """int cast with the hot-path fix: a bad value keeps the string (ports) - or becomes None (severity) instead of crashing the parse.""" + """int cast with the hot-path fix: a non-castable value keeps the original + string (ports) or becomes None (severity) instead of crashing the parse. + Non-string values (dict/list from JSON) become None either way — only a + string is worth preserving verbatim.""" try: return int(value) except (TypeError, ValueError): - return value if keep else None + return value if (keep and isinstance(value, str)) else None def _unmapped_regex(all_fields: dict[str, Any], mapped: set[str]) -> dict[str, Any]: @@ -311,12 +324,14 @@ def _pattern_cap_errors(pattern: str) -> list[str]: def _document_errors(doc: dict) -> list[str]: - """ocsf_schema.validator.validate_document, or [] when ocsf-schema is not - installed. The two libs are always installed together in practice (CI, dev - setup); the fallback mirrors the validator's own jsonschema-optional - contract and keeps ulpf-core importable standalone.""" + """ocsf_schema.validator.validate_document; FAILS CLOSED when ocsf-schema + is not installed — a missing validator is an activation error, never a + silent pass (the activation gate must not approve rules it could not + check). ulpf-core keeps no hard dependency on ocsf-schema (the plan + installs the two libs side by side); the validator's own jsonschema + fallback inside ocsf_schema is separate and runtime-canary-scoped.""" try: from ocsf_schema.validator import validate_document - except ImportError: - return [] + except ImportError as exc: + return [f"schema validator unavailable: {exc}"] return validate_document(doc) From 84b8045d481c54fadf500c5b55a4d4aa1f486443 Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 13:40:23 +0530 Subject: [PATCH 06/18] feat: compose stack, migrations with least-privilege roles, topic init Co-Authored-By: Claude Code --- .gitattributes | 3 + .gitignore | 1 + deploy/.env.example | 14 +++++ deploy/docker-compose.yml | 53 ++++++++++++++++++ deploy/migrate.Dockerfile | 11 ++++ deploy/migrate.py | 67 +++++++++++++++++++++++ deploy/migrations/001_init.sql | 82 ++++++++++++++++++++++++++++ deploy/migrations/002_partitions.sql | 20 +++++++ deploy/redpanda-init.sh | 5 ++ 9 files changed, 256 insertions(+) create mode 100644 .gitattributes create mode 100644 deploy/.env.example create mode 100644 deploy/docker-compose.yml create mode 100644 deploy/migrate.Dockerfile create mode 100644 deploy/migrate.py create mode 100644 deploy/migrations/001_init.sql create mode 100644 deploy/migrations/002_partitions.sql create mode 100644 deploy/redpanda-init.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..85433aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Shell scripts run inside Linux containers (e.g. deploy/redpanda-init.sh is +# bind-mounted into the redpanda image) — they must stay LF on every platform. +*.sh text eol=lf diff --git a/.gitignore b/.gitignore index aeb1679..ae12b44 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ ENV/ # Env / secrets .env +deploy/.env *.env.local # OS / editor cruft diff --git a/deploy/.env.example b/deploy/.env.example new file mode 100644 index 0000000..73b5553 --- /dev/null +++ b/deploy/.env.example @@ -0,0 +1,14 @@ +# deploy/.env.example — copy to deploy/.env and adjust as needed. +# docker compose reads deploy/.env automatically; the real file is gitignored, +# so never commit it. The values below are the M1 development defaults that +# docker-compose.yml also hardcodes for the in-network services. + +# Pipeline service — least-privilege role: INSERT+SELECT on raw_events, +# raw_batches, normalized_events, onboarding_samples; SELECT on rules. +PIPELINE_DATABASE_URL=postgresql://pipeline_role:pipeline_dev@postgres:5432/ulpf + +# Gateway service — read-only role: SELECT on all ulpf tables. +GATEWAY_DATABASE_URL=postgresql://gateway_role:gateway_dev@postgres:5432/ulpf + +# Kafka bootstrap address for the streaming services (inside the compose network). +KAFKA_BOOTSTRAP=redpanda:9092 diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..767ddd9 --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,53 @@ +# deploy/docker-compose.yml — M1 profile. Ollama/onboarding join in M2. +name: ulpf +services: + postgres: + image: postgres:16 + environment: { POSTGRES_USER: postgres, POSTGRES_PASSWORD: postgres, POSTGRES_DB: ulpf } + ports: ["5432:5432"] + volumes: [pgdata:/var/lib/postgresql/data] + healthcheck: { test: ["CMD-SHELL", "pg_isready -U postgres -d ulpf"], interval: 3s, retries: 20 } + migrate: + build: { context: .., dockerfile: deploy/migrate.Dockerfile } + environment: { ADMIN_DATABASE_URL: postgresql://postgres:postgres@postgres:5432/ulpf } + depends_on: { postgres: { condition: service_healthy } } + redpanda: + image: redpandadata/redpanda:v24.1.7 + command: [ redpanda, start, --overprovisioned, --smp, "1", --memory, 1G, + --kafka-addr, "PLAINTEXT://0.0.0.0:9092", --advertise-kafka-addr, "PLAINTEXT://redpanda:9092" ] + ports: ["9092:9092"] + healthcheck: { test: ["CMD", "rpk", "cluster", "info"], interval: 5s, retries: 20 } + topics: + image: redpandadata/redpanda:v24.1.7 + depends_on: { redpanda: { condition: service_healthy } } + # rpk defaults to 127.0.0.1:9092, which is not the broker from this container. + environment: { RPK_BROKERS: redpanda:9092 } + entrypoint: [ "/bin/sh", "-c", "/init.sh" ] + volumes: [ ./redpanda-init.sh:/init.sh ] + collector: + build: { context: .., dockerfile: services/collector/Dockerfile } + environment: { KAFKA_BOOTSTRAP: redpanda:9092 } + ports: ["5514:5514/udp", "5514:5514", "8080:8080"] + depends_on: { topics: { condition: service_completed_successfully } } + pipeline: + build: { context: .., dockerfile: services/pipeline/Dockerfile } + environment: + KAFKA_BOOTSTRAP: redpanda:9092 + DATABASE_URL: postgresql://pipeline_role:pipeline_dev@postgres:5432/ulpf + depends_on: { migrate: { condition: service_completed_successfully }, topics: { condition: service_completed_successfully } } + gateway: + build: { context: .., dockerfile: services/gateway/Dockerfile } + environment: + DATABASE_URL: postgresql://gateway_role:gateway_dev@postgres:5432/ulpf + ports: ["8000:8000"] + depends_on: { migrate: { condition: service_completed_successfully } } + web: + build: { context: .., dockerfile: web/Dockerfile } + ports: ["3000:3000"] + depends_on: [ gateway ] + simulator: + profiles: [ sim ] + build: { context: .., dockerfile: simulator/Dockerfile } + environment: { COLLECTOR_UDP: collector:5514, COLLECTOR_HTTP: http://collector:8080 } + depends_on: [ collector ] +volumes: { pgdata: {} } diff --git a/deploy/migrate.Dockerfile b/deploy/migrate.Dockerfile new file mode 100644 index 0000000..071baa4 --- /dev/null +++ b/deploy/migrate.Dockerfile @@ -0,0 +1,11 @@ +# deploy/migrate.Dockerfile — one-shot schema migration runner (compose service `migrate`). +# Build context is the repo root (compose build.context: ..). +FROM python:3.12-slim + +RUN pip install --no-cache-dir "psycopg[binary]==3.2.1" + +WORKDIR /app +COPY deploy/migrate.py /app/deploy/migrate.py +COPY deploy/migrations/ /app/deploy/migrations/ + +ENTRYPOINT ["python", "/app/deploy/migrate.py", "--dir", "/app/deploy/migrations"] diff --git a/deploy/migrate.py b/deploy/migrate.py new file mode 100644 index 0000000..a8f6cd4 --- /dev/null +++ b/deploy/migrate.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""ULPF schema migration runner. + +Plain runner, stdlib + psycopg only. Connects as the admin (superuser) +via the ADMIN_DATABASE_URL environment variable, ensures the +schema_migrations bookkeeping table exists, then applies each .sql file +in --dir that is not yet recorded there. Each migration runs in its own +transaction and is recorded in the same transaction, so a failed +migration leaves no half-applied state and recorded migrations are +never re-applied. +""" + +import argparse +import os +import sys +from pathlib import Path + +import psycopg + + +def main() -> int: + parser = argparse.ArgumentParser(description="Apply pending ULPF SQL migrations.") + parser.add_argument("--dir", default="deploy/migrations", + help="directory containing .sql migration files") + args = parser.parse_args() + + admin_url = os.environ.get("ADMIN_DATABASE_URL") + if not admin_url: + print("error: ADMIN_DATABASE_URL is not set", file=sys.stderr) + return 2 + + migration_dir = Path(args.dir) + files = sorted(migration_dir.glob("*.sql")) + if not files: + print(f"error: no .sql files found in {migration_dir}", file=sys.stderr) + return 2 + + # autocommit=True so each conn.transaction() below is a real top-level + # transaction that commits per file, independently of later files. + with psycopg.connect(admin_url, autocommit=True) as conn: + with conn.cursor() as cur: + cur.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + "name TEXT PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())" + ) + cur.execute("SELECT name FROM schema_migrations") + applied = {row[0] for row in cur.fetchall()} + + pending = [f for f in files if f.name not in applied] + if not pending: + print("migrations: up to date") + return 0 + + for path in pending: + print(f"applying {path.name}") + with conn.transaction(): + cur.execute(path.read_text(encoding="utf-8")) + cur.execute( + "INSERT INTO schema_migrations (name) VALUES (%s)", + (path.name,), + ) + print(f"applied {path.name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/migrations/001_init.sql b/deploy/migrations/001_init.sql new file mode 100644 index 0000000..01c5145 --- /dev/null +++ b/deploy/migrations/001_init.sql @@ -0,0 +1,82 @@ +-- deploy/migrations/001_init.sql — spec §4. Owner is the migration user; app roles get least privilege. +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='pipeline_role') THEN + CREATE ROLE pipeline_role LOGIN PASSWORD 'pipeline_dev'; END IF; + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='gateway_role') THEN + CREATE ROLE gateway_role LOGIN PASSWORD 'gateway_dev'; END IF; +END $$; + +CREATE TABLE raw_events ( + raw_id UUID NOT NULL, + received_at TIMESTAMPTZ NOT NULL, + source_id TEXT NOT NULL, + transport TEXT NOT NULL, + format_hint TEXT, + fingerprint_id TEXT, + content_hash TEXT NOT NULL, + raw_text TEXT NOT NULL, + PRIMARY KEY (raw_id, received_at) +) PARTITION BY RANGE (received_at); + +CREATE TABLE raw_batches ( + partition_id INT NOT NULL, -- kafka partition + batch_seq BIGINT NOT NULL, + prev_hash TEXT NOT NULL, -- previous batch's merkle_root; 64 zeros for genesis + merkle_root TEXT NOT NULL, + row_hashes JSONB NOT NULL, -- ordered content hashes of the batch (verifier input) + count INT NOT NULL, + first_offset BIGINT NOT NULL, + last_offset BIGINT NOT NULL, + written_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (partition_id, batch_seq) +); + +CREATE TABLE normalized_events ( + event_id UUID NOT NULL, + raw_id UUID NOT NULL, + raw_received_at TIMESTAMPTZ NOT NULL, + parsed_at TIMESTAMPTZ NOT NULL, -- set to raw received_at (deterministic; spec §4 note) + fingerprint_id TEXT NOT NULL, + rule_id INT, + rule_version INT, + status TEXT NOT NULL CHECK (status IN ('parsed','unparsed','quarantined','parse_error')), + ocsf JSONB, + superseded_by_event_id UUID, + PRIMARY KEY (event_id, parsed_at), + FOREIGN KEY (raw_id, raw_received_at) REFERENCES raw_events (raw_id, received_at) +) PARTITION BY RANGE (parsed_at); +CREATE INDEX ne_current_idx ON normalized_events (parsed_at DESC) WHERE superseded_by_event_id IS NULL; +CREATE INDEX ne_fp_idx ON normalized_events (fingerprint_id, parsed_at); + +CREATE TABLE rules ( + id SERIAL PRIMARY KEY, + fingerprint_id TEXT NOT NULL, + version INT NOT NULL, + pattern TEXT NOT NULL, + mappings JSONB NOT NULL, + provenance TEXT NOT NULL CHECK (provenance IN ('slm','slm-edited','human')), + confidence REAL, + status TEXT NOT NULL DEFAULT 'pending_review' + CHECK (status IN ('pending_review','active','superseded','deactivated','rejected')), + quarantined_fields TEXT[] NOT NULL DEFAULT '{}', + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + activated_at TIMESTAMPTZ, + deactivated_at TIMESTAMPTZ, + UNIQUE (fingerprint_id, version) +); +CREATE UNIQUE INDEX rules_one_active ON rules (fingerprint_id) WHERE status = 'active'; + +CREATE TABLE audit_log ( + id BIGSERIAL PRIMARY KEY, ts TIMESTAMPTZ NOT NULL DEFAULT now(), + actor TEXT NOT NULL, action TEXT NOT NULL, entity TEXT NOT NULL, detail JSONB +); +CREATE TABLE onboarding_samples ( + id BIGSERIAL PRIMARY KEY, fingerprint_id TEXT NOT NULL, raw_id UUID NOT NULL, + raw_text TEXT NOT NULL, captured_at TIMESTAMPTZ NOT NULL DEFAULT now(), + role TEXT NOT NULL DEFAULT 'unused' +); + +GRANT INSERT, SELECT ON raw_events, raw_batches, normalized_events, onboarding_samples TO pipeline_role; +GRANT SELECT ON rules TO pipeline_role; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO gateway_role; diff --git a/deploy/migrations/002_partitions.sql b/deploy/migrations/002_partitions.sql new file mode 100644 index 0000000..37758a0 --- /dev/null +++ b/deploy/migrations/002_partitions.sql @@ -0,0 +1,20 @@ +-- deploy/migrations/002_partitions.sql — daily partitions for the next 30 days +-- plus a DEFAULT partition on each range-partitioned table (overflow safety). +-- Partition names follow raw_events_y2026mmdd / normalized_events_y2026mmdd. +DO $$ +DECLARE + d date; +BEGIN + FOR d IN SELECT generate_series(now()::date, now()::date + 30, '1 day') + LOOP + EXECUTE format( + 'CREATE TABLE IF NOT EXISTS %I PARTITION OF raw_events FOR VALUES FROM (%L) TO (%L)', + 'raw_events_y' || to_char(d, 'YYYYMMDD'), d, d + 1); + EXECUTE format( + 'CREATE TABLE IF NOT EXISTS %I PARTITION OF normalized_events FOR VALUES FROM (%L) TO (%L)', + 'normalized_events_y' || to_char(d, 'YYYYMMDD'), d, d + 1); + END LOOP; + + EXECUTE 'CREATE TABLE IF NOT EXISTS raw_events_default PARTITION OF raw_events DEFAULT'; + EXECUTE 'CREATE TABLE IF NOT EXISTS normalized_events_default PARTITION OF normalized_events DEFAULT'; +END $$; diff --git a/deploy/redpanda-init.sh b/deploy/redpanda-init.sh new file mode 100644 index 0000000..0723ffa --- /dev/null +++ b/deploy/redpanda-init.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -e +rpk topic create raw.logs -p 6 +rpk topic create normalized.events -p 6 +rpk topic create pipeline.dlq -p 3 From 2a89cbe22dc1076080d7a321cba7b48772546f3b Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 14:08:13 +0530 Subject: [PATCH 07/18] feat: collector service (syslog udp/tcp + http ingest) producing raw.logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wraps every inbound line in a RawEnvelope and produces it to raw.logs, keyed by source_id; zero parsing (spec §7). UDP datagrams and TCP lines on :5514 derive source_id from the peer address (udp-/tcp-, controller R7 ruling — no device registry until post-M1); HTTP POST /v1/ingest takes the request-supplied source_id and optional format_hint. Envelopes are validated against RawEnvelope before produce; oversized/empty lines -> HTTP 422, produce failures -> HTTP 503 / syslog drop with error log (M1 queue-down deviation; spooling is M3). Tests use an injected fake producer with UDP disabled; root .dockerignore (R12) keeps build contexts from shipping the whole worktree. Co-Authored-By: Claude Code --- .dockerignore | 11 ++ pyproject.toml | 2 +- services/collector/Dockerfile | 19 ++ services/collector/__init__.py | 1 + services/collector/app.py | 192 +++++++++++++++++++++ services/collector/producer.py | 36 ++++ services/collector/requirements.txt | 4 + services/collector/tests/__init__.py | 0 services/collector/tests/test_collector.py | 39 +++++ 9 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 services/collector/Dockerfile create mode 100644 services/collector/__init__.py create mode 100644 services/collector/app.py create mode 100644 services/collector/producer.py create mode 100644 services/collector/requirements.txt create mode 100644 services/collector/tests/__init__.py create mode 100644 services/collector/tests/test_collector.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fe95f25 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +# R12: every service builds with compose build.context: .. — never ship these to the daemon. +.git +.claude +.superpowers +references +legacy-demo +**/__pycache__ +**/*.egg-info +.venv +node_modules +.pytest_cache diff --git a/pyproject.toml b/pyproject.toml index 17be3ab..4293f8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.pytest.ini_options] testpaths = ["libs", "services"] -pythonpath = ["."] +pythonpath = [".", "services/collector"] [tool.ruff] line-length = 100 diff --git a/services/collector/Dockerfile b/services/collector/Dockerfile new file mode 100644 index 0000000..d58c00d --- /dev/null +++ b/services/collector/Dockerfile @@ -0,0 +1,19 @@ +# services/collector/Dockerfile — syslog UDP/TCP + HTTP ingest producing raw.logs. +# Build context is the repo root (compose build.context: ..; see /.dockerignore). +FROM python:3.12-slim + +WORKDIR /app + +# ulpf-core first, then deps (layer caching), then the service package. +# /app contains collector/ and libs/ so `python -m collector.app` imports cleanly. +COPY libs/ulpf-core/pyproject.toml /app/libs/ulpf-core/pyproject.toml +COPY libs/ulpf-core/ulpf_core/ /app/libs/ulpf-core/ulpf_core/ +COPY services/collector/requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt \ + && pip install --no-cache-dir /app/libs/ulpf-core + +COPY services/collector/__init__.py /app/collector/__init__.py +COPY services/collector/app.py /app/collector/app.py +COPY services/collector/producer.py /app/collector/producer.py + +CMD ["python", "-m", "collector.app"] diff --git a/services/collector/__init__.py b/services/collector/__init__.py new file mode 100644 index 0000000..83991eb --- /dev/null +++ b/services/collector/__init__.py @@ -0,0 +1 @@ +# services/collector — syslog UDP/TCP + HTTP ingest producing raw.logs (M1 walking skeleton). diff --git a/services/collector/app.py b/services/collector/app.py new file mode 100644 index 0000000..8f923e3 --- /dev/null +++ b/services/collector/app.py @@ -0,0 +1,192 @@ +# services/collector/app.py — collector ingest service (M1 walking skeleton). +# Wraps every inbound line in a RawEnvelope and produces it to raw.logs. Zero parsing. +# +# M1 queue-down behavior (documented deviation, spooling arrives in M3): +# produce errors -> HTTP 503 / syslog drop with an error log. +import asyncio +import contextlib +import logging +import os +from datetime import datetime, timezone + +import uvicorn +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel, ValidationError + +from collector.producer import KafkaProducer +from ulpf_core.models import RawEnvelope + +log = logging.getLogger("collector") + +SYSLOG_PORT = 5514 +HTTP_PORT = 8080 +MAX_LINE_CHARS = 65536 # RawEnvelope.raw cap, spec §10 + + +def make_envelope(transport: str, source_id: str, raw: str) -> dict: + """Wrap one raw line in a RawEnvelope-shaped dict (validated before produce).""" + return { + "transport": transport, + "source_id": source_id, + "received_at": datetime.now(timezone.utc).isoformat(), + "raw": raw, + } + + +def source_id_for(transport: str, peer: tuple | None) -> str: + """R7: UDP/TCP syslog source_id derives from the peer address (udp-/tcp-). + + Falls back to the transport name if the peer address is unavailable. + """ + if peer is None: + return transport + prefix = "udp" if transport == "syslog-udp" else "tcp" + return f"{prefix}-{peer[0]}" + + +def _validated(env: dict) -> dict: + """Validate the envelope against RawEnvelope (empty/oversized lines raise).""" + RawEnvelope(**env) + return env + + +class SyslogUdpProtocol(asyncio.DatagramProtocol): + """One datagram = one event. Produce runs off-loop via the default executor.""" + + def __init__(self, producer): + self.producer = producer + self._in_flight: set = set() + + def datagram_received(self, data: bytes, addr): + source_id = source_id_for("syslog-udp", addr) + raw = data.decode("utf-8", errors="replace").rstrip("\r\n") + try: + env = _validated(make_envelope("syslog-udp", source_id, raw)) + except ValidationError: + log.warning("udp %s: datagram rejected (%d chars), dropped", source_id, len(raw)) + return + fut = asyncio.get_running_loop().run_in_executor( + None, self.producer.produce_raw, env + ) + self._in_flight.add(fut) + fut.add_done_callback(self._on_produced) + + def _on_produced(self, fut) -> None: + self._in_flight.discard(fut) + if fut.cancelled(): + return + if exc := fut.exception(): + log.error("udp produce failed, event dropped: %s", exc) + + +async def run_syslog_udp(producer, host: str = "0.0.0.0", port: int = SYSLOG_PORT): + """Bind the UDP socket and serve datagrams until cancelled.""" + loop = asyncio.get_running_loop() + transport, _ = await loop.create_datagram_endpoint( + lambda: SyslogUdpProtocol(producer), local_addr=(host, port) + ) + log.info("syslog udp listening on %s:%d", host, port) + try: + await asyncio.Future() # run until cancelled + finally: + transport.close() + + +async def handle_syslog_tcp(producer, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): + """One line = one event. source_id from the peer address (R7).""" + source_id = source_id_for("syslog-tcp", writer.get_extra_info("peername")) + try: + while line := await reader.readline(): + raw = line.decode("utf-8", errors="replace").rstrip("\r\n") + try: + env = _validated(make_envelope("syslog-tcp", source_id, raw)) + except ValidationError: + log.warning("tcp %s: line rejected (%d chars), dropped", source_id, len(raw)) + continue + producer.produce_raw(env) + except (ConnectionError, asyncio.IncompleteReadError): + pass # client went away mid-line; nothing to salvage + except Exception: + log.exception("tcp %s: handler error", source_id) + finally: + with contextlib.suppress(Exception): + writer.close() + await writer.wait_closed() + + +async def run_syslog_tcp(producer, host: str = "0.0.0.0", port: int = SYSLOG_PORT): + """TCP line server; serves until cancelled.""" + server = await asyncio.start_server( + lambda r, w: handle_syslog_tcp(producer, r, w), host, port + ) + log.info("syslog tcp listening on %s:%d", host, port) + async with server: + await server.serve_forever() + + +class IngestRequest(BaseModel): + source_id: str + lines: list[str] + format_hint: str | None = None + + +def build_app(producer, udp: bool = True): + """FastAPI app factory; the Kafka producer is injected (tests use a fake).""" + + @contextlib.asynccontextmanager + async def lifespan(app): + tasks: list[asyncio.Task] = [] + if udp: + tasks.append(asyncio.create_task(run_syslog_udp(producer))) + tasks.append(asyncio.create_task(run_syslog_tcp(producer))) + try: + yield + finally: + for t in tasks: + t.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + if flush := getattr(producer, "flush", None): + remaining = flush(5) + log.info("shutdown flush: %d message(s) remaining", remaining) + + app = FastAPI(lifespan=lifespan) + + @app.post("/v1/ingest", status_code=202) + async def ingest(body: IngestRequest): + envelopes = [] + for line in body.lines: + env = make_envelope("http", body.source_id, line) + if body.format_hint is not None: + env["format_hint"] = body.format_hint + try: + envelopes.append(_validated(env)) + except ValidationError: + raise HTTPException( + status_code=422, + detail="line rejected: must be 1..65536 chars", + ) + try: + for env in envelopes: + producer.produce_raw(env) + except Exception: + log.exception("http produce failed") + raise HTTPException(status_code=503, detail="ingest temporarily unavailable") + return {"accepted": len(envelopes)} + + return app + + +def main(): + logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s" + ) + producer = KafkaProducer(os.environ.get("KAFKA_BOOTSTRAP", "localhost:9092")) + app = build_app(producer, udp=True) + # uvicorn installs the SIGTERM/SIGINT handlers; graceful shutdown runs the + # lifespan shutdown above, which flushes the producer. + uvicorn.run(app, host="0.0.0.0", port=HTTP_PORT, log_level="info") + producer.flush(10) # belt-and-suspenders once the event loop is down + + +if __name__ == "__main__": + main() diff --git a/services/collector/producer.py b/services/collector/producer.py new file mode 100644 index 0000000..bcdd3a3 --- /dev/null +++ b/services/collector/producer.py @@ -0,0 +1,36 @@ +# services/collector/producer.py — thin wrapper over confluent_kafka.Producer. +# Produces RawEnvelope JSON to raw.logs, keyed by source_id. No parsing happens here. +import json +import logging + +log = logging.getLogger("collector.producer") + +TOPIC = "raw.logs" + + +class KafkaProducer: + """confluent_kafka.Producer wrapper: produce_raw(env) + flush() on shutdown. + + confluent_kafka is imported lazily so this module stays importable in test + environments without librdkafka (tests inject a fake producer instead). + """ + + def __init__(self, bootstrap: str): + import confluent_kafka # deferred: keeps the module importable without librdkafka + + self._producer = confluent_kafka.Producer({"bootstrap.servers": bootstrap}) + + def produce_raw(self, env: dict) -> None: + self._producer.produce( + TOPIC, + key=env["source_id"].encode(), + value=json.dumps(env).encode(), + ) + self._producer.poll(0) # serve delivery callbacks without blocking + + def flush(self, timeout: float = 10.0) -> int: + """Flush on shutdown; returns the number of messages still queued.""" + remaining = self._producer.flush(timeout) + if remaining: + log.error("flush timeout: %d message(s) still queued", remaining) + return remaining diff --git a/services/collector/requirements.txt b/services/collector/requirements.txt new file mode 100644 index 0000000..3c2b689 --- /dev/null +++ b/services/collector/requirements.txt @@ -0,0 +1,4 @@ +confluent-kafka==2.5.0 +fastapi==0.111.0 +uvicorn[standard]==0.30.1 +httpx # tests (FastAPI TestClient), dev diff --git a/services/collector/tests/__init__.py b/services/collector/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/collector/tests/test_collector.py b/services/collector/tests/test_collector.py new file mode 100644 index 0000000..9148389 --- /dev/null +++ b/services/collector/tests/test_collector.py @@ -0,0 +1,39 @@ +# services/collector/tests/test_collector.py — collector unit tests (fake producer, no Kafka). +import pytest +from collector.app import build_app # FastAPI app factory; producer injected + +class FakeProducer: + def __init__(self): self.produced = [] + def produce_raw(self, env_dict): self.produced.append(env_dict) + +@pytest.fixture +def client(): + prod = FakeProducer() + app = build_app(producer=prod, udp=False) # UDP disabled in tests + from fastapi.testclient import TestClient + return TestClient(app), prod + +def test_http_ingest_builds_envelopes(client): + c, prod = client + r = c.post("/v1/ingest", json={"source_id": "ids01", "lines": ['{"a":1}', '{"a":2}']}) + assert r.status_code == 202 and r.json() == {"accepted": 2} + assert len(prod.produced) == 2 + env = prod.produced[0] + assert env["source_id"] == "ids01" and env["transport"] == "http" and env["raw"] == '{"a":1}' + assert env["received_at"] # ISO8601 present + +def test_http_rejects_oversized_line(client): + c, prod = client + r = c.post("/v1/ingest", json={"source_id": "x", "lines": ["y" * 70000]}) + assert r.status_code == 422 + +def test_syslog_handler_wraps_datagram(): + from collector.app import make_envelope + env = make_envelope("syslog-udp", "fw01", "Aug 27 14:32:07 fw01 kernel: X") + assert env["transport"] == "syslog-udp" and env["source_id"] == "fw01" + +def test_source_id_derived_from_peer_address(): + # R7: UDP/TCP syslog source_id comes from the peer address, not a static name. + from collector.app import source_id_for + assert source_id_for("syslog-udp", ("10.1.2.3", 1234)) == "udp-10.1.2.3" + assert source_id_for("syslog-tcp", ("10.1.2.4", 5555)) == "tcp-10.1.2.4" From daa786b478c91136f13698e3688a411a1830974d Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 14:40:36 +0530 Subject: [PATCH 08/18] =?UTF-8?q?feat:=20pipeline=20worker=20=E2=80=94=20i?= =?UTF-8?q?dempotent=20raw-first=20batches,=20hash=20chain,=20parse,=20nor?= =?UTF-8?q?malized=20emit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sole raw.logs consumer (group id exactly "pipeline" for Task 12's forced replay, R3b). Per 500-msg/2s batch, spec §4 ordering: build_rows assembles RawRows with fingerprint_id classified at build time (R3 — pipeline_role has no UPDATE) -> persist_batch commits one txn of staging COPY -> INSERT .. ON CONFLICT (raw_id, received_at) DO NOTHING -> per-partition raw_batches append (batch_seq = COALESCE(MAX,0)+1, prev_hash -> prior merkle_root, genesis 64 zeros, ordered row_hashes JSONB) -> active rules refreshed -> parse (cap re-checked; parse_error + pipeline.dlq with raw_id/fingerprint/error; canary validates every 100th parsed doc) -> normalized.events produced keyed fingerprint_id -> persist_normalized committed on (event_id, parsed_at) -> only then consumer.commit(). parsed_at := raw received_at so replays collide instead of duplicating; superseded_by_event_id stays NULL in M1 (R9). psycopg/confluent_kafka import lazily so unit tests run without them (R14). Dockerfile installs both libs editable (R15); regex~=2024.7 (R8); redpanda-init.sh topics --if-not-exists (R10); root pythonpath gains services/pipeline (R13). Co-Authored-By: Claude Code --- deploy/redpanda-init.sh | 8 +- pyproject.toml | 2 +- services/pipeline/Dockerfile | 25 ++ services/pipeline/__init__.py | 2 + services/pipeline/db.py | 210 ++++++++++++++ services/pipeline/hashchain.py | 37 +++ services/pipeline/requirements.txt | 4 + services/pipeline/rules.py | 47 +++ services/pipeline/tests/__init__.py | 0 services/pipeline/tests/test_batch_builder.py | 102 +++++++ services/pipeline/tests/test_hashchain.py | 21 ++ services/pipeline/worker.py | 268 ++++++++++++++++++ 12 files changed, 722 insertions(+), 4 deletions(-) create mode 100644 services/pipeline/Dockerfile create mode 100644 services/pipeline/__init__.py create mode 100644 services/pipeline/db.py create mode 100644 services/pipeline/hashchain.py create mode 100644 services/pipeline/requirements.txt create mode 100644 services/pipeline/rules.py create mode 100644 services/pipeline/tests/__init__.py create mode 100644 services/pipeline/tests/test_batch_builder.py create mode 100644 services/pipeline/tests/test_hashchain.py create mode 100644 services/pipeline/worker.py diff --git a/deploy/redpanda-init.sh b/deploy/redpanda-init.sh index 0723ffa..115c51e 100644 --- a/deploy/redpanda-init.sh +++ b/deploy/redpanda-init.sh @@ -1,5 +1,7 @@ #!/bin/sh set -e -rpk topic create raw.logs -p 6 -rpk topic create normalized.events -p 6 -rpk topic create pipeline.dlq -p 3 +# --if-not-exists (controller R10): any stack re-run against a surviving broker +# must not fail the topics service and block depends_on consumers. +rpk topic create --if-not-exists raw.logs -p 6 +rpk topic create --if-not-exists normalized.events -p 6 +rpk topic create --if-not-exists pipeline.dlq -p 3 diff --git a/pyproject.toml b/pyproject.toml index 4293f8d..83fbce7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.pytest.ini_options] testpaths = ["libs", "services"] -pythonpath = [".", "services/collector"] +pythonpath = [".", "services/collector", "services/pipeline"] [tool.ruff] line-length = 100 diff --git a/services/pipeline/Dockerfile b/services/pipeline/Dockerfile new file mode 100644 index 0000000..fc896f1 --- /dev/null +++ b/services/pipeline/Dockerfile @@ -0,0 +1,25 @@ +# services/pipeline/Dockerfile — sole raw.logs consumer: idempotent raw-first +# batches with per-partition hash chain, parse, normalized emit. +# Build context is the repo root (compose build.context: ..; see /.dockerignore). +FROM python:3.12-slim + +WORKDIR /app + +# BOTH libs first (controller R15: the canary imports ocsf_schema.validator), +# then deps (layer caching), then the service package. +# /app contains pipeline/ and libs/ so `python -m pipeline.worker` imports cleanly. +COPY libs/ulpf-core/pyproject.toml /app/libs/ulpf-core/pyproject.toml +COPY libs/ulpf-core/ulpf_core/ /app/libs/ulpf-core/ulpf_core/ +COPY libs/ocsf-schema/pyproject.toml /app/libs/ocsf-schema/pyproject.toml +COPY libs/ocsf-schema/ocsf_schema/ /app/libs/ocsf-schema/ocsf_schema/ +COPY services/pipeline/requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt \ + && pip install --no-cache-dir -e /app/libs/ulpf-core -e /app/libs/ocsf-schema + +COPY services/pipeline/__init__.py /app/pipeline/__init__.py +COPY services/pipeline/db.py /app/pipeline/db.py +COPY services/pipeline/hashchain.py /app/pipeline/hashchain.py +COPY services/pipeline/rules.py /app/pipeline/rules.py +COPY services/pipeline/worker.py /app/pipeline/worker.py + +CMD ["python", "-m", "pipeline.worker"] diff --git a/services/pipeline/__init__.py b/services/pipeline/__init__.py new file mode 100644 index 0000000..39f5f73 --- /dev/null +++ b/services/pipeline/__init__.py @@ -0,0 +1,2 @@ +# services/pipeline — sole raw.logs consumer: idempotent raw-first persistence +# with a per-partition hash chain, then parse/emit (M1 walking skeleton). diff --git a/services/pipeline/db.py b/services/pipeline/db.py new file mode 100644 index 0000000..d1215a8 --- /dev/null +++ b/services/pipeline/db.py @@ -0,0 +1,210 @@ +# services/pipeline/db.py — idempotent raw-first persistence + chain append. +# +# persist_batch() is the system's correctness core, one transaction per batch: +# staging COPY -> INSERT .. ON CONFLICT (raw_id, received_at) DO NOTHING +# -> per-partition raw_batches append (prev_hash -> prior merkle_root). +# Raw rows land BEFORE any parsing happens (raw-first, spec §4); a forced +# replay re-runs the same statements, the ON CONFLICT dedups raw_events while +# raw_batches honestly records the re-delivery (Task 12 Assert D). +# +# psycopg is imported lazily inside the persist functions only, so unit tests +# exercise build_rows()/the dataclasses without psycopg installed (mirrors the +# collector's deferred confluent_kafka import). persist_* commit on success and +# leave the transaction aborted on failure — the worker rolls back and does NOT +# commit offsets, so failed batches are redelivered (at-least-once). +from __future__ import annotations + +import dataclasses +import hashlib +import json +import logging +import uuid +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING + +from ulpf_core.fingerprint import fingerprint_id +from ulpf_core.ids import raw_id_for +from ulpf_core.models import RawEnvelope + +from pipeline.hashchain import GENESIS_PREV_HASH, merkle_root + +if TYPE_CHECKING: + import psycopg + +log = logging.getLogger("pipeline.db") + +_RAW_COLUMNS = ( + "raw_id", "received_at", "source_id", "transport", "format_hint", + "fingerprint_id", "content_hash", "raw_text", +) + + +@dataclass(frozen=True) +class RawRow: + """One raw_events row; field order matches _RAW_COLUMNS for the COPY.""" + + raw_id: uuid.UUID + received_at: datetime + source_id: str + transport: str + format_hint: str | None + fingerprint_id: str + content_hash: str + raw_text: str + + +@dataclass +class PartitionMeta: + """Per-Kafka-partition grouping of one consume batch (offsets in order).""" + + partition_id: int + first_offset: int + last_offset: int + content_hashes: list[str] + + +@dataclass(frozen=True) +class NormalizedRow: + """One normalized_events row. superseded_by_event_id stays NULL: M1 never + supersedes (single active rule version per fingerprint, controller R9).""" + + event_id: uuid.UUID + raw_id: uuid.UUID + raw_received_at: datetime + parsed_at: datetime + fingerprint_id: str + rule_id: int | None + rule_version: int | None + status: str + ocsf: dict | None + + +def build_rows(msgs) -> tuple[list[RawRow], dict[int, PartitionMeta]]: + """Assemble RawRows + per-partition PartitionMeta from Kafka messages. + + Pure logic: each message's JSON value is validated against RawEnvelope, + the fingerprint is classified at build time (controller R3 — pipeline_role + cannot UPDATE post-insert, so fingerprint_id must be set before the COPY), + and content_hash is sha256 of the raw line's UTF-8 bytes. Undecodable or + invalid envelopes are logged and skipped (the offset still advances past + them — a poison envelope must not wedge the group). Same message twice + yields identical rows: deterministic idempotency precondition. + """ + rows: list[RawRow] = [] + parts: dict[int, PartitionMeta] = {} + for msg in msgs: + topic, partition, offset = msg.topic(), msg.partition(), msg.offset() + try: + env = RawEnvelope(**json.loads(msg.value())) + except (ValueError, TypeError) as exc: + log.warning("skipping %s[%d@%d]: undecodable/invalid envelope: %s", + topic, partition, offset, exc) + continue + raw_text = env.raw + meta = parts.get(partition) + if meta is None: + meta = parts[partition] = PartitionMeta(partition, offset, offset, []) + meta.last_offset = offset + row = RawRow( + raw_id=raw_id_for(topic, partition, offset), + received_at=datetime.fromisoformat(env.received_at), + source_id=env.source_id, + transport=env.transport, + format_hint=env.format_hint, + fingerprint_id=fingerprint_id(raw_text), + content_hash=hashlib.sha256(raw_text.encode()).hexdigest(), + raw_text=raw_text, + ) + rows.append(row) + meta.content_hashes.append(row.content_hash) + return rows, parts + + +def persist_batch(conn: psycopg.Connection, rows: list[RawRow], + partitions: dict[int, PartitionMeta]) -> None: + """Insert raw rows idempotently + append the per-partition chain, ONE txn. + + staging COPY -> INSERT .. ON CONFLICT (raw_id, received_at) DO NOTHING -> + per-partition raw_batches append with batch_seq = COALESCE(MAX, 0)+1 (safe: + one consumer per partition, spec §4), prev_hash = prior batch's merkle_root + (GENESIS_PREV_HASH for the first), row_hashes = the batch's ordered + content_hash list, and merkle_root computed from exactly that list. + Commits on success; on failure the caller rolls back and skips the offset + commit so Kafka redelivers the batch. + """ + if not partitions: + return + from psycopg.types.json import Json # deferred: keeps psycopg off the test path + + with conn.cursor() as cur: + cur.execute("CREATE TEMP TABLE staging_raw (LIKE raw_events INCLUDING DEFAULTS) ON COMMIT DROP") + with cur.copy( + f"COPY staging_raw ({', '.join(_RAW_COLUMNS)}) FROM STDIN" + ) as copy: + for row in rows: + copy.write_row(dataclasses.astuple(row)) + cur.execute( + f"INSERT INTO raw_events ({', '.join(_RAW_COLUMNS)}) " + f"SELECT {', '.join(_RAW_COLUMNS)} FROM staging_raw " + "ON CONFLICT (raw_id, received_at) DO NOTHING" + ) + for partition_id in sorted(partitions): + meta = partitions[partition_id] + cur.execute( + "SELECT COALESCE(MAX(batch_seq), 0) + 1 FROM raw_batches WHERE partition_id = %s", + (partition_id,), + ) + batch_seq = cur.fetchone()[0] + cur.execute( + "SELECT merkle_root FROM raw_batches WHERE partition_id = %s " + "ORDER BY batch_seq DESC LIMIT 1", + (partition_id,), + ) + prev = cur.fetchone() + prev_hash = prev[0] if prev else GENESIS_PREV_HASH + cur.execute( + "INSERT INTO raw_batches (partition_id, batch_seq, prev_hash, merkle_root, " + "row_hashes, count, first_offset, last_offset) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s)", + ( + partition_id, batch_seq, prev_hash, merkle_root(meta.content_hashes), + Json(meta.content_hashes), len(meta.content_hashes), + meta.first_offset, meta.last_offset, + ), + ) + conn.commit() + log.info("raw batch persisted: %d row(s) across %d partition(s)", len(rows), len(partitions)) + + +def persist_normalized(conn: psycopg.Connection, events: list[NormalizedRow]) -> None: + """Insert normalized rows idempotently (staged ON CONFLICT on + (event_id, parsed_at)), ONE txn; commits on success. + + The FK to raw_events holds because persist_batch already committed the raw + rows; a replay collides on the PK and is skipped (spec §4 idempotency). + """ + if not events: + return + from psycopg.types.json import Json # deferred: keeps psycopg off the test path + + columns = ( + "event_id", "raw_id", "raw_received_at", "parsed_at", "fingerprint_id", + "rule_id", "rule_version", "status", "ocsf", + ) + with conn.cursor() as cur: + cur.execute("CREATE TEMP TABLE staging_ne (LIKE normalized_events INCLUDING DEFAULTS) ON COMMIT DROP") + with cur.copy(f"COPY staging_ne ({', '.join(columns)}) FROM STDIN") as copy: + for ev in events: + copy.write_row(( + ev.event_id, ev.raw_id, ev.raw_received_at, ev.parsed_at, + ev.fingerprint_id, ev.rule_id, ev.rule_version, ev.status, + Json(ev.ocsf) if ev.ocsf is not None else None, + )) + cur.execute( + f"INSERT INTO normalized_events ({', '.join(columns)}) " + f"SELECT {', '.join(columns)} FROM staging_ne " + "ON CONFLICT (event_id, parsed_at) DO NOTHING" + ) + conn.commit() + log.info("normalized batch persisted: %d row(s)", len(events)) diff --git a/services/pipeline/hashchain.py b/services/pipeline/hashchain.py new file mode 100644 index 0000000..0cb268e --- /dev/null +++ b/services/pipeline/hashchain.py @@ -0,0 +1,37 @@ +"""Merkle root over batch content hashes — the raw_batches chain math (spec §4). + +Pure functions, no dependencies: hex sha256 strings in, hex sha256 string out. +Each batch stores the merkle root of its ordered content_hash list in +raw_batches.row_hashes + merkle_root, and prev_hash links the previous batch's +merkle_root (GENESIS_PREV_HASH = 64 zeros for each partition's first batch). +The same function independently re-verifies every stored root in Task 12's +smoke (recompute merkle_root(row_hashes) and compare), so this stays +dependency-free and dead simple on purpose. +""" + +from __future__ import annotations + +import hashlib + +# prev_hash of the first raw_batches row on each partition (spec §4). +GENESIS_PREV_HASH = "0" * 64 + + +def merkle_root(content_hashes: list[str]) -> str: + """Merkle root of ordered hex sha256 leaves. + + Pairwise sha256 of concatenated raw bytes; an odd level duplicates its last + hash (Bitcoin-style). A single leaf is its own root. Empty input is a + programmer error — a batch always carries at least one row — so it raises. + """ + if not content_hashes: + raise ValueError("merkle_root of an empty batch") + level = list(content_hashes) + while len(level) > 1: + if len(level) % 2 == 1: + level.append(level[-1]) + level = [ + hashlib.sha256(bytes.fromhex(level[i]) + bytes.fromhex(level[i + 1])).hexdigest() + for i in range(0, len(level), 2) + ] + return level[0] diff --git a/services/pipeline/requirements.txt b/services/pipeline/requirements.txt new file mode 100644 index 0000000..e4fc506 --- /dev/null +++ b/services/pipeline/requirements.txt @@ -0,0 +1,4 @@ +confluent-kafka==2.5.0 +psycopg[binary]==3.2.1 +regex~=2024.7 # controller R8: no cp313 wheel for ==2024.7.24; ~= matches ulpf-core's own pin +jsonschema==4.23.0 # canary: ocsf_schema.validator (spec §6.4) diff --git a/services/pipeline/rules.py b/services/pipeline/rules.py new file mode 100644 index 0000000..33b2473 --- /dev/null +++ b/services/pipeline/rules.py @@ -0,0 +1,47 @@ +# services/pipeline/rules.py — active-rule cache loader for the worker. +# +# One SELECT per batch (the worker refreshes every consume batch, so a rule +# activated mid-stream is picked up on the next batch — no restart, no push). +# mappings arrives as JSONB (list of {source_field, ocsf_path} dicts) and is +# rebuilt into the typed Mapping model before parse() ever sees the rule. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ulpf_core.models import Mapping, Rule + +if TYPE_CHECKING: + import psycopg + + +class ActiveRule(Rule): + """Rule plus the rules-table serial id. + + load_active_rules must return dict[str, Rule] per the plan contract, but + normalized_events.rule_id wants the DB id — so the id rides on this Rule + subclass (an ActiveRule IS a Rule; parse() and the type contract are + unaffected, and the worker reads ``rule.id`` for attribution). + """ + + id: int + + +def load_active_rules(conn: psycopg.Connection) -> dict[str, Rule]: + """All status='active' rules keyed by fingerprint_id (one per fingerprint, + enforced by the rules_one_active partial unique index).""" + rules: dict[str, Rule] = {} + with conn.cursor() as cur: + cur.execute( + "SELECT id, fingerprint_id, version, pattern, mappings, provenance " + "FROM rules WHERE status = 'active'" + ) + for rule_id, fingerprint, version, pattern, mappings, provenance in cur.fetchall(): + rules[fingerprint] = ActiveRule( + id=rule_id, + fingerprint_id=fingerprint, + version=version, + pattern=pattern, + mappings=[Mapping(**m) for m in mappings], + provenance=provenance, + ) + return rules diff --git a/services/pipeline/tests/__init__.py b/services/pipeline/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/pipeline/tests/test_batch_builder.py b/services/pipeline/tests/test_batch_builder.py new file mode 100644 index 0000000..1869378 --- /dev/null +++ b/services/pipeline/tests/test_batch_builder.py @@ -0,0 +1,102 @@ +# services/pipeline/tests/test_batch_builder.py — build_rows() pure batch assembly. +# No DB, no Kafka: fakes duck-type the confluent_kafka Message interface +# (topic()/partition()/offset()/value()); psycopg is never imported (db.py +# keeps it lazy), so this runs on any host with ulpf-core installed. +import hashlib +import json + +from ulpf_core.fingerprint import fingerprint_id +from ulpf_core.ids import raw_id_for + +from pipeline.db import build_rows + +TOPIC = "raw.logs" + + +class FakeMsg: + """Duck-typed confluent_kafka.Message carrying one RawEnvelope JSON value.""" + + def __init__(self, partition: int, offset: int, envelope: dict): + self._partition = partition + self._offset = offset + self._value = json.dumps(envelope).encode() + + def topic(self): + return TOPIC + + def partition(self): + return self._partition + + def offset(self): + return self._offset + + def value(self): + return self._value + + +def envelope(raw='{"a":1}', source_id="ids01", transport="http", + received_at="2026-09-19T12:00:00+00:00", **extra): + env = {"source_id": source_id, "transport": transport, + "received_at": received_at, "raw": raw} + env.update(extra) + return env + + +def test_build_rows_maps_raw_fields(): + raw = "CEF:0|acme|gw|1|probe|deny|high|src=198.51.100.7" + rows, parts = build_rows([FakeMsg(2, 41, envelope(raw=raw))]) + row = rows[0] + assert row.raw_id == raw_id_for(TOPIC, 2, 41) + assert row.received_at.isoformat() == "2026-09-19T12:00:00+00:00" + assert row.source_id == "ids01" and row.transport == "http" + assert row.format_hint is None + assert row.raw_text == raw + assert row.content_hash == hashlib.sha256(raw.encode()).hexdigest() + # R3: fingerprint classified at raw-insert time, before the batch COPY. + assert row.fingerprint_id == fingerprint_id(raw) + meta = parts[2] + assert meta.partition_id == 2 + assert meta.first_offset == 41 and meta.last_offset == 41 + assert meta.content_hashes == [row.content_hash] + + +def test_partitions_grouped_in_message_order(): + msgs = [ + FakeMsg(0, 7, envelope()), + FakeMsg(1, 3, envelope()), + FakeMsg(0, 8, envelope(raw='{"b":2}')), + ] + rows, parts = build_rows(msgs) + assert [r.raw_id for r in rows] == [ + raw_id_for(TOPIC, 0, 7), raw_id_for(TOPIC, 1, 3), raw_id_for(TOPIC, 0, 8), + ] + assert parts[0].first_offset == 7 and parts[0].last_offset == 8 + assert len(parts[0].content_hashes) == 2 + assert parts[1].first_offset == 3 and parts[1].last_offset == 3 + assert len(parts[1].content_hashes) == 1 + + +def test_same_message_twice_identical_rows(): + # Idempotency precondition: replay yields byte-identical rows + metas, + # so the ON CONFLICT insert and the chain append see the same values. + msg = FakeMsg(1, 9, envelope()) + rows1, parts1 = build_rows([msg]) + rows2, parts2 = build_rows([FakeMsg(1, 9, envelope())]) + assert rows1 == rows2 + assert parts1 == parts2 + + +def test_malformed_envelope_skipped_not_fatal(): + bad = FakeMsg(0, 1, {"source_id": "x"}) # missing transport/received_at/raw + good = FakeMsg(0, 2, envelope()) + rows, parts = build_rows([bad, good]) + assert len(rows) == 1 + assert rows[0].raw_id == raw_id_for(TOPIC, 0, 2) + assert parts[0].first_offset == 2 # poison offset simply advances past + + +def test_oversized_line_skipped(): + # RawEnvelope's 65536-char cap (spec §10) applies at build time too. + big = FakeMsg(0, 5, envelope(raw="x" * 70000)) + rows, parts = build_rows([big]) + assert rows == [] and parts == {} diff --git a/services/pipeline/tests/test_hashchain.py b/services/pipeline/tests/test_hashchain.py new file mode 100644 index 0000000..7173aff --- /dev/null +++ b/services/pipeline/tests/test_hashchain.py @@ -0,0 +1,21 @@ +# services/pipeline/tests/test_hashchain.py — merkle_root math (pure, no deps). +# Test bodies verbatim from the plan; the hashlib import sits at module level +# (the plan had it function-local in test_two_leaves_pair, which would leave +# test_odd_duplicates_last with no hashlib binding). +import hashlib + +from pipeline.hashchain import merkle_root + + +def test_single_leaf(): + assert merkle_root(["aa"]) == "aa" # single hash is its own root +def test_two_leaves_pair(): + expected = hashlib.sha256(bytes.fromhex("aa") + bytes.fromhex("bb")).hexdigest() + assert merkle_root(["aa", "bb"]) == expected +def test_odd_duplicates_last(): + r3 = merkle_root(["aa", "bb", "cc"]) + h01 = hashlib.sha256(bytes.fromhex("aa") + bytes.fromhex("bb")).hexdigest() + h22 = hashlib.sha256(bytes.fromhex("cc") + bytes.fromhex("cc")).hexdigest() + assert r3 == hashlib.sha256(bytes.fromhex(h01) + bytes.fromhex(h22)).hexdigest() +def test_order_matters(): + assert merkle_root(["aa", "bb"]) != merkle_root(["bb", "aa"]) diff --git a/services/pipeline/worker.py b/services/pipeline/worker.py new file mode 100644 index 0000000..939871b --- /dev/null +++ b/services/pipeline/worker.py @@ -0,0 +1,268 @@ +# services/pipeline/worker.py — the sole raw.logs consumer (M1 walking skeleton). +# +# Per batch of up to 500 messages (or 2s), the spec-critical ordering (§4): +# 1. build_rows (fingerprint at build time, controller R3) +# 2. persist_batch — raw_events + raw_batches chain, COMMITTED +# 3. refresh active rules +# 4. per event: apply active rule or 'unparsed'; parse failures -> parse_error +# + pipeline.dlq; parsed docs -> normalized.events (key=fingerprint_id) +# 5. persist_normalized — COMMITTED +# 6. only then consumer.commit() +# Crash anywhere before step 6 and the batch is redelivered; every write is +# deterministic and ON CONFLICT-guarded, so redelivery is a no-op (spec §4). +# +# parsed_at is the raw event's received_at (not wall clock) so a replay mints +# the same event_id and collides instead of duplicating. M1 never supersedes +# (superseded_by_event_id stays NULL, controller R9). +import json +import logging +import os + +from ocsf_schema.validator import validate_document +from ulpf_core.ids import event_id_for +from ulpf_core.parsing import parse + +from pipeline.db import NormalizedRow, build_rows, persist_batch, persist_normalized +from pipeline.rules import load_active_rules + +log = logging.getLogger("pipeline.worker") + +TOPIC_RAW = "raw.logs" +TOPIC_NORMALIZED = "normalized.events" +TOPIC_DLQ = "pipeline.dlq" +GROUP_ID = "pipeline" # exact name: Task 12's forced-replay deletes this group (controller R3b) + +BATCH_SIZE = 500 +BATCH_TIMEOUT_S = 2.0 +MAX_LINE_CHARS = 65536 # spec §10 line cap — re-checked before parse (collector checks at ingest) +CANARY_EVERY = 100 # every 100th parsed doc through validate_document (spec §6.4) + +# Sentinel rule version for events no active rule matched: deterministic, so a +# replay of the same raw line yields the same event_id and is skipped. +UNPARSED_RULE_VERSION = 0 + + +class KafkaConsumer: + """confluent_kafka.Consumer wrapper; import deferred so this module (and + the unit tests) load without librdkafka on the host (controller R14).""" + + def __init__(self, bootstrap: str, group_id: str = GROUP_ID): + import confluent_kafka # deferred: unit tests never need librdkafka + + self._consumer = confluent_kafka.Consumer({ + "bootstrap.servers": bootstrap, + "group.id": group_id, + "auto.offset.reset": "earliest", # raw-first: never silently skip history + "enable.auto.commit": False, # offsets only after both DB txns + }) + + def subscribe(self, topic: str) -> None: + self._consumer.subscribe([topic]) + + def consume(self, num_messages: int, timeout: float) -> list: + return self._consumer.consume(num_messages=num_messages, timeout=timeout) + + def commit(self) -> None: + self._consumer.commit() + + def close(self) -> None: + self._consumer.close() + + +class KafkaProducer: + """confluent_kafka.Producer wrapper for normalized.events / pipeline.dlq.""" + + def __init__(self, bootstrap: str): + import confluent_kafka # deferred: unit tests never need librdkafka + + self._producer = confluent_kafka.Producer({"bootstrap.servers": bootstrap}) + + def produce(self, topic: str, key: str, value: dict) -> None: + self._producer.produce(topic, key=key.encode(), value=json.dumps(value).encode()) + self._producer.poll(0) # serve delivery callbacks without blocking + + def flush(self, timeout: float = 10.0) -> int: + return self._producer.flush(timeout) + + +class PipelineWorker: + """Wires consumer + producer + DB connection into the batch loop. + + Collaborators are injected (tests and main() pass fakes or real wrappers); + nothing here imports psycopg or confluent_kafka at module import time. + """ + + def __init__(self, consumer, producer, conn): + self.consumer = consumer + self.producer = producer + self.conn = conn + self.canary_failures = 0 # spec §6.4: canary failures logged + counted + self._parsed_total = 0 + + def run(self) -> None: + self.consumer.subscribe(TOPIC_RAW) + log.info("pipeline consuming %s as group %r", TOPIC_RAW, GROUP_ID) + while True: + msgs = self.consumer.consume(num_messages=BATCH_SIZE, timeout=BATCH_TIMEOUT_S) + data = [] + for msg in msgs: + if msg.error() is not None: # broker/codec event, not a data message + log.warning("consumer event (non-data): %s", msg.error()) + continue + data.append(msg) + if data: + self.process_batch(data) + + def process_batch(self, msgs) -> bool: + """One full batch; returns True iff offsets were committed. + + Any persist failure aborts the batch BEFORE the offset commit: the + transaction is rolled back and Kafka redelivers (at-least-once). + """ + rows, partitions = build_rows(msgs) + + # Raw first: raw rows + chain are durable before any parsing (spec §4). + try: + persist_batch(self.conn, rows, partitions) + except Exception: + self.conn.rollback() + log.exception("raw persist failed; offsets NOT committed; batch redelivered") + return False + + rules = load_active_rules(self.conn) # refreshed every batch + events: list[NormalizedRow] = [] + parsed = unparsed = parse_errors = 0 + for row in rows: + rule = rules.get(row.fingerprint_id) + if rule is None: + events.append(NormalizedRow( + event_id=event_id_for(row.raw_id, UNPARSED_RULE_VERSION), + raw_id=row.raw_id, + raw_received_at=row.received_at, + parsed_at=row.received_at, # := raw received_at (deterministic, spec §4) + fingerprint_id=row.fingerprint_id, + rule_id=None, + rule_version=UNPARSED_RULE_VERSION, + status="unparsed", + ocsf=None, + )) + unparsed += 1 + continue + + # Line cap re-checked here: parse() runs untrusted text on the hot path. + if len(row.raw_text) > MAX_LINE_CHARS: + events.append(self._parse_error_row( + row, rule, f"line exceeds {MAX_LINE_CHARS} char cap")) + parse_errors += 1 + continue + + doc, err = parse(row.raw_text, rule) + if err is not None: + events.append(self._parse_error_row(row, rule, err)) + parse_errors += 1 + continue + + # parse() leaves time None when the rule maps no timestamp; the + # pipeline fills ingestion time before emission (ulpf_core.parsing). + if doc["time"] is None: + doc["time"] = row.received_at.isoformat() + + self._parsed_total += 1 + if self._parsed_total % CANARY_EVERY == 0: # canary, never per-event (spec §6.4) + problems = validate_document(doc) + if problems: + self.canary_failures += 1 + log.error("canary doc invalid for %s: %s", row.fingerprint_id, problems) + + events.append(NormalizedRow( + event_id=event_id_for(row.raw_id, rule.version), + raw_id=row.raw_id, + raw_received_at=row.received_at, + parsed_at=row.received_at, + fingerprint_id=row.fingerprint_id, + rule_id=rule.id, + rule_version=rule.version, + status="parsed", + ocsf=doc, + )) + try: + self.producer.produce(TOPIC_NORMALIZED, row.fingerprint_id, doc) + except Exception: + # The normalized row is durable in step 5 regardless; a lost + # emission is visible via the DB and must not stall the batch. + log.exception("produce to %s failed for %s", TOPIC_NORMALIZED, row.raw_id) + parsed += 1 + + try: + persist_normalized(self.conn, events) + except Exception: + self.conn.rollback() + log.exception("normalized persist failed; offsets NOT committed; batch redelivered") + return False + + self.consumer.commit() # both DB txns durable -> safe to advance offsets + log.info("batch: %d consumed, %d parsed, %d unparsed, %d parse_error, " + "%d skipped-envelope, canary_failures=%d", + len(msgs), parsed, unparsed, parse_errors, len(msgs) - len(rows), + self.canary_failures) + return True + + def _parse_error_row(self, row, rule, error: str) -> NormalizedRow: + """DLQ produce + the parse_error row, which joins the same normalized + txn as the rest of the batch.""" + try: + self.producer.produce(TOPIC_DLQ, row.fingerprint_id, { + "raw_id": str(row.raw_id), + "fingerprint_id": row.fingerprint_id, + "error": error, + }) + except Exception: + # Same reasoning as normalized.events: the DB row below is the + # record of truth; a lost DLQ emission must not stall the batch. + log.exception("produce to %s failed for %s", TOPIC_DLQ, row.raw_id) + return NormalizedRow( + event_id=event_id_for(row.raw_id, rule.version), + raw_id=row.raw_id, + raw_received_at=row.received_at, + parsed_at=row.received_at, + fingerprint_id=row.fingerprint_id, + rule_id=rule.id, + rule_version=rule.version, + status="parse_error", + ocsf=None, + ) + + +def connect_db(url: str): + """psycopg connection for pipeline_role (INSERT+SELECT only, no UPDATE/DELETE).""" + import psycopg # deferred: unit tests never need psycopg + + return psycopg.connect(url) + + +def main(): + logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s" + ) + bootstrap = os.environ.get("KAFKA_BOOTSTRAP", "localhost:9092") + db_url = os.environ.get( + "DATABASE_URL", "postgresql://pipeline_role:pipeline_dev@localhost:5432/ulpf" + ) + consumer = KafkaConsumer(bootstrap) + producer = KafkaProducer(bootstrap) + conn = connect_db(db_url) + worker = PipelineWorker(consumer, producer, conn) + try: + worker.run() + except KeyboardInterrupt: + log.info("shutdown requested") + finally: + consumer.close() + remaining = producer.flush(10) + if remaining: + log.error("shutdown flush: %d message(s) still queued", remaining) + conn.close() + + +if __name__ == "__main__": + main() From a8d2b12b1fe96dd949f53d18255b232a03ae3edc Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 15:08:08 +0530 Subject: [PATCH 09/18] feat: validated seed rules for golden formats (AcmeGW deliberately unseeded) Co-Authored-By: Claude Code --- deploy/docker-compose.yml | 11 +- deploy/seed.Dockerfile | 29 +++ services/pipeline/db.py | 6 +- services/pipeline/seed_rules.py | 206 ++++++++++++++++++ services/pipeline/seeds/cef_ciscoasa.json | 13 ++ services/pipeline/seeds/cef_fortigate.json | 13 ++ services/pipeline/seeds/cef_paloalto.json | 13 ++ services/pipeline/seeds/json.json | 15 ++ services/pipeline/seeds/syslog.json | 11 + services/pipeline/tests/test_batch_builder.py | 12 + services/pipeline/tests/test_seed_rules.py | 127 +++++++++++ services/pipeline/tests/test_worker.py | 85 ++++++++ services/pipeline/worker.py | 8 +- 13 files changed, 546 insertions(+), 3 deletions(-) create mode 100644 deploy/seed.Dockerfile create mode 100644 services/pipeline/seed_rules.py create mode 100644 services/pipeline/seeds/cef_ciscoasa.json create mode 100644 services/pipeline/seeds/cef_fortigate.json create mode 100644 services/pipeline/seeds/cef_paloalto.json create mode 100644 services/pipeline/seeds/json.json create mode 100644 services/pipeline/seeds/syslog.json create mode 100644 services/pipeline/tests/test_seed_rules.py create mode 100644 services/pipeline/tests/test_worker.py diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 767ddd9..9bac225 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -11,6 +11,14 @@ services: build: { context: .., dockerfile: deploy/migrate.Dockerfile } environment: { ADMIN_DATABASE_URL: postgresql://postgres:postgres@postgres:5432/ulpf } depends_on: { postgres: { condition: service_healthy } } + seed: + build: { context: .., dockerfile: deploy/seed.Dockerfile } + # pipeline_role is SELECT-only on rules — seeding runs as the admin, + # after migrations (controller R18). Validated at load: a broken seed + # exits 1 and loads nothing; a re-run is a no-op (ON CONFLICT). + environment: { ADMIN_DATABASE_URL: postgresql://postgres:postgres@postgres:5432/ulpf } + restart: "no" + depends_on: { migrate: { condition: service_completed_successfully } } redpanda: image: redpandadata/redpanda:v24.1.7 command: [ redpanda, start, --overprovisioned, --smp, "1", --memory, 1G, @@ -34,7 +42,8 @@ services: environment: KAFKA_BOOTSTRAP: redpanda:9092 DATABASE_URL: postgresql://pipeline_role:pipeline_dev@postgres:5432/ulpf - depends_on: { migrate: { condition: service_completed_successfully }, topics: { condition: service_completed_successfully } } + restart: unless-stopped + depends_on: { migrate: { condition: service_completed_successfully }, topics: { condition: service_completed_successfully }, seed: { condition: service_completed_successfully } } gateway: build: { context: .., dockerfile: services/gateway/Dockerfile } environment: diff --git a/deploy/seed.Dockerfile b/deploy/seed.Dockerfile new file mode 100644 index 0000000..019ca8e --- /dev/null +++ b/deploy/seed.Dockerfile @@ -0,0 +1,29 @@ +# deploy/seed.Dockerfile — one-shot seed-rule loader (compose service `seed`). +# pipeline_role is SELECT-only on rules by design, so seeding connects as the +# admin via ADMIN_DATABASE_URL and runs after migrations (controller R18). +# Build context is the repo root (compose build.context: ..; see /.dockerignore). +FROM python:3.12-slim + +WORKDIR /app + +# BOTH libs: validation needs ulpf_core.parsing, and its activation check +# FAILS CLOSED when ocsf_schema is unimportable — so the seed image installs +# exactly what the worker image installs. Deps (layer caching) next. +COPY libs/ulpf-core/pyproject.toml /app/libs/ulpf-core/pyproject.toml +COPY libs/ulpf-core/ulpf_core/ /app/libs/ulpf-core/ulpf_core/ +COPY libs/ocsf-schema/pyproject.toml /app/libs/ocsf-schema/pyproject.toml +COPY libs/ocsf-schema/ocsf_schema/ /app/libs/ocsf-schema/ocsf_schema/ +COPY services/pipeline/requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt \ + && pip install --no-cache-dir -e /app/libs/ulpf-core -e /app/libs/ocsf-schema + +COPY services/pipeline/__init__.py /app/pipeline/__init__.py +COPY services/pipeline/seed_rules.py /app/pipeline/seed_rules.py +COPY services/pipeline/seeds/ /app/pipeline/seeds/ + +# The golden corpora the seeds validate against at load time (fail closed: +# an unreadable corpus is an error, never a silent load). +COPY libs/ulpf-core/tests/data/golden/ /app/golden/ +ENV ULPF_GOLDEN_DIR=/app/golden + +CMD ["python", "-m", "pipeline.seed_rules"] diff --git a/services/pipeline/db.py b/services/pipeline/db.py index d1215a8..73389fd 100644 --- a/services/pipeline/db.py +++ b/services/pipeline/db.py @@ -97,6 +97,10 @@ def build_rows(msgs) -> tuple[list[RawRow], dict[int, PartitionMeta]]: topic, partition, offset = msg.topic(), msg.partition(), msg.offset() try: env = RawEnvelope(**json.loads(msg.value())) + # The timestamp parse stays inside this guard (R16): a well-formed + # envelope with a malformed received_at is a poison record to skip, + # not a worker-killing exception. + received_at = datetime.fromisoformat(env.received_at) except (ValueError, TypeError) as exc: log.warning("skipping %s[%d@%d]: undecodable/invalid envelope: %s", topic, partition, offset, exc) @@ -108,7 +112,7 @@ def build_rows(msgs) -> tuple[list[RawRow], dict[int, PartitionMeta]]: meta.last_offset = offset row = RawRow( raw_id=raw_id_for(topic, partition, offset), - received_at=datetime.fromisoformat(env.received_at), + received_at=received_at, source_id=env.source_id, transport=env.transport, format_hint=env.format_hint, diff --git a/services/pipeline/seed_rules.py b/services/pipeline/seed_rules.py new file mode 100644 index 0000000..88320ba --- /dev/null +++ b/services/pipeline/seed_rules.py @@ -0,0 +1,206 @@ +# services/pipeline/seed_rules.py — idempotent loader for the hand-authored +# seed rules (Task 8). Runnable one-shot: `python -m pipeline.seed_rules`. +# +# Order of operations is the safety story (spec §6.4): +# 1. load every seeds/*.json into a Rule; +# 2. validate ALL of them (validate_rule_output) against their OWN format's +# golden corpus lines — each seed JSON names its corpus, and the corpus +# lines are grouped by fingerprint_id(line) so a mixed corpus (the CEF +# file carries three vendors) only ever proves a seed against its own +# vendor (controller R8: ids come from the function, never the plan); +# 3. only if every seed passes, INSERT them — ON CONFLICT (fingerprint_id, +# version) DO NOTHING, status='active', provenance='human', +# created_by='seed', activated_at=now(). +# A broken seed therefore exits 1 naming the rule and loads NOTHING, and a +# re-run over an already-seeded DB inserts nothing and exits 0 (idempotent). +# +# The connection is the ADMIN url (ADMIN_DATABASE_URL, same env the migrate +# service uses): pipeline_role is deliberately SELECT-only on rules, so the +# one-shot seed service cannot run as the pipeline (controller R18). psycopg +# is imported lazily so the validation half is unit-testable without a DB. +from __future__ import annotations + +import json +import logging +import os +import sys +from dataclasses import dataclass +from pathlib import Path + +from ulpf_core.fingerprint import fingerprint_id +from ulpf_core.models import Mapping, Rule +from ulpf_core.parsing import validate_rule_output + +log = logging.getLogger("pipeline.seed_rules") + +SEEDS_DIR = Path(__file__).resolve().parent / "seeds" + +SEED_VERSION = 1 +SEED_PROVENANCE = "human" +SEED_CREATED_BY = "seed" + +# One statement, every seed: the (fingerprint_id, version) unique constraint +# makes a re-run a no-op, and the column set is exactly the R18 contract. +_INSERT_SQL = ( + "INSERT INTO rules (fingerprint_id, version, pattern, mappings, " + "status, provenance, created_by, activated_at) " + "VALUES (%s, %s, %s, %s, 'active', 'human', 'seed', now()) " + "ON CONFLICT (fingerprint_id, version) DO NOTHING" +) + + +class SeedError(Exception): + """A seed file could not be loaded — the runner must exit 1, never load.""" + + +@dataclass(frozen=True) +class Seed: + """One seed rule plus the golden corpus it must prove itself against.""" + + rule: Rule + corpus: str + + +def golden_dir() -> Path: + """Where the golden corpora live: ULPF_GOLDEN_DIR if set (the seed + container), else the repo checkout's libs/ulpf-core test data.""" + env = os.environ.get("ULPF_GOLDEN_DIR") + if env: + return Path(env) + # services/pipeline/seed_rules.py -> parents[2] is the repo root + return Path(__file__).resolve().parents[2] / "libs" / "ulpf-core" / "tests" / "data" / "golden" + + +def corpus_lines(corpus: str) -> list[str]: + """Non-blank lines of one golden corpus file.""" + path = golden_dir() / corpus + try: + text = path.read_text(encoding="utf-8") + except OSError as exc: + raise SeedError(f"corpus {corpus!r} unreadable at {path}: {exc}") from exc + return [line.strip() for line in text.splitlines() if line.strip()] + + +def load_seed(path: Path) -> Seed: + """One seed JSON -> Seed; any malformation is a SeedError naming the file.""" + try: + data = json.loads(path.read_text(encoding="utf-8")) + rule = Rule( + fingerprint_id=data["fingerprint_id"], + version=SEED_VERSION, + pattern=data["pattern"], + mappings=[Mapping(**m) for m in data["mappings"]], + provenance=SEED_PROVENANCE, + ) + corpus = data["corpus"] + except Exception as exc: # bad JSON, missing key, model violation — all fatal + raise SeedError(f"{path.name}: {exc}") from exc + if not corpus_lines(corpus): + raise SeedError(f"{path.name}: corpus {corpus!r} is empty") + return Seed(rule=rule, corpus=corpus) + + +def discover_seeds() -> dict[str, Seed]: + """Every seeds/*.json keyed by fingerprint_id; duplicates are fatal.""" + if not SEEDS_DIR.is_dir(): + raise SeedError(f"seeds directory not found: {SEEDS_DIR}") + files = sorted(SEEDS_DIR.glob("*.json")) + if not files: + raise SeedError(f"no seed files in {SEEDS_DIR}") + seeds: dict[str, Seed] = {} + for path in files: + seed = load_seed(path) + if seed.rule.fingerprint_id in seeds: + raise SeedError( + f"{path.name}: duplicate fingerprint_id {seed.rule.fingerprint_id!r}" + ) + seeds[seed.rule.fingerprint_id] = seed + return seeds + + +def samples_for(seed: Seed) -> list[str]: + """The seed's OWN lines: corpus lines whose fingerprint_id equals the + seed's (R8 — a shared corpus, like the three-vendor CEF file, contributes + only the lines the fingerprint function itself assigns to this seed).""" + return [ + line for line in corpus_lines(seed.corpus) + if fingerprint_id(line) == seed.rule.fingerprint_id + ] + + +def validate_seeds(seeds: dict[str, Seed]) -> dict[str, list[str]]: + """Activation check per seed against its own corpus lines; only failures + come back, keyed by fingerprint_id (empty dict = all seeds may load).""" + problems: dict[str, list[str]] = {} + for fp_id, seed in sorted(seeds.items()): + samples = samples_for(seed) + if not samples: + problems[fp_id] = [( + f"no lines in {seed.corpus} fingerprint as {fp_id!r} " + "(id drifted from fingerprint_id()'s output — R8)" + )] + continue + errors = validate_rule_output(seed.rule, samples) + if errors: + problems[fp_id] = errors + return problems + + +def insert_seeds(conn, seeds: dict[str, Seed]) -> int: + """Insert every seed idempotently in ONE transaction; returns rows added.""" + from psycopg.types.json import Json # deferred: keeps psycopg off the test path + + inserted = 0 + with conn.cursor() as cur: + for fp_id, seed in sorted(seeds.items()): + rule = seed.rule + cur.execute(_INSERT_SQL, ( + rule.fingerprint_id, + rule.version, + rule.pattern, + Json([m.model_dump() for m in rule.mappings]), + )) + inserted += cur.rowcount + conn.commit() + return inserted + + +def seed_database(conn) -> int: + """Load, validate, insert. Returns the process exit code: 0 seeded (or all + already present), 1 a seed is broken — nothing inserted, failure named.""" + try: + seeds = discover_seeds() + except SeedError as exc: + log.error("seed load failed, nothing inserted: %s", exc) + return 1 + + problems = validate_seeds(seeds) + if problems: + for fp_id, errors in problems.items(): + log.error("seed %s failed activation validation: %s", fp_id, "; ".join(errors)) + log.error("seed run aborted: %d of %d seed(s) broken, nothing inserted", + len(problems), len(seeds)) + return 1 + + inserted = insert_seeds(conn, seeds) + log.info("seed complete: %d inserted, %d already present", inserted, len(seeds) - inserted) + return 0 + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s" + ) + admin_url = os.environ.get("ADMIN_DATABASE_URL") + if not admin_url: + print("error: ADMIN_DATABASE_URL is not set", file=sys.stderr) + return 2 + + import psycopg # deferred: validation half stays importable without psycopg + + with psycopg.connect(admin_url) as conn: + return seed_database(conn) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/services/pipeline/seeds/cef_ciscoasa.json b/services/pipeline/seeds/cef_ciscoasa.json new file mode 100644 index 0000000..42ed54e --- /dev/null +++ b/services/pipeline/seeds/cef_ciscoasa.json @@ -0,0 +1,13 @@ +{ + "fingerprint_id": "cef_ciscoasa", + "pattern": "^.*\\|(?P.*)$", + "mappings": [ + {"source_field": "src", "ocsf_path": "src_endpoint.ip"}, + {"source_field": "spt", "ocsf_path": "src_endpoint.port"}, + {"source_field": "dst", "ocsf_path": "dst_endpoint.ip"}, + {"source_field": "dpt", "ocsf_path": "dst_endpoint.port"}, + {"source_field": "act", "ocsf_path": "action"}, + {"source_field": "msg", "ocsf_path": "message"} + ], + "corpus": "raw_logs_cef.txt" +} diff --git a/services/pipeline/seeds/cef_fortigate.json b/services/pipeline/seeds/cef_fortigate.json new file mode 100644 index 0000000..206851b --- /dev/null +++ b/services/pipeline/seeds/cef_fortigate.json @@ -0,0 +1,13 @@ +{ + "fingerprint_id": "cef_fortigate", + "pattern": "^.*\\|(?P.*)$", + "mappings": [ + {"source_field": "src", "ocsf_path": "src_endpoint.ip"}, + {"source_field": "spt", "ocsf_path": "src_endpoint.port"}, + {"source_field": "dst", "ocsf_path": "dst_endpoint.ip"}, + {"source_field": "dpt", "ocsf_path": "dst_endpoint.port"}, + {"source_field": "act", "ocsf_path": "action"}, + {"source_field": "msg", "ocsf_path": "message"} + ], + "corpus": "raw_logs_cef.txt" +} diff --git a/services/pipeline/seeds/cef_paloalto.json b/services/pipeline/seeds/cef_paloalto.json new file mode 100644 index 0000000..f83c17d --- /dev/null +++ b/services/pipeline/seeds/cef_paloalto.json @@ -0,0 +1,13 @@ +{ + "fingerprint_id": "cef_paloalto", + "pattern": "^.*\\|(?P.*)$", + "mappings": [ + {"source_field": "src", "ocsf_path": "src_endpoint.ip"}, + {"source_field": "spt", "ocsf_path": "src_endpoint.port"}, + {"source_field": "dst", "ocsf_path": "dst_endpoint.ip"}, + {"source_field": "dpt", "ocsf_path": "dst_endpoint.port"}, + {"source_field": "act", "ocsf_path": "action"}, + {"source_field": "msg", "ocsf_path": "message"} + ], + "corpus": "raw_logs_cef.txt" +} diff --git a/services/pipeline/seeds/json.json b/services/pipeline/seeds/json.json new file mode 100644 index 0000000..f1a50d2 --- /dev/null +++ b/services/pipeline/seeds/json.json @@ -0,0 +1,15 @@ +{ + "fingerprint_id": "json", + "pattern": "__JSON__", + "mappings": [ + {"source_field": "src_ip", "ocsf_path": "src_endpoint.ip"}, + {"source_field": "src_port", "ocsf_path": "src_endpoint.port"}, + {"source_field": "dest_ip", "ocsf_path": "dst_endpoint.ip"}, + {"source_field": "dest_port", "ocsf_path": "dst_endpoint.port"}, + {"source_field": "timestamp", "ocsf_path": "time"}, + {"source_field": "alert.severity", "ocsf_path": "severity_id"}, + {"source_field": "event_type", "ocsf_path": "action"}, + {"source_field": "alert.signature", "ocsf_path": "message"} + ], + "corpus": "raw_logs_json.txt" +} diff --git a/services/pipeline/seeds/syslog.json b/services/pipeline/seeds/syslog.json new file mode 100644 index 0000000..1ec40cc --- /dev/null +++ b/services/pipeline/seeds/syslog.json @@ -0,0 +1,11 @@ +{ + "fingerprint_id": "syslog", + "pattern": "^.*?IPTABLES-\\w+:\\s*(?P.*)$", + "mappings": [ + {"source_field": "SRC", "ocsf_path": "src_endpoint.ip"}, + {"source_field": "SPT", "ocsf_path": "src_endpoint.port"}, + {"source_field": "DST", "ocsf_path": "dst_endpoint.ip"}, + {"source_field": "DPT", "ocsf_path": "dst_endpoint.port"} + ], + "corpus": "raw_logs_syslog.txt" +} diff --git a/services/pipeline/tests/test_batch_builder.py b/services/pipeline/tests/test_batch_builder.py index 1869378..6c4499b 100644 --- a/services/pipeline/tests/test_batch_builder.py +++ b/services/pipeline/tests/test_batch_builder.py @@ -100,3 +100,15 @@ def test_oversized_line_skipped(): big = FakeMsg(0, 5, envelope(raw="x" * 70000)) rows, parts = build_rows([big]) assert rows == [] and parts == {} + + +def test_poison_received_at_skipped_not_fatal(): + # R16a: the received_at timestamp parse sits inside the envelope guard — + # a well-formed envelope carrying a malformed timestamp is just another + # poison record to skip, never a worker-killing exception. + bad = FakeMsg(0, 1, envelope(received_at="not-a-timestamp")) + good = FakeMsg(0, 2, envelope()) + rows, parts = build_rows([bad, good]) + assert len(rows) == 1 + assert rows[0].raw_id == raw_id_for(TOPIC, 0, 2) + assert parts[0].first_offset == 2 diff --git a/services/pipeline/tests/test_seed_rules.py b/services/pipeline/tests/test_seed_rules.py new file mode 100644 index 0000000..f49f2b7 --- /dev/null +++ b/services/pipeline/tests/test_seed_rules.py @@ -0,0 +1,127 @@ +# services/pipeline/tests/test_seed_rules.py — the activation gate for the +# hand-authored seed rules (spec §6.4, Task 8). +# +# Contract: every seed must validate against its OWN format's golden lines +# (the CEF corpus mixes three vendors — fingerprint_id picks each seed's own +# vendor lines, controller R8) before it may ever load; the unknown formats +# (auto_*: AcmeGW, Zenwall) must stay unseeded so they arrive 'unparsed' and +# remain M2's onboarding heroes. The runner is also pinned here: a broken seed +# exits without inserting anything, and inserts always carry the ON CONFLICT +# idempotency guard. +import pytest +from ulpf_core.fingerprint import fingerprint_id +from ulpf_core.parsing import OCSF_TARGETS, validate_rule_output + +from pipeline import seed_rules + +EXPECTED_SEED_IDS = {"cef_ciscoasa", "cef_fortigate", "cef_paloalto", "json", "syslog"} + + +def test_discovers_exactly_five_golden_seeds(): + seeds = seed_rules.discover_seeds() + assert set(seeds) == EXPECTED_SEED_IDS + for seed in seeds.values(): + assert seed.rule.version == 1 + assert seed.rule.provenance == "human" + + +@pytest.mark.parametrize("fp_id", sorted(EXPECTED_SEED_IDS)) +def test_seed_validates_against_its_own_corpus_lines(fp_id): + seed = seed_rules.discover_seeds()[fp_id] + samples = seed_rules.samples_for(seed) + assert samples, ( + f"no {fp_id} lines in {seed.corpus}: the corpus or fingerprint_id " + "drifted from the function's output (R8) — never seed a guessed id" + ) + assert validate_rule_output(seed.rule, samples) == [] + + +def test_unknown_formats_are_not_seeded(): + # AcmeGW + Zenwall fingerprint to auto_* shape ids: no seed may claim them. + seeded = set(seed_rules.discover_seeds()) + for corpus in ("raw_logs_acmegw.txt", "raw_logs_zenwall.txt"): + lines = seed_rules.corpus_lines(corpus) + assert lines + ids = {fingerprint_id(line) for line in lines} + assert ids + assert all(fid.startswith("auto_") for fid in ids) + assert not (ids & seeded), f"{corpus} must stay unseeded (M2 onboarding hero)" + + +def test_syslog_seed_leaves_flags_unmapped(): + # The legacy FLAGS->action mapping was wrong (action is a verdict, not a + # TCP flag) — the seed maps SRC/DST/SPT/DPT only. + rule = seed_rules.discover_seeds()["syslog"].rule + sources = {m.source_field for m in rule.mappings} + assert {"SRC", "DST", "SPT", "DPT"} <= sources + assert "FLAGS" not in sources + + +def test_all_mappings_within_ocsf_allow_list(): + # With no samples, validate_rule_output still checks pattern caps, compile + # and the mapping allow-list — so an empty-sample pass pins the allow-list. + for seed in seed_rules.discover_seeds().values(): + assert validate_rule_output(seed.rule, []) == [] + for mapping in seed.rule.mappings: + assert mapping.ocsf_path in OCSF_TARGETS + + +class RecordingConn: + """Fake psycopg connection: records execute() calls, never touches a DB.""" + + def __init__(self): + self.executed = [] + + def cursor(self): + return self + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params=None): + self.executed.append((sql, params)) + + rowcount = 1 + + def commit(self): + pass + + +def test_seed_database_inserts_with_idempotency_guard(): + conn = RecordingConn() + assert seed_rules.seed_database(conn) == 0 + inserts = [sql for sql, _ in conn.executed] + assert len(inserts) == len(EXPECTED_SEED_IDS) + for sql in inserts: + assert "ON CONFLICT (fingerprint_id, version) DO NOTHING" in sql + assert "status" in sql and "'active'" in sql + assert "'human'" in sql and "'seed'" in sql + # mappings ride as JSONB: the Json adapter wraps plain dicts (json-serializable), + # never pydantic models + params = next(p for _, p in conn.executed if p[0] == "cef_paloalto") + assert isinstance(params[3].obj, list) and isinstance(params[3].obj[0], dict) + + +def test_broken_seed_never_inserts(monkeypatch, caplog): + seeds = seed_rules.discover_seeds() + broken = dict(seeds) + from ulpf_core.models import Mapping, Rule + broken["cef_paloalto"] = seed_rules.Seed( + rule=Rule( + fingerprint_id="cef_paloalto", version=1, + pattern="(?P[", # does not compile + mappings=[Mapping(source_field="src", ocsf_path="src_endpoint.ip")], + provenance="human", + ), + corpus="raw_logs_cef.txt", + ) + monkeypatch.setattr(seed_rules, "discover_seeds", lambda: broken) + conn = RecordingConn() + + with caplog.at_level("ERROR", logger="pipeline.seed_rules"): + assert seed_rules.seed_database(conn) == 1 + assert conn.executed == [] # nothing loaded — not even the healthy seeds + assert "cef_paloalto" in caplog.text # the failure names the broken rule diff --git a/services/pipeline/tests/test_worker.py b/services/pipeline/tests/test_worker.py new file mode 100644 index 0000000..eb81ea8 --- /dev/null +++ b/services/pipeline/tests/test_worker.py @@ -0,0 +1,85 @@ +# services/pipeline/tests/test_worker.py — run-loop resilience (controller R16b). +# +# A batch that blows up anywhere (here: the unguarded rules SELECT surfacing a +# transient DB drop) must degrade to retry-next-batch: run() logs and keeps +# consuming — never process death, never an offset commit for the failed batch +# (§10: Postgres down -> workers stop committing offsets -> replay on recovery). +# No Kafka, no real DB: the conn is a stub whose every cursor() raises. +import pytest + +from pipeline.worker import PipelineWorker + + +class InterruptingConsumer: + """First consume() returns one batch; the second raises to stop the loop + (the same way main() shuts down — KeyboardInterrupt).""" + + def __init__(self, batch): + self._batch = batch + self.calls = 0 + + def subscribe(self, topic): + pass + + def consume(self, num_messages, timeout): + self.calls += 1 + if self.calls == 1: + return self._batch + raise KeyboardInterrupt + + def commit(self): + raise AssertionError("offsets must never be committed for a failed batch") + + def close(self): + pass + + +class BrokenConn: + """Stands in for a transiently-down Postgres: every cursor() raises.""" + + def rollback(self): + pass + + def cursor(self): + raise RuntimeError("db down") + + +class SilentProducer: + def produce(self, *args, **kwargs): + pass + + def flush(self, timeout=10.0): + return 0 + + +class FakeMsg: + def __init__(self, partition, offset, value: bytes): + self._p, self._o, self._v = partition, offset, value + + def topic(self): + return "raw.logs" + + def partition(self): + return self._p + + def offset(self): + return self._o + + def error(self): + return None + + def value(self): + return self._v + + +def test_run_survives_batch_crash_and_keeps_consuming(caplog): + # Undecodable value -> build_rows skips it -> persist_batch is a no-op -> + # the unguarded rules SELECT (load_active_rules) hits the broken conn and + # raises out of process_batch. run() must absorb that, log it, and come + # back around for the next batch (R16b: degrade to retry, not death). + msgs = [FakeMsg(0, 7, b"not-json")] + worker = PipelineWorker(InterruptingConsumer(msgs), SilentProducer(), BrokenConn()) + with caplog.at_level("ERROR", logger="pipeline.worker"), pytest.raises(KeyboardInterrupt): + worker.run() # must reach consume #2, i.e. survive batch 1 + assert worker.consumer.calls == 2 + assert "offsets NOT committed" in caplog.text diff --git a/services/pipeline/worker.py b/services/pipeline/worker.py index 939871b..5c692bc 100644 --- a/services/pipeline/worker.py +++ b/services/pipeline/worker.py @@ -111,7 +111,13 @@ def run(self) -> None: continue data.append(msg) if data: - self.process_batch(data) + # The WHOLE iteration is guarded (R16): a transient DB/Kafka + # drop anywhere in the batch must degrade to retry-next-batch + # (offsets stay uncommitted -> redelivery), not process death. + try: + self.process_batch(data) + except Exception: + log.exception("batch failed; offsets NOT committed; batch redelivered") def process_batch(self, msgs) -> bool: """One full batch; returns True iff offsets were committed. From 0993aab706e4dca7a4042d0f6589e1fff26b3344 Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 15:27:49 +0530 Subject: [PATCH 10/18] =?UTF-8?q?feat:=20gateway=20API=20=E2=80=94=20stats?= =?UTF-8?q?,=20current-view=20events,=20raw=20trace,=20SSE,=20chain=20head?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Code --- pyproject.toml | 2 +- services/gateway/Dockerfile | 17 ++ services/gateway/__init__.py | 2 + services/gateway/app.py | 118 ++++++++++++ services/gateway/queries.py | 149 +++++++++++++++ services/gateway/requirements.txt | 4 + services/gateway/tests/__init__.py | 0 services/gateway/tests/test_gateway.py | 243 +++++++++++++++++++++++++ 8 files changed, 534 insertions(+), 1 deletion(-) create mode 100644 services/gateway/Dockerfile create mode 100644 services/gateway/__init__.py create mode 100644 services/gateway/app.py create mode 100644 services/gateway/queries.py create mode 100644 services/gateway/requirements.txt create mode 100644 services/gateway/tests/__init__.py create mode 100644 services/gateway/tests/test_gateway.py diff --git a/pyproject.toml b/pyproject.toml index 83fbce7..d44d6f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.pytest.ini_options] testpaths = ["libs", "services"] -pythonpath = [".", "services/collector", "services/pipeline"] +pythonpath = [".", "services/collector", "services/pipeline", "services/gateway"] [tool.ruff] line-length = 100 diff --git a/services/gateway/Dockerfile b/services/gateway/Dockerfile new file mode 100644 index 0000000..14d7ad5 --- /dev/null +++ b/services/gateway/Dockerfile @@ -0,0 +1,17 @@ +# services/gateway/Dockerfile — FastAPI gateway: stats, current-view events, +# raw trace, SSE, chain head. Read-only DB access as gateway_role (SELECT only). +# Build context is the repo root (compose build.context: ..; see /.dockerignore). +FROM python:3.12-slim + +WORKDIR /app + +# No ulpf-core/ocsf-schema dependency: the gateway speaks plain SQL via psycopg. +COPY services/gateway/requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +# /app contains gateway/ so `python -m gateway.app` imports cleanly. +COPY services/gateway/__init__.py /app/gateway/__init__.py +COPY services/gateway/app.py /app/gateway/app.py +COPY services/gateway/queries.py /app/gateway/queries.py + +CMD ["python", "-m", "gateway.app"] diff --git a/services/gateway/__init__.py b/services/gateway/__init__.py new file mode 100644 index 0000000..19cc745 --- /dev/null +++ b/services/gateway/__init__.py @@ -0,0 +1,2 @@ +# services/gateway — FastAPI read API for the dashboard: stats, current-view +# events, raw trace, SSE, chain head (M1 walking skeleton). diff --git a/services/gateway/app.py b/services/gateway/app.py new file mode 100644 index 0000000..96e0a9e --- /dev/null +++ b/services/gateway/app.py @@ -0,0 +1,118 @@ +# services/gateway/app.py — FastAPI gateway for the M1 dashboard (spec §15: +# the M1 API contract is frozen at M1 exit; Task 10 mirrors EventRow exactly). +# +# CORS is deliberately NOT enabled: the dashboard is same-origin through Caddy +# (reverse_proxy /api/* gateway:8000), so there is no cross-origin caller. +# +# Every DB touch runs in a worker thread (asyncio.to_thread) so psycopg's +# blocking I/O never stalls the event loop — including the SSE poll loop, which +# awaits asyncio.sleep(1) between polls and emits a `: keepalive` comment after +# 15s of silence. Streams are bounded via max_polls ONLY for tests; production +# runs unbounded and is cancelled by client disconnect. +import asyncio +import json +import logging +import time +import uuid +from datetime import datetime + +import uvicorn +from fastapi import FastAPI, HTTPException, Query +from fastapi.responses import StreamingResponse + +from gateway import queries + +log = logging.getLogger("gateway") + +HTTP_PORT = 8000 +SSE_POLL_SECONDS = 1.0 +SSE_HEARTBEAT_SECONDS = 15.0 + + +def _json_default(value): + """SSE bodies are hand-serialized: datetimes must come out ISO-8601 (with + the T separator), which str(datetime) would not guarantee.""" + if isinstance(value, datetime): + return value.isoformat() + raise TypeError(f"unserializable SSE value: {type(value).__name__}") + + +async def sse_events(queries_module, poll_seconds, heartbeat_seconds, max_polls): + """Yield one `data:` line per new current-view row, then poll. + + First poll (last_seen=None) snapshots the latest rows and emits them + oldest-first; later polls fetch rows strictly newer than the last emitted + parsed_at (brief: WHERE parsed_at > last_seen ... ORDER BY parsed_at). A + `: keepalive` comment goes out after heartbeat_seconds without output so + intermediaries never time the stream out. + """ + last_seen = None + last_activity = time.monotonic() + polls = 0 + while max_polls is None or polls < max_polls: + polls += 1 + if time.monotonic() - last_activity >= heartbeat_seconds: + yield ": keepalive\n\n" + last_activity = time.monotonic() + rows = await asyncio.to_thread(queries_module.poll_events, last_seen) + for row in rows: + last_seen = row["parsed_at"] + last_activity = time.monotonic() + yield f"data: {json.dumps(row, default=_json_default)}\n\n" + await asyncio.sleep(poll_seconds) + + +def build_app(queries_module=queries, poll_seconds=SSE_POLL_SECONDS, + heartbeat_seconds=SSE_HEARTBEAT_SECONDS, max_polls=None): + """FastAPI app factory; the queries module is injected (tests monkeypatch + gateway.queries attributes — routes resolve them at request time).""" + + app = FastAPI(title="ulpf-gateway", version="0.1.0") + + @app.get("/api/stats") + async def stats(): + return await asyncio.to_thread(queries_module.fetch_stats) + + @app.get("/api/events") + async def events( + status: str | None = None, + fingerprint: str | None = None, + limit: int = Query(default=queries.DEFAULT_LIMIT, ge=1, le=queries.MAX_LIMIT), + before: datetime | None = None, + ): + rows = await asyncio.to_thread( + queries_module.fetch_events, status, fingerprint, limit, before + ) + return {"events": rows} + + @app.get("/api/events/{event_id}/raw") + async def raw_trace(event_id: uuid.UUID): + trace = await asyncio.to_thread(queries_module.fetch_raw_trace, event_id) + if trace is None: + raise HTTPException(status_code=404, detail=f"event {event_id} not found") + return trace + + @app.get("/api/audit/chain/head") + async def chain_head(): + return {"heads": await asyncio.to_thread(queries_module.fetch_chain_heads)} + + @app.get("/api/stream/events") + async def stream_events(): + return StreamingResponse( + sse_events(queries_module, poll_seconds, heartbeat_seconds, max_polls), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + return app + + +def main(): + logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s" + ) + uvicorn.run(build_app(), host="0.0.0.0", port=HTTP_PORT, log_level="info") + + +if __name__ == "__main__": + main() diff --git a/services/gateway/queries.py b/services/gateway/queries.py new file mode 100644 index 0000000..be4ff6e --- /dev/null +++ b/services/gateway/queries.py @@ -0,0 +1,149 @@ +# services/gateway/queries.py — read-only SQL behind the gateway endpoints. +# +# gateway_role is SELECT-only (Task 5 DDL), and every value goes through %s +# placeholders — parameterized SQL only; nothing user-controlled is ever +# interpolated into a statement string. psycopg is imported lazily inside +# _connect() so unit tests (which monkeypatch these functions) need neither a +# DB nor the driver on the test path (mirrors services/pipeline/db.py). +# +# Controller R9: every query over normalized_events carries the current-view +# predicate `superseded_by_event_id IS NULL`. M1 never supersedes, so it always +# matches today — but it stays in the SQL so M2+ inherits correct semantics. +from __future__ import annotations + +import os +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import psycopg + +DEFAULT_LIMIT = 50 +MAX_LIMIT = 500 +SNAPSHOT_LIMIT = 100 # SSE initial snapshot: latest N rows, emitted oldest-first + +_STATUS_KEYS = ("parsed", "unparsed", "parse_error", "quarantined") + + +def _connect() -> psycopg.Connection: + """One connection per request; dict rows so queries return contract dicts.""" + import psycopg + from psycopg.rows import dict_row + + return psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row) + + +def clamp_limit(limit: int) -> int: + """Defensive LIMIT clamp: 1..MAX_LIMIT regardless of caller input.""" + return max(1, min(int(limit), MAX_LIMIT)) + + +def zero_fill_statuses(rows) -> dict[str, int]: + """by_status GROUP BY rows -> contract dict with all four keys, zero-filled.""" + out = {status: 0 for status in _STATUS_KEYS} + for status, count in rows: + out[status] = count + return out + + +def fetch_stats() -> dict: + """GET /api/stats body (contract: dlq_total is null + note in M1 — no rpk + broker-side DLQ counting exists in the walking skeleton).""" + with _connect() as conn, conn.cursor() as cur: + cur.execute("SELECT count(*) AS n FROM raw_events") + raw_total = cur.fetchone()["n"] + cur.execute( + "SELECT status, count(*) AS n FROM normalized_events " + "WHERE superseded_by_event_id IS NULL GROUP BY status" + ) + by_status = zero_fill_statuses(cur.fetchall()) + cur.execute( + "SELECT fingerprint_id, count(*) AS total, " + "count(*) FILTER (WHERE status = 'parsed') AS parsed " + "FROM normalized_events WHERE superseded_by_event_id IS NULL " + "GROUP BY fingerprint_id ORDER BY total DESC, fingerprint_id" + ) + by_fingerprint = cur.fetchall() + cur.execute( + "SELECT count(*) AS n FROM normalized_events " + "WHERE superseded_by_event_id IS NULL " + "AND parsed_at >= now() - interval '1 minute'" + ) + events_last_minute = cur.fetchone()["n"] + return { + "raw_total": raw_total, + "by_status": by_status, + "by_fingerprint": by_fingerprint, + "events_last_minute": events_last_minute, + "dlq_total": None, + "note": "dlq_total is not counted in M1 (needs broker-side inspection of " + "pipeline.dlq); always null for M1", + } + + +def fetch_events(status=None, fingerprint=None, limit=DEFAULT_LIMIT, before=None): + """GET /api/events rows: current view, newest first, EventRow columns only + (exactly the 7 contract keys — rule_id is deliberately not exposed).""" + limit = clamp_limit(limit) + with _connect() as conn, conn.cursor() as cur: + cur.execute( + "SELECT event_id, raw_id, fingerprint_id, rule_version, status, parsed_at, ocsf " + "FROM normalized_events " + "WHERE superseded_by_event_id IS NULL " + "AND (%s::text IS NULL OR status = %s) " + "AND (%s::text IS NULL OR fingerprint_id = %s) " + "AND (%s::timestamptz IS NULL OR parsed_at < %s) " + "ORDER BY parsed_at DESC LIMIT %s", + (status, status, fingerprint, fingerprint, before, before, limit), + ) + return cur.fetchall() + + +def fetch_raw_trace(event_id) -> dict | None: + """GET /api/events/{event_id}/raw body: the raw line behind a normalized + event, joined on the raw_events composite FK. None -> 404 upstream.""" + with _connect() as conn, conn.cursor() as cur: + cur.execute( + "SELECT r.raw_id, r.received_at, r.source_id, r.transport, " + "r.content_hash, r.raw_text " + "FROM raw_events r " + "JOIN normalized_events n " + " ON n.raw_id = r.raw_id AND n.raw_received_at = r.received_at " + "WHERE n.event_id = %s AND n.superseded_by_event_id IS NULL " + "LIMIT 1", + (event_id,), + ) + return cur.fetchone() + + +def poll_events(after=None, limit=SNAPSHOT_LIMIT): + """SSE poll: current-view rows strictly newer than `after` (R9 predicate in + both branches), chronological for ordered streaming. after=None snapshots + the latest rows (reversed to oldest-first).""" + limit = clamp_limit(limit) + with _connect() as conn, conn.cursor() as cur: + if after is None: + cur.execute( + "SELECT event_id, raw_id, fingerprint_id, rule_version, status, parsed_at, ocsf " + "FROM normalized_events WHERE superseded_by_event_id IS NULL " + "ORDER BY parsed_at DESC LIMIT %s", + (limit,), + ) + return list(reversed(cur.fetchall())) + cur.execute( + "SELECT event_id, raw_id, fingerprint_id, rule_version, status, parsed_at, ocsf " + "FROM normalized_events WHERE superseded_by_event_id IS NULL AND parsed_at > %s " + "ORDER BY parsed_at LIMIT %s", + (after, limit), + ) + return cur.fetchall() + + +def fetch_chain_heads() -> list[dict]: + """GET /api/audit/chain/head: the latest committed batch per Kafka partition + (the chain head of each per-partition hash chain).""" + with _connect() as conn, conn.cursor() as cur: + cur.execute( + "SELECT DISTINCT ON (partition_id) partition_id, batch_seq, merkle_root, count " + "FROM raw_batches ORDER BY partition_id, batch_seq DESC" + ) + return cur.fetchall() diff --git a/services/gateway/requirements.txt b/services/gateway/requirements.txt new file mode 100644 index 0000000..c20a40c --- /dev/null +++ b/services/gateway/requirements.txt @@ -0,0 +1,4 @@ +fastapi==0.111.0 +uvicorn[standard]==0.30.1 +psycopg[binary]==3.2.1 +httpx # tests (FastAPI TestClient), dev diff --git a/services/gateway/tests/__init__.py b/services/gateway/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/gateway/tests/test_gateway.py b/services/gateway/tests/test_gateway.py new file mode 100644 index 0000000..e0554a5 --- /dev/null +++ b/services/gateway/tests/test_gateway.py @@ -0,0 +1,243 @@ +# services/gateway/tests/test_gateway.py — gateway contract tests (M1 API contract, +# frozen at M1 exit; the Task 10 dashboard mirrors these shapes in web/src/types.ts). +# +# No DB in unit tests: every queries function is monkeypatched with fixed returns +# (controller R13 / brief Step 1). Real-SQL correctness rides on the Task 11/12 +# end-to-end smoke. SSE is exercised with a bounded stream (max_polls) so +# TestClient consumes it fully — no connection is held open. +import json +import uuid +from datetime import UTC, datetime + +import pytest + +# Exact key sets — Task 10's types.ts mirrors EventRow EXACTLY. +EVENT_ROW_KEYS = { + "event_id", "raw_id", "fingerprint_id", "rule_version", "status", "parsed_at", "ocsf", +} +RAW_TRACE_KEYS = {"raw_id", "received_at", "source_id", "transport", "content_hash", "raw_text"} +HEAD_KEYS = {"partition_id", "batch_seq", "merkle_root", "count"} + + +def event_row(**overrides) -> dict: + """One EventRow-shaped dict (contract keys only).""" + row = { + "event_id": str(uuid.uuid4()), + "raw_id": str(uuid.uuid4()), + "fingerprint_id": "fp_ssh_denied", + "rule_version": 3, + "status": "parsed", + "parsed_at": "2026-09-19T12:00:00+00:00", + "ocsf": {"class_uid": 1004, "activity_name": "Create"}, + } + row.update(overrides) + return row + + +@pytest.fixture +def queries(): + import gateway.queries as q + + return q + + +@pytest.fixture +def client(): + from fastapi.testclient import TestClient + + from gateway.app import build_app + + # Bounded SSE (poll_seconds=0, max_polls=2) so TestClient can drain the stream. + return TestClient(build_app(poll_seconds=0.0, heartbeat_seconds=0.0, max_polls=2)) + + +# ---------- GET /api/stats ---------- + +STATS = { + "raw_total": 42, + "by_status": {"parsed": 30, "unparsed": 9, "parse_error": 2, "quarantined": 1}, + "by_fingerprint": [{"fingerprint_id": "fp_ssh_denied", "total": 12, "parsed": 11}], + "events_last_minute": 7, + "dlq_total": None, + "note": "dlq_total is not counted in M1", +} + + +def test_stats_shape(client, queries, monkeypatch): + monkeypatch.setattr(queries, "fetch_stats", lambda: STATS) + r = client.get("/api/stats") + assert r.status_code == 200 + assert r.json() == STATS + assert r.json()["dlq_total"] is None # controller R3: null + note, not a number + assert isinstance(r.json()["note"], str) + + +# ---------- GET /api/events ---------- + +def test_events_shape(client, queries, monkeypatch): + rows = [event_row(), event_row(status="unparsed", rule_version=0, ocsf=None)] + seen = {} + + def fake(status=None, fingerprint=None, limit=50, before=None): + seen.update(status=status, fingerprint=fingerprint, limit=limit, before=before) + return rows + + monkeypatch.setattr(queries, "fetch_events", fake) + r = client.get("/api/events") + assert r.status_code == 200 + assert r.json() == {"events": rows} + assert set(r.json()["events"][0]) == EVENT_ROW_KEYS # exact keys, no extras + # defaults: no filters, limit 50 + assert seen == {"status": None, "fingerprint": None, "limit": 50, "before": None} + + +def test_events_filters_forwarded(client, queries, monkeypatch): + seen = {} + + def fake(status=None, fingerprint=None, limit=50, before=None): + seen.update(status=status, fingerprint=fingerprint, limit=limit, before=before) + return [] + + monkeypatch.setattr(queries, "fetch_events", fake) + r = client.get("/api/events", params={ + "status": "parsed", "fingerprint": "fp_x", "limit": 10, + "before": "2026-09-19T00:00:00Z", + }) + assert r.status_code == 200 + assert r.json() == {"events": []} + assert seen["status"] == "parsed" and seen["fingerprint"] == "fp_x" + assert seen["limit"] == 10 + assert seen["before"] == datetime(2026, 9, 19, tzinfo=UTC) + + +def test_events_limit_bounds_rejected(client): + # sane LIMIT: 1..500 accepted (default 50); nonsense bounds -> 422 FastAPI shape + assert client.get("/api/events", params={"limit": 501}).status_code == 422 + assert client.get("/api/events", params={"limit": 0}).status_code == 422 + + +# ---------- GET /api/events/{event_id}/raw ---------- + +def test_raw_trace_shape(client, queries, monkeypatch): + trace = { + "raw_id": str(uuid.uuid4()), + "received_at": "2026-09-19T11:59:59+00:00", + "source_id": "udp-10.1.2.3", + "transport": "syslog-udp", + "content_hash": "ab" * 32, + "raw_text": "Aug 27 14:32:07 fw01 kernel: X", + } + eid = uuid.uuid4() + seen = {} + + def fake(event_id): + seen["event_id"] = event_id + return trace + + monkeypatch.setattr(queries, "fetch_raw_trace", fake) + r = client.get(f"/api/events/{eid}/raw") + assert r.status_code == 200 + assert r.json() == trace + assert set(r.json()) == RAW_TRACE_KEYS # exact keys, no extras + assert seen["event_id"] == eid # route hands the query a real UUID + + +def test_raw_404(client, queries, monkeypatch): + monkeypatch.setattr(queries, "fetch_raw_trace", lambda event_id: None) + r = client.get(f"/api/events/{uuid.uuid4()}/raw") + assert r.status_code == 404 + body = r.json() + assert "detail" in body and isinstance(body["detail"], str) # FastAPI {"detail": ...} + + +def test_raw_malformed_id_422(client): + assert client.get("/api/events/not-a-uuid/raw").status_code == 422 + + +# ---------- GET /api/audit/chain/head ---------- + +def test_chain_head(client, queries, monkeypatch): + heads = [ + {"partition_id": 0, "batch_seq": 4, "merkle_root": "c" * 64, "count": 500}, + {"partition_id": 1, "batch_seq": 2, "merkle_root": "d" * 64, "count": 120}, + ] + monkeypatch.setattr(queries, "fetch_chain_heads", lambda: heads) + r = client.get("/api/audit/chain/head") + assert r.status_code == 200 + assert r.json() == {"heads": heads} + assert all(set(h) == HEAD_KEYS for h in r.json()["heads"]) + + +# ---------- GET /api/stream/events (SSE) ---------- + +def test_sse_headers(client, queries, monkeypatch): + monkeypatch.setattr(queries, "poll_events", lambda after, limit=None: [event_row()]) + r = client.get("/api/stream/events") + assert r.status_code == 200 + assert r.headers["content-type"].startswith("text/event-stream") + + +def test_sse_first_data_line(client, queries, monkeypatch): + r1 = event_row() + r2 = event_row(status="unparsed", rule_version=0, ocsf=None) + calls = [] + + def fake(after, limit=None): + calls.append(after) + return [r1, r2] if len(calls) == 1 else [] + + monkeypatch.setattr(queries, "poll_events", fake) + r = client.get("/api/stream/events") + data_lines = [ln for ln in r.text.splitlines() if ln.startswith("data: ")] + assert len(data_lines) == 2 # one data: line per new current-view row + assert json.loads(data_lines[0][len("data: "):]) == r1 + assert json.loads(data_lines[1][len("data: "):]) == r2 + # incremental poll: first call snapshots (after=None), then resumes strictly + # past the last emitted row's parsed_at + assert calls[0] is None + assert calls[1] == r2["parsed_at"] + + +def test_sse_keepalive_comment(client, queries, monkeypatch): + monkeypatch.setattr(queries, "poll_events", lambda after, limit=None: []) + r = client.get("/api/stream/events") + assert ": keepalive" in r.text # heartbeat comment emitted when idle + + +def test_sse_serializes_datetimes_iso(client, queries, monkeypatch): + row = event_row(parsed_at=datetime(2026, 9, 19, 12, 0, 0, tzinfo=UTC)) + monkeypatch.setattr(queries, "poll_events", lambda after, limit=None: [row]) + r = client.get("/api/stream/events") + data_lines = [ln for ln in r.text.splitlines() if ln.startswith("data: ")] + parsed = json.loads(data_lines[0][len("data: "):]) + assert parsed["parsed_at"] == "2026-09-19T12:00:00+00:00" # ISO-8601, not str() + + +# ---------- queries pure helpers (no DB) ---------- + +def test_zero_fill_statuses(queries): + assert queries.zero_fill_statuses([("parsed", 5), ("quarantined", 2)]) == { + "parsed": 5, "unparsed": 0, "parse_error": 0, "quarantined": 2, + } + assert queries.zero_fill_statuses([]) == { + "parsed": 0, "unparsed": 0, "parse_error": 0, "quarantined": 0, + } + + +def test_clamp_limit(queries): + assert queries.clamp_limit(-5) == 1 + assert queries.clamp_limit(10**9) == 500 + assert queries.clamp_limit(50) == 50 + assert queries.clamp_limit(500) == 500 + + +def test_r9_current_view_predicate_in_every_normalized_query(queries): + # Controller R9: EVERY query over normalized_events carries + # `superseded_by_event_id IS NULL` — fetch_stats (3 normalized queries), + # fetch_events, fetch_raw_trace (join), poll_events. fetch_chain_heads is + # deliberately absent (raw_batches has no supersede column). + import inspect + + for fn_name in ("fetch_stats", "fetch_events", "fetch_raw_trace", "poll_events"): + src = inspect.getsource(getattr(queries, fn_name)) + assert "superseded_by_event_id IS NULL" in src, f"{fn_name} missing R9 predicate" From 91b79e57edecfa4ca6edf42283dc0b1887a5b6d3 Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 15:39:02 +0530 Subject: [PATCH 11/18] fix: serialize UUIDs in SSE, correct by_status dict-row unpacking, cap by_fingerprint Review fixes: _json_default now serializes uuid.UUID (dict_row rows carry UUID objects; SSE died on the first real row), zero_fill_statuses consumes the {"status","n"} dict rows the query actually returns, by_fingerprint is capped at 500 (R20), limit=500 boundary accepted, and test fixtures rebuilt to mirror real psycopg dict_row shapes so this bug class gets caught here. Co-Authored-By: Claude Code --- services/gateway/app.py | 7 ++- services/gateway/queries.py | 10 +++-- services/gateway/tests/test_gateway.py | 59 +++++++++++++++++++------- 3 files changed, 55 insertions(+), 21 deletions(-) diff --git a/services/gateway/app.py b/services/gateway/app.py index 96e0a9e..73eb813 100644 --- a/services/gateway/app.py +++ b/services/gateway/app.py @@ -30,8 +30,11 @@ def _json_default(value): - """SSE bodies are hand-serialized: datetimes must come out ISO-8601 (with - the T separator), which str(datetime) would not guarantee.""" + """SSE bodies are hand-serialized: dict_row rows carry uuid.UUID for UUID + columns (psycopg3's default UUID loader) and datetime for timestamptz — + both must become JSON scalars (str / ISO-8601 with the T separator).""" + if isinstance(value, uuid.UUID): + return str(value) if isinstance(value, datetime): return value.isoformat() raise TypeError(f"unserializable SSE value: {type(value).__name__}") diff --git a/services/gateway/queries.py b/services/gateway/queries.py index be4ff6e..81af2d9 100644 --- a/services/gateway/queries.py +++ b/services/gateway/queries.py @@ -38,10 +38,11 @@ def clamp_limit(limit: int) -> int: def zero_fill_statuses(rows) -> dict[str, int]: - """by_status GROUP BY rows -> contract dict with all four keys, zero-filled.""" + """by_status GROUP BY rows (dict_row dicts: {"status": ..., "n": ...}) -> + contract dict with all four keys, zero-filled.""" out = {status: 0 for status in _STATUS_KEYS} - for status, count in rows: - out[status] = count + for row in rows: + out[row["status"]] = row["n"] return out @@ -60,7 +61,8 @@ def fetch_stats() -> dict: "SELECT fingerprint_id, count(*) AS total, " "count(*) FILTER (WHERE status = 'parsed') AS parsed " "FROM normalized_events WHERE superseded_by_event_id IS NULL " - "GROUP BY fingerprint_id ORDER BY total DESC, fingerprint_id" + "GROUP BY fingerprint_id ORDER BY total DESC, fingerprint_id " + "LIMIT 500" # controller R20: hard cap — unbounded once M2 onboarding mints fingerprints ) by_fingerprint = cur.fetchall() cur.execute( diff --git a/services/gateway/tests/test_gateway.py b/services/gateway/tests/test_gateway.py index e0554a5..a55372f 100644 --- a/services/gateway/tests/test_gateway.py +++ b/services/gateway/tests/test_gateway.py @@ -5,6 +5,12 @@ # (controller R13 / brief Step 1). Real-SQL correctness rides on the Task 11/12 # end-to-end smoke. SSE is exercised with a bounded stream (max_polls) so # TestClient consumes it fully — no connection is held open. +# +# Fixture shapes mirror real psycopg dict_row output EXACTLY: uuid.UUID objects +# for id columns (psycopg3 default UUID loader), real datetimes for timestamptz, +# dicts for JSONB, {"status": ..., "n": ...} dicts for GROUP BY rows. The +# jsonable() helper states what FastAPI/json.dumps put on the wire, so a +# serializer gap (e.g. an unhandled UUID) fails loudly here. import json import uuid from datetime import UTC, datetime @@ -20,20 +26,30 @@ def event_row(**overrides) -> dict: - """One EventRow-shaped dict (contract keys only).""" + """One EventRow as psycopg dict_row returns it: UUID objects, real datetime.""" row = { - "event_id": str(uuid.uuid4()), - "raw_id": str(uuid.uuid4()), + "event_id": uuid.uuid4(), + "raw_id": uuid.uuid4(), "fingerprint_id": "fp_ssh_denied", "rule_version": 3, "status": "parsed", - "parsed_at": "2026-09-19T12:00:00+00:00", + "parsed_at": datetime(2026, 9, 19, 12, 0, 0, tzinfo=UTC), "ocsf": {"class_uid": 1004, "activity_name": "Create"}, } row.update(overrides) return row +def jsonable(row: dict) -> dict: + """The row as it appears on the wire: UUID -> str, datetime -> ISO-8601 + (what FastAPI's encoder and the SSE _json_default must both produce).""" + return { + key: value.isoformat() if isinstance(value, datetime) else + str(value) if isinstance(value, uuid.UUID) else value + for key, value in row.items() + } + + @pytest.fixture def queries(): import gateway.queries as q @@ -85,7 +101,7 @@ def fake(status=None, fingerprint=None, limit=50, before=None): monkeypatch.setattr(queries, "fetch_events", fake) r = client.get("/api/events") assert r.status_code == 200 - assert r.json() == {"events": rows} + assert r.json() == {"events": [jsonable(row) for row in rows]} assert set(r.json()["events"][0]) == EVENT_ROW_KEYS # exact keys, no extras # defaults: no filters, limit 50 assert seen == {"status": None, "fingerprint": None, "limit": 50, "before": None} @@ -116,12 +132,19 @@ def test_events_limit_bounds_rejected(client): assert client.get("/api/events", params={"limit": 0}).status_code == 422 +def test_events_limit_upper_boundary_accepted(client, queries, monkeypatch): + monkeypatch.setattr(queries, "fetch_events", lambda *a, **k: []) + r = client.get("/api/events", params={"limit": 500}) # the le= boundary itself + assert r.status_code == 200 + assert r.json() == {"events": []} + + # ---------- GET /api/events/{event_id}/raw ---------- def test_raw_trace_shape(client, queries, monkeypatch): trace = { - "raw_id": str(uuid.uuid4()), - "received_at": "2026-09-19T11:59:59+00:00", + "raw_id": uuid.uuid4(), + "received_at": datetime(2026, 9, 19, 11, 59, 59, tzinfo=UTC), "source_id": "udp-10.1.2.3", "transport": "syslog-udp", "content_hash": "ab" * 32, @@ -137,7 +160,7 @@ def fake(event_id): monkeypatch.setattr(queries, "fetch_raw_trace", fake) r = client.get(f"/api/events/{eid}/raw") assert r.status_code == 200 - assert r.json() == trace + assert r.json() == jsonable(trace) assert set(r.json()) == RAW_TRACE_KEYS # exact keys, no extras assert seen["event_id"] == eid # route hands the query a real UUID @@ -169,6 +192,8 @@ def test_chain_head(client, queries, monkeypatch): # ---------- GET /api/stream/events (SSE) ---------- +# SSE rows keep their real dict_row shape (uuid.UUID ids, datetime parsed_at) — +# the endpoint's own serializer must cope, so any gap dies here, not in the demo. def test_sse_headers(client, queries, monkeypatch): monkeypatch.setattr(queries, "poll_events", lambda after, limit=None: [event_row()]) @@ -190,8 +215,8 @@ def fake(after, limit=None): r = client.get("/api/stream/events") data_lines = [ln for ln in r.text.splitlines() if ln.startswith("data: ")] assert len(data_lines) == 2 # one data: line per new current-view row - assert json.loads(data_lines[0][len("data: "):]) == r1 - assert json.loads(data_lines[1][len("data: "):]) == r2 + assert json.loads(data_lines[0][len("data: "):]) == jsonable(r1) + assert json.loads(data_lines[1][len("data: "):]) == jsonable(r2) # incremental poll: first call snapshots (after=None), then resumes strictly # past the last emitted row's parsed_at assert calls[0] is None @@ -204,19 +229,23 @@ def test_sse_keepalive_comment(client, queries, monkeypatch): assert ": keepalive" in r.text # heartbeat comment emitted when idle -def test_sse_serializes_datetimes_iso(client, queries, monkeypatch): - row = event_row(parsed_at=datetime(2026, 9, 19, 12, 0, 0, tzinfo=UTC)) +def test_sse_serializes_uuids_and_datetimes(client, queries, monkeypatch): + row = event_row(parsed_at=datetime(2026, 9, 19, 12, 30, 45, tzinfo=UTC)) monkeypatch.setattr(queries, "poll_events", lambda after, limit=None: [row]) r = client.get("/api/stream/events") data_lines = [ln for ln in r.text.splitlines() if ln.startswith("data: ")] parsed = json.loads(data_lines[0][len("data: "):]) - assert parsed["parsed_at"] == "2026-09-19T12:00:00+00:00" # ISO-8601, not str() + assert parsed["event_id"] == str(row["event_id"]) # UUID -> str, not a crash + assert parsed["raw_id"] == str(row["raw_id"]) + assert parsed["parsed_at"] == "2026-09-19T12:30:45+00:00" # ISO-8601, not str() -# ---------- queries pure helpers (no DB) ---------- +# ---------- queries pure helpers (no DB, real dict_row shapes) ---------- def test_zero_fill_statuses(queries): - assert queries.zero_fill_statuses([("parsed", 5), ("quarantined", 2)]) == { + assert queries.zero_fill_statuses( + [{"status": "parsed", "n": 5}, {"status": "quarantined", "n": 2}] + ) == { "parsed": 5, "unparsed": 0, "parse_error": 0, "quarantined": 2, } assert queries.zero_fill_statuses([]) == { From 1dccf73367a8700406abbf873133e83f14cd1441 Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 16:10:05 +0530 Subject: [PATCH 12/18] =?UTF-8?q?feat:=20web=20Overview=20=E2=80=94=20SSE?= =?UTF-8?q?=20live=20feed,=20stats,=20raw=20traceability=20drawer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5 --- .dockerignore | 1 + web/.gitignore | 24 + web/.gitkeep | 0 web/.oxlintrc.json | 8 + web/Caddyfile | 23 + web/Dockerfile | 19 + web/README.md | 25 + web/index.html | 15 + web/package-lock.json | 2313 ++++++++++++++++++++++++++++++++ web/package.json | 30 + web/public/favicon.svg | 1 + web/src/App.css | 244 ++++ web/src/App.tsx | 224 ++++ web/src/api.ts | 41 + web/src/main.tsx | 10 + web/src/types.ts | 73 + web/src/useEventStream.test.ts | 134 ++ web/src/useEventStream.ts | 68 + web/tsconfig.app.json | 26 + web/tsconfig.json | 7 + web/tsconfig.node.json | 23 + web/vite.config.ts | 19 + 22 files changed, 3328 insertions(+) create mode 100644 web/.gitignore delete mode 100644 web/.gitkeep create mode 100644 web/.oxlintrc.json create mode 100644 web/Caddyfile create mode 100644 web/Dockerfile create mode 100644 web/README.md create mode 100644 web/index.html create mode 100644 web/package-lock.json create mode 100644 web/package.json create mode 100644 web/public/favicon.svg create mode 100644 web/src/App.css create mode 100644 web/src/App.tsx create mode 100644 web/src/api.ts create mode 100644 web/src/main.tsx create mode 100644 web/src/types.ts create mode 100644 web/src/useEventStream.test.ts create mode 100644 web/src/useEventStream.ts create mode 100644 web/tsconfig.app.json create mode 100644 web/tsconfig.json create mode 100644 web/tsconfig.node.json create mode 100644 web/vite.config.ts diff --git a/.dockerignore b/.dockerignore index fe95f25..e82dbad 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,4 +8,5 @@ legacy-demo **/*.egg-info .venv node_modules +**/node_modules .pytest_cache diff --git a/web/.gitignore b/web/.gitignore new file mode 100644 index 0000000..a547bf3 --- /dev/null +++ b/web/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/web/.gitkeep b/web/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/web/.oxlintrc.json b/web/.oxlintrc.json new file mode 100644 index 0000000..6fa991d --- /dev/null +++ b/web/.oxlintrc.json @@ -0,0 +1,8 @@ +{ + "$schema": "./node_modules/oxlint/configuration_schema.json", + "plugins": ["react", "typescript", "oxc"], + "rules": { + "react/rules-of-hooks": "error", + "react/only-export-components": ["warn", { "allowConstantExport": true }] + } +} diff --git a/web/Caddyfile b/web/Caddyfile new file mode 100644 index 0000000..55dc8f9 --- /dev/null +++ b/web/Caddyfile @@ -0,0 +1,23 @@ +# web/Caddyfile — serves the built SPA and reverse-proxies /api/* to the +# gateway. Same-origin only: the gateway ships no CORS, and this is the only +# front door in production. +# +# NOTE (deviation from the plan's literal snippet): try_files/file_server and +# reverse_proxy cannot share one scope here — Caddy's canonical directive +# order runs try_files BEFORE reverse_proxy, so an unmatched /api/stats was +# rewritten to /index.html before the /api/* matcher was evaluated and the +# API silently served the SPA (verified empirically with caddy:2-alpine: +# GET /api/stats returned 200 text/html). Two `handle` blocks enforce the +# routing explicitly; verified: / -> SPA, /api/* -> proxy (502 when gateway +# is down, not index.html). Caddy detects text/event-stream and streams SSE +# unbuffered. +:3000 { + root * /srv + handle /api/* { + reverse_proxy gateway:8000 + } + handle { + try_files {path} /index.html + file_server + } +} diff --git a/web/Dockerfile b/web/Dockerfile new file mode 100644 index 0000000..2ef1974 --- /dev/null +++ b/web/Dockerfile @@ -0,0 +1,19 @@ +# web/Dockerfile — build the Overview SPA, then serve it (plus /api/* via the +# bundled Caddyfile) with caddy:2-alpine. Compose builds this with +# context: .. (repo root), so every COPY path is prefixed web/. +# Air-gap: build inputs are local; the served app makes no external requests. +FROM node:22 AS build +WORKDIR /app + +# Lockfile first for layer caching; ci is exact, i is the fallback. +COPY web/package.json web/package-lock.json ./ +RUN npm ci || npm i + +# Sources (root .dockerignore keeps host node_modules out of the context). +COPY web/ ./ +RUN npm run build + +FROM caddy:2-alpine +COPY web/Caddyfile /etc/caddy/Caddyfile +COPY --from=build /app/dist /srv +EXPOSE 3000 diff --git a/web/README.md b/web/README.md new file mode 100644 index 0000000..25b91df --- /dev/null +++ b/web/README.md @@ -0,0 +1,25 @@ +# web/ — ULPF Overview dashboard (M1) + +React + Vite SPA consuming the frozen M1 gateway API (`services/gateway`) +same-origin under `/api`: live SSE feed, stats cards, raw-traceability drawer. +Air-gapped: system fonts, no CDN, no external requests. + +## Run + +```sh +npm i +npm run dev # Vite dev server; /api proxied to http://localhost:8000 +``` + +## Verify + +```sh +npm run test # vitest (useEventStream hook tests) +npm run build # tsc -b && vite build +``` + +## Deploy + +`deploy/docker-compose.yml` builds `web/Dockerfile` (node:22 build stage → +caddy:2-alpine serving `dist/` + `Caddyfile`, which reverse-proxies `/api/*` +to `gateway:8000`) on port 3000. diff --git a/web/index.html b/web/index.html new file mode 100644 index 0000000..e2165d7 --- /dev/null +++ b/web/index.html @@ -0,0 +1,15 @@ + + + + + + + + ULPF Overview + + +
+ + + diff --git a/web/package-lock.json b/web/package-lock.json new file mode 100644 index 0000000..2776557 --- /dev/null +++ b/web/package-lock.json @@ -0,0 +1,2313 @@ +{ + "name": "web", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "web", + "version": "0.0.0", + "dependencies": { + "react": "^19.2.8", + "react-dom": "^19.2.8" + }, + "devDependencies": { + "@testing-library/dom": "^10.4.2", + "@testing-library/react": "^16.3.3", + "@types/node": "^24.13.3", + "@types/react": "^19.2.18", + "@types/react-dom": "^19.2.7", + "@vitejs/plugin-react": "^6.1.1", + "jsdom": "^29.1.1", + "oxlint": "^1.81.0", + "typescript": "~6.0.2", + "vite": "^8.3.0", + "vitest": "^5.0.1" + } + }, + "node_modules/@asamuzakjp/css-color": { + "version": "5.1.11", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.11.tgz", + "integrity": "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@asamuzakjp/generational-cache": "^1.0.1", + "@csstools/css-calc": "^3.2.0", + "@csstools/css-color-parser": "^4.1.0", + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/@asamuzakjp/dom-selector": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-7.1.1.tgz", + "integrity": "sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@asamuzakjp/generational-cache": "^1.0.1", + "@asamuzakjp/nwsapi": "^2.3.9", + "bidi-js": "^1.0.3", + "css-tree": "^3.2.1", + "is-potential-custom-element-name": "^1.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/@asamuzakjp/generational-cache": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@asamuzakjp/generational-cache/-/generational-cache-1.0.1.tgz", + "integrity": "sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/@asamuzakjp/nwsapi": { + "version": "2.3.9", + "resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz", + "integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bramus/specificity": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/@bramus/specificity/-/specificity-2.4.2.tgz", + "integrity": "sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "css-tree": "^3.0.0" + }, + "bin": { + "specificity": "bin/cli.js" + } + }, + "node_modules/@csstools/color-helpers": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.1.1.tgz", + "integrity": "sha512-gLNsunvwf3mCi5u5o46/Z/JcJMnhbHSaZ69rkgPzNM3J4s8hWwpPUQB6/tt0EDFyCiWzxANlx+2LJwpYj4zS1w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@csstools/css-calc": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.4.0.tgz", + "integrity": "sha512-XQKj5B7QiZcHiegCOCAzcAOJdhGgWOHbbu62h5e5mkHnn8lWcfiJhllkqWmxu5zWR9jucPHuo1iTB56P033hcg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.2.3.tgz", + "integrity": "sha512-y4LpL+lmpuyKDiEFq2PnZUVFdAjsoB/qQJod79yLNokXyW7jewi+/WJ69EfItj8A2unWtxXnGjw6LYXgXu5ZjA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/color-helpers": "^6.1.1", + "@csstools/css-calc": "^3.4.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.0.tgz", + "integrity": "sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^4.0.0" + } + }, + "node_modules/@csstools/css-syntax-patches-for-csstree": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.14.tgz", + "integrity": "sha512-HpbVXyrofRXpHpgkNIjU/3EWR4WJvOkO3emNK/L6X/mTJU7bGUI3AkkpoTNXznQLp0KRjLHELTGeKI5dIkI9JQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "peerDependencies": { + "css-tree": "^3.2.1" + }, + "peerDependenciesMeta": { + "css-tree": { + "optional": true + } + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-4.0.1.tgz", + "integrity": "sha512-bPlN9S9O1A0euCpEWE4qnvB5YDuyYVsUTrxSgmAM1Is0j4tICHoVyOVAXfWMP/kS9ZrjvyIXWV2PmomiAXXqOw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@exodus/bytes": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.1.tgz", + "integrity": "sha512-S6mL0yNB/Abt9Ei4tq8gDhcczc4S3+vQ4ra7vxnAf+YHC02srtqxKKZghx2Dq6p0e66THKwR6r8N6P95wEty7Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + }, + "peerDependencies": { + "@noble/hashes": "^1.8.0 || ^2.0.0" + }, + "peerDependenciesMeta": { + "@noble/hashes": { + "optional": true + } + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.150.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.150.0.tgz", + "integrity": "sha512-rDS5/31E9HfPl/CIzGrn0DOlvBbXFseQ5URJ9sYMfstbKLD/c6Gm9vmRzRGDdAXyOIL4zmO37lc9RIwYqVruZw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/oxc-project" + } + }, + "node_modules/@oxlint/binding-android-arm-eabi": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.83.0.tgz", + "integrity": "sha512-0yGY24EwsLk5YDe6F+VkmZyRHSwJDALa3nIrPpq7FXmp2lV2d0TzvBCGeZk+wgiULRGr5blhyr4QMp5KCXJUqA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-android-arm64": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.83.0.tgz", + "integrity": "sha512-hHfJ0vc17A4iUjH5p9BsTUPYbYRNxGpvD2lbu1aBRk54bzNIx9o5TtYF39QPZcV95DagZd+4DEAw2RH3G2ZsMg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-arm64": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.83.0.tgz", + "integrity": "sha512-hsOjYjszLb/3zym/TkzUMPAoQlTJcuzSyEPOAyA+skXJIX9M0o+4JfOtqopX/Vf4hSLrJ98j0nvFo23gzk8auQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-x64": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.83.0.tgz", + "integrity": "sha512-mjh5oH2EA+wl5yRJYT9K9G61O2zFlpuv+yf2JwZOi0+dq2FnTUtm1h8i+5Ik0fXPWIu/k84I1psZR9aQsLAnyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-freebsd-x64": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.83.0.tgz", + "integrity": "sha512-fNHr64/YaO8YssuoDVC8+F4Uk5enR86q5uxfHkQrjAPs1dbAILOrD2uaud+J7MO8Fx774g44ERLD0IGIvZE48w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-gnueabihf": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.83.0.tgz", + "integrity": "sha512-Qpwy3zzAwMj+8/lyYItHmkSMwbkprFNWTK7jPYDOxSyxEhaSLOWYUTCMkjF334J8/WD0nznCCsoBbIH6hpsuIw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-musleabihf": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.83.0.tgz", + "integrity": "sha512-s+BirYLFq7JL2k9sP0XI3ZXJ9dYvJ8sX3jLCLoag7tt+zrSHpZxP0jqznfL+Gdgwu7ay0dYgGYJXrQvq3iWloA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-gnu": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.83.0.tgz", + "integrity": "sha512-7lihXt3vKr+GIyapNbHrnFHm/biiW30le6Zv/DExbAFPF6YwCQXVFlONPFehxs0CpGO4CBfYPM9rdDT+XMoIlg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-musl": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.83.0.tgz", + "integrity": "sha512-q63JalLYVkZiZvls1z3PPUnpmQluOMXp0khqQMznCeAPLGydfNY8JhvuA4WlK57JfrvikU8wB5lPVveqpIXvew==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-ppc64-gnu": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.83.0.tgz", + "integrity": "sha512-krQmDF+dRbxvdqVPV88ZuOoPPu8X5BuqDA8Hd+qcS4YMRQCb+nexA57DazgGsc/rGdKBe3QmV0mnv0bdpW/p5g==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-gnu": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.83.0.tgz", + "integrity": "sha512-MmOl8Y6txEAXZU1RG8Rr264jQ6D7VPmqFsU/45x/FeWsGe32hklTqGrLE6UxHzp5Rjt0wP+20tY8YXKgSFB3mw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-musl": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.83.0.tgz", + "integrity": "sha512-u1rMymh0W3JZkq370kzQsYPULGWqhE09pZRqnZvUSoYaI9pVO5yVX+iYIslmWuEgwuzH9YAaOsScJiobWCHoOw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-s390x-gnu": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.83.0.tgz", + "integrity": "sha512-y0zK3HNwGysu7rqtE+BQG/d0bx5gh/KwlOtghN8oWeK1KcWzeaLqtZrbm8owqdma1lFyrce/hTO5ismuNu+INQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-gnu": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.83.0.tgz", + "integrity": "sha512-rS5gM0NgD7ngmuJmbIehsidtrOwKkLFwCQbKEeb9KuyQrrWNq5Zkn0uV6AYdXOMJ0grrWEiLwBuvMxt8w5vsNw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-musl": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.83.0.tgz", + "integrity": "sha512-W2IH4EtpcPaWcvNGCA95YoDg4vxqE/ZiPCi3arrxEEpsK7+JQN9WYwrlYFx9pcdP6KPXqRqkv3zdQPHcx7b6YQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-openharmony-arm64": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.83.0.tgz", + "integrity": "sha512-6LyKkUyoajssTPLlZmDbZIbu4IZ5B4bGuRUnBgCGpEvHP3FQMaYITncHA/unPUo7q+Z+pIu2HhdkQ+8d1SG7iA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-arm64-msvc": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.83.0.tgz", + "integrity": "sha512-Uz/fObEtF0jmNJQJ8CGRBKfefYstS0/wjD3s6IGzP8nUwsJykHQJBiN3npHwKiGRGn/vvBEgNr4B3cCzmmatvg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-ia32-msvc": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.83.0.tgz", + "integrity": "sha512-u7XcvPW6Bk58tY5iWs2ESb0vJjoE/kuSpHxopbwp/p3ZtWVQXZ6wor5w3ssVTHOqd/v8b+QdhSFWQ4grEUNWpA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-x64-msvc": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.83.0.tgz", + "integrity": "sha512-LZRubd7ph13QmAg4fFecTYVZkiYbROR2Htaxh/ufWRkDhPOm2wrwaEYR89e0YpPFD3dqBrPoxS7myBw5hmYA7Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.9.tgz", + "integrity": "sha512-tNISae1QEf/vkb3xkRcjV5SEdzPE97We5IVaa2Z8jSszQPZ8U60B/YCYpw4QI7VidYsBtKavczXf+DyDs9WGxw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.9.tgz", + "integrity": "sha512-YC8YsI30o606GTZi0VyzYlsDKFP8W61i/QzayHDkLbNEz/IShqAmTa+hsJRj13xTHA0H+6fk4b2UmGn+Q/cMlg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.9.tgz", + "integrity": "sha512-IwhlH3qK5urrY8hZiEgGkHKEFN901p/p2bjxCxJlr4GyNnF7wYpUvK+Y43uaRYuC4hpfjzbR3SJC3arX1jGvmw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.9.tgz", + "integrity": "sha512-XxpJfVzFh+jilRxIXUqcfYAYcunIc/XEzIizsOL1fcJee5Sf7H3mH8WlLmfHfluz5amqR88QQo9izKtmMlavAw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.9.tgz", + "integrity": "sha512-kSfvhmgeWyfkbT3p/1s5vSgboogoah2zkm9fX2zjg2hHxSV7T4KhMWRUUaRk4OXNqoD3QAUeRqLcs1aZOK4U1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.9.tgz", + "integrity": "sha512-1RVzG17pxqbTfYLC352JlLt6kKLG+6Hr30n8DlIJqsnV5luUDd2Qdx9Ayw1Cabfyb1K9k0jXEZ7evxkRoT+uiw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.9.tgz", + "integrity": "sha512-BXqPvZ2drqVD+/Z8UpKwcs4Mp7grM+eGFku4CAEKrEtcbAsUpzREphK1sogCRZGreVPiMkiiBtw0n3TPteuqvw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.9.tgz", + "integrity": "sha512-11vWvo8YDwLzukt27J3aYDWU+gg2P7J+ZOmiJ0hkF5BXZDW7pVya7r40MXDy6ya0i9KamoENSVKIugvJNgFXIA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.9.tgz", + "integrity": "sha512-a1tijMkdwsIARtc0F39ApURROkf3NwqinI6TOiSSWCTR7dT96dffNvMUtDHnq64wKNTIZOIlzKrFvvFUznJiyw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.9.tgz", + "integrity": "sha512-x6SQNdAvv4c3hWqTMaWuawzMX9myaCs/yEmlGsxJzkdClnHW7FbrjQuSiRDhuSYzEYoEMhsaJy9qHG/XNemJPQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.9.tgz", + "integrity": "sha512-9s0AZ8BFK5/n7B/TBoa2yJE3gI3KURrbXcPBlsAsvjU4VeJKgE90y1YtNxyEUIcHPQkg6/yfF3qihUrcM/Kf0Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.9.tgz", + "integrity": "sha512-P7VWAmV+WdJluH7ovnRGoiv2i8To7GAZ+kGzfGup635cyL7SyYl3lSUaA3Gp5THf0n/Co5EyEqb2zbqq+nMOHQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.9.tgz", + "integrity": "sha512-1qixtsE4BK8h+yS3BfmZ09UhA7O/N4IACva6YBr7EBvCJraByTuRcgOTaiA62Tm0vey3UcKXLOaoGHtYmNGEVg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.9.tgz", + "integrity": "sha512-ok8IQjcEPs1AKZfuEUznVBrJw+gK4soq+bx8b1X2XoMqVClarc1q5JDmVtWXY1xfr6ZuHTAsPXHTgTrqKTZeww==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.9.tgz", + "integrity": "sha512-Ip2mXoU0hM0boq3Rf+ekuT653OROSo6aSYcPT1VHE4q52KvyxgFkQgrgb/IEsxOuvQ2fZZbs8khJAyCEPM24/g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@testing-library/dom": { + "version": "10.4.2", + "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.2.tgz", + "integrity": "sha512-yzr2S9HyAIdhz2/6qHgbs665Q7PKVcDF05vsOlHPxG1mo36gKVesdYVeDLnXgfjJ03CrKRk08knc6+E/9m8v2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.10.4", + "@babel/runtime": "^7.12.5", + "@types/aria-query": "^5.0.1", + "aria-query": "5.3.0", + "dom-accessibility-api": "^0.5.9", + "lz-string": "^1.5.0", + "picocolors": "1.1.1", + "pretty-format": "^27.0.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@testing-library/react": { + "version": "16.3.3", + "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-16.3.3.tgz", + "integrity": "sha512-Uo193NgQbPMz6lrrhtRQQFcMC6Re/ELLFbbuVL30WDlZxlpZf9/lMHTAVxPRLw1q1iu9OJmR1c2BLiENRstdBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.12.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@testing-library/dom": "^10.0.0", + "@types/react": "^18.0.0 || ^19.0.0", + "@types/react-dom": "^18.0.0 || ^19.0.0", + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@types/aria-query": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", + "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.13.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.6.tgz", + "integrity": "sha512-SGrw/h3KPFshy3OE6ZL53LMBG5vGQQ8/gIpiqz/kRZhPJ7HgwCEs8LBuNtWLa8dvGZVpSF7+Bf+c11HUrCb/yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@types/react": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.3.0.tgz", + "integrity": "sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.3.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.1.1.tgz", + "integrity": "sha512-yxLaQV9gkhS8ezJqCM6+ndU7mDY6gqAg75NQ+0IjwEI8IYOmQCgkRwHKVSfWXW076DsqMo0Dk+0FK1U+M5RgFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rolldown/pluginutils": "^1.0.1" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", + "babel-plugin-react-compiler": "^1.0.0", + "oxc-transform-react": "^0.145.0", + "vite": "^8.0.0" + }, + "peerDependenciesMeta": { + "@rolldown/plugin-babel": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + }, + "oxc-transform-react": { + "optional": true + } + } + }, + "node_modules/@vitest/mocker": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-5.0.1.tgz", + "integrity": "sha512-6K1DoBNAPGvuOcSsGA4D6x+5zEEff/KmOOP3uetT2TrGpVfI+HRHRnJJfKi5ib/g1vx8IYHQD8s0pbJz8WQI7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.31", + "@vitest/spy": "5.0.1", + "estree-walker": "^3.0.3", + "magic-string": "^1.2.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/spy": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-5.0.1.tgz", + "integrity": "sha512-rbto/mF/SGERxEgYOek7Xm6B9b+y+mVoo+f4b2LymYO8zM1b7uB5nHuhVMTP2hxdzgxvGiZYGxGIaMvL5y180Q==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/aria-query": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", + "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "dequal": "^2.0.3" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/bidi-js": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.1.0.tgz", + "integrity": "sha512-fX1Onk0tdVPC7obPWB5EbJ1z7NVhLq4m2xZLq2YXBkxzMXIGRpNMU88n0EPgWseKl12J7zXs7qrDxPK4sRs2fg==", + "dev": true, + "license": "MIT", + "dependencies": { + "require-from-string": "^2.0.2" + } + }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/css-tree": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", + "integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "mdn-data": "2.27.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/data-urls": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", + "integrity": "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-mimetype": "^5.0.0", + "whatwg-url": "^16.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "dev": true, + "license": "MIT" + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/dom-accessibility-api": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", + "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", + "dev": true, + "license": "MIT" + }, + "node_modules/entities": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-8.1.0.tgz", + "integrity": "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-module-lexer": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", + "dev": true, + "license": "MIT" + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/html-encoding-sniffer": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-6.0.0.tgz", + "integrity": "sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@exodus/bytes": "^1.6.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/jsdom": { + "version": "29.1.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.1.1.tgz", + "integrity": "sha512-ECi4Fi2f7BdJtUKTflYRTiaMxIB0O6zfR1fX0GXpUrf6flp8QIYn1UT20YQqdSOfk2dfkCwS8LAFoJDEppNK5Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@asamuzakjp/css-color": "^5.1.11", + "@asamuzakjp/dom-selector": "^7.1.1", + "@bramus/specificity": "^2.4.2", + "@csstools/css-syntax-patches-for-csstree": "^1.1.3", + "@exodus/bytes": "^1.15.0", + "css-tree": "^3.2.1", + "data-urls": "^7.0.0", + "decimal.js": "^10.6.0", + "html-encoding-sniffer": "^6.0.0", + "is-potential-custom-element-name": "^1.0.1", + "lru-cache": "^11.3.5", + "parse5": "^8.0.1", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^6.0.1", + "undici": "^7.25.0", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^8.0.1", + "whatwg-mimetype": "^5.0.0", + "whatwg-url": "^16.0.1", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24.0.0" + }, + "peerDependencies": { + "canvas": "^3.0.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lru-cache": { + "version": "11.5.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.3.tgz", + "integrity": "sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/lz-string": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz", + "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", + "dev": true, + "license": "MIT", + "bin": { + "lz-string": "bin/bin.js" + } + }, + "node_modules/magic-string": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.4.1.tgz", + "integrity": "sha512-8lyCu36ErXR0J9uaGKlKQoiLZKmtI63YGLE8G2o9jyRPdr4X47LusSOwgOJOzcVtp81fTAAjxR7BwKz682Jhow==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.6.0" + } + }, + "node_modules/mdn-data": { + "version": "2.27.1", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", + "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", + "dev": true, + "license": "CC0-1.0" + }, + "node_modules/nanoid": { + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/obug": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", + "integrity": "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/oxlint": { + "version": "1.83.0", + "resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.83.0.tgz", + "integrity": "sha512-cyDzSzaw3uzP0TeCeq3lLRPPoaUxkbB4ZOXj+kn+5r+BX9V+4bNVGk9lxer+WrgcpebH4JxLlJ3KQjveVztOLQ==", + "dev": true, + "license": "MIT", + "bin": { + "oxlint": "bin/oxlint" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/oxc-project" + }, + "optionalDependencies": { + "@oxlint/binding-android-arm-eabi": "1.83.0", + "@oxlint/binding-android-arm64": "1.83.0", + "@oxlint/binding-darwin-arm64": "1.83.0", + "@oxlint/binding-darwin-x64": "1.83.0", + "@oxlint/binding-freebsd-x64": "1.83.0", + "@oxlint/binding-linux-arm-gnueabihf": "1.83.0", + "@oxlint/binding-linux-arm-musleabihf": "1.83.0", + "@oxlint/binding-linux-arm64-gnu": "1.83.0", + "@oxlint/binding-linux-arm64-musl": "1.83.0", + "@oxlint/binding-linux-ppc64-gnu": "1.83.0", + "@oxlint/binding-linux-riscv64-gnu": "1.83.0", + "@oxlint/binding-linux-riscv64-musl": "1.83.0", + "@oxlint/binding-linux-s390x-gnu": "1.83.0", + "@oxlint/binding-linux-x64-gnu": "1.83.0", + "@oxlint/binding-linux-x64-musl": "1.83.0", + "@oxlint/binding-openharmony-arm64": "1.83.0", + "@oxlint/binding-win32-arm64-msvc": "1.83.0", + "@oxlint/binding-win32-ia32-msvc": "1.83.0", + "@oxlint/binding-win32-x64-msvc": "1.83.0" + }, + "peerDependencies": { + "oxlint-tsgolint": ">=7.0.2001", + "vite-plus": "*" + }, + "peerDependenciesMeta": { + "oxlint-tsgolint": { + "optional": true + }, + "vite-plus": { + "optional": true + } + } + }, + "node_modules/parse5": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", + "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==", + "dev": true, + "license": "MIT", + "dependencies": { + "entities": "^8.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.18", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/pretty-format": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", + "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1", + "ansi-styles": "^5.0.0", + "react-is": "^17.0.1" + }, + "engines": { + "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/react": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.3.0.tgz", + "integrity": "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.28.0" + }, + "peerDependencies": { + "react": "^19.3.0" + } + }, + "node_modules/react-is": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", + "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", + "dev": true, + "license": "MIT" + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/rolldown": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.9.tgz", + "integrity": "sha512-hx/Pv0N1haXRb11qkfnK5MXB/iqr7i0yjWQqmO9uHqZpBgQSqzc8UsSnEpalsh+j1I8qQ2CkXAkJC8Br3dKSlg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.150.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.9", + "@rolldown/binding-android-arm64": "1.2.9", + "@rolldown/binding-darwin-arm64": "1.2.9", + "@rolldown/binding-darwin-x64": "1.2.9", + "@rolldown/binding-freebsd-x64": "1.2.9", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.9", + "@rolldown/binding-linux-arm64-gnu": "1.2.9", + "@rolldown/binding-linux-arm64-musl": "1.2.9", + "@rolldown/binding-linux-ppc64-gnu": "1.2.9", + "@rolldown/binding-linux-s390x-gnu": "1.2.9", + "@rolldown/binding-linux-x64-gnu": "1.2.9", + "@rolldown/binding-linux-x64-musl": "1.2.9", + "@rolldown/binding-openharmony-arm64": "1.2.9", + "@rolldown/binding-win32-arm64-msvc": "1.2.9", + "@rolldown/binding-win32-x64-msvc": "1.2.9" + } + }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "dev": true, + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } + }, + "node_modules/scheduler": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", + "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", + "license": "MIT" + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", + "integrity": "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==", + "dev": true, + "license": "MIT" + }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinybench": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.1.4.tgz", + "integrity": "sha512-9APumHG7r4yOk4X4WlkmE71aZcv1gvin1czO3OQ1U9iJcFA5Ja/ygyb0vPOVHTthFozUYs8CLoLUlM8grb2lTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/tinyexec": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.0.tgz", + "integrity": "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tldts": { + "version": "7.4.13", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.4.13.tgz", + "integrity": "sha512-iHtaIWWIbMDkCeJdTBzZFGgbluE5J+oHlb2g7+oAz1S1gpuVpabRZdQyd471Vl8UUkcz2vXSL8xZH2kyCe8tfA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tldts-core": "^7.4.13" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "7.4.13", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.13.tgz", + "integrity": "sha512-mbYsrih5FRtGxs3Usvl/PqwJsNpp+jsmrdFviiK02teHDG0/HebBG/pqCylje3kzgXYzuLoHJF/0mz9W53t8Xg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tough-cookie": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.2.tgz", + "integrity": "sha512-exgYmnmL/sJpR3upZfXG5PoatXQii55xAiXGXzY+sROLZ/Y+SLcp9PgJNI9Vz37HpQ74WvDcLT8eqm+kV3FzrA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^7.0.5" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/tr46": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz", + "integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici": { + "version": "7.29.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", + "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/vite": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.0.tgz", + "integrity": "sha512-lhZBVvEHefgE+HQZC9O7EBJgCU/nVzFNl7vkS4RE0APtWLP02/8QVIkQtzBxPquh7lq5/78NHipTj7ODQ6XuyQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.33.0", + "picomatch": "^4.0.7", + "postcss": "^8.5.28", + "rolldown": "~1.2.6", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.7.1", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-5.0.1.tgz", + "integrity": "sha512-iA95lQbKEkvrtTkdAgnWbXfbipWiiWe/hDl2P5tMi6WFwD76G0NxXAGp/M9EOcYupeGJRr6wppMc7CoA41TQjg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/mocker": "5.0.1", + "chai": "^6.2.2", + "es-module-lexer": "^2.3.2", + "expect-type": "^1.4.0", + "magic-string": "^1.2.3", + "obug": "^2.1.4", + "picomatch": "^4.0.7", + "std-env": "^4.2.0", + "tinybench": "6.1.4", + "tinyexec": "1.3.0", + "tinyglobby": "^0.2.17", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^22.12.0 || ^24.0.0 || >=26.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "5.0.1", + "@vitest/browser-preview": "5.0.1", + "@vitest/browser-webdriverio": "^5.0.0-beta.5 || >=5.0.0", + "@vitest/coverage-istanbul": "5.0.1", + "@vitest/coverage-v8": "5.0.1", + "@vitest/ui": "5.0.1", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.4.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/w3c-xmlserializer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", + "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/webidl-conversions": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz", + "integrity": "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=20" + } + }, + "node_modules/whatwg-mimetype": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz", + "integrity": "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/whatwg-url": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz", + "integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@exodus/bytes": "^1.11.0", + "tr46": "^6.0.0", + "webidl-conversions": "^8.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/xml-name-validator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", + "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/web/package.json b/web/package.json new file mode 100644 index 0000000..d8f2f5f --- /dev/null +++ b/web/package.json @@ -0,0 +1,30 @@ +{ + "name": "web", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "lint": "oxlint", + "preview": "vite preview", + "test": "vitest run" + }, + "dependencies": { + "react": "^19.2.8", + "react-dom": "^19.2.8" + }, + "devDependencies": { + "@testing-library/dom": "^10.4.2", + "@testing-library/react": "^16.3.3", + "@types/node": "^24.13.3", + "@types/react": "^19.2.18", + "@types/react-dom": "^19.2.7", + "@vitejs/plugin-react": "^6.1.1", + "jsdom": "^29.1.1", + "oxlint": "^1.81.0", + "typescript": "~6.0.2", + "vite": "^8.3.0", + "vitest": "^5.0.1" + } +} diff --git a/web/public/favicon.svg b/web/public/favicon.svg new file mode 100644 index 0000000..6893eb1 --- /dev/null +++ b/web/public/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/web/src/App.css b/web/src/App.css new file mode 100644 index 0000000..843da95 --- /dev/null +++ b/web/src/App.css @@ -0,0 +1,244 @@ +/* web/src/App.css — Overview page styles. + Air-gap rules: system font stack only, no external fonts/icons/CDN. */ + +:root { + color-scheme: dark; + --bg: #0e1116; + --panel: #161b22; + --panel-2: #1c222b; + --border: #2a313c; + --text: #e6edf3; + --muted: #8b949e; + --accent: #4493f8; + --ok: #3fb950; + --warn: #d29922; + --err: #f85149; + --quar: #a371f7; + font-family: ui-sans-serif, system-ui, -apple-system, 'Segoe UI', Roboto, + 'Helvetica Neue', Arial, sans-serif; +} + +body { + margin: 0; + background: var(--bg); + color: var(--text); +} + +.mono { + font-family: ui-monospace, 'Cascadia Mono', Consolas, 'Courier New', monospace; + font-size: 0.92em; +} + +.muted { + color: var(--muted); +} + +.app { + max-width: 1100px; + margin: 0 auto; + padding: 16px 24px 48px; +} + +.topbar { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 12px; + flex-wrap: wrap; + border-bottom: 1px solid var(--border); + padding-bottom: 10px; + margin-bottom: 16px; +} + +.topbar h1 { + font-size: 20px; + margin: 0; +} + +.banner-error { + background: rgba(248, 81, 73, 0.12); + border: 1px solid var(--err); + border-radius: 6px; + padding: 8px 12px; + margin: 0 0 16px; + color: var(--err); +} + +/* --- stat cards ------------------------------------------------------- */ + +.cards { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); + gap: 12px; + margin-bottom: 20px; +} + +.card { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 8px; + padding: 14px 16px; +} + +.card-value { + font-size: 26px; + font-weight: 600; + line-height: 1.2; +} + +.card-label { + color: var(--muted); + font-size: 12px; + text-transform: uppercase; + letter-spacing: 0.06em; + margin-top: 4px; +} + +/* --- live feed table --------------------------------------------------- */ + +.feed { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 8px; + overflow: hidden; +} + +.feed-table { + width: 100%; + border-collapse: collapse; + font-size: 14px; +} + +.feed-table th, +.feed-table td { + text-align: left; + padding: 8px 12px; + border-bottom: 1px solid var(--border); +} + +.feed-table thead th { + background: var(--panel-2); + color: var(--muted); + font-size: 12px; + text-transform: uppercase; + letter-spacing: 0.05em; + position: sticky; + top: 0; +} + +.feed-table tbody tr { + cursor: pointer; +} + +.feed-table tbody tr:hover, +.feed-table tbody tr:focus-visible { + background: var(--panel-2); + outline: none; +} + +.feed-table .empty { + text-align: center; + padding: 32px 12px; + cursor: default; +} + +.badge { + display: inline-block; + border-radius: 999px; + padding: 1px 9px; + font-size: 12px; + border: 1px solid currentColor; +} + +.badge-parsed { color: var(--ok); } +.badge-unparsed { color: var(--warn); } +.badge-parse_error { color: var(--err); } +.badge-quarantined { color: var(--quar); } + +/* --- traceability drawer ----------------------------------------------- */ + +.drawer { + position: fixed; + top: 0; + right: 0; + bottom: 0; + width: min(520px, 92vw); + background: var(--panel); + border-left: 1px solid var(--border); + box-shadow: -12px 0 32px rgba(0, 0, 0, 0.45); + padding: 16px 20px 32px; + overflow-y: auto; + z-index: 10; +} + +.drawer-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + margin-bottom: 8px; +} + +.drawer-head h2 { + font-size: 16px; + margin: 0; +} + +.drawer-close { + background: var(--panel-2); + color: var(--text); + border: 1px solid var(--border); + border-radius: 6px; + padding: 4px 10px; + cursor: pointer; +} + +.drawer-close:hover { + border-color: var(--accent); +} + +.drawer h3 { + font-size: 13px; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--muted); + margin: 18px 0 6px; +} + +.drawer-error { + color: var(--err); + background: rgba(248, 81, 73, 0.12); + border: 1px solid var(--err); + border-radius: 6px; + padding: 8px 12px; +} + +.code { + background: var(--bg); + border: 1px solid var(--border); + border-radius: 6px; + padding: 10px 12px; + font-family: ui-monospace, 'Cascadia Mono', Consolas, 'Courier New', monospace; + font-size: 12px; + overflow-x: auto; + white-space: pre-wrap; + word-break: break-word; + margin: 0; +} + +.kv { + display: grid; + grid-template-columns: 110px 1fr; + gap: 4px 12px; + margin: 8px 0; + font-size: 13px; +} + +.kv dt { + color: var(--muted); +} + +.kv dd { + margin: 0; + word-break: break-all; +} diff --git a/web/src/App.tsx b/web/src/App.tsx new file mode 100644 index 0000000..7e553c0 --- /dev/null +++ b/web/src/App.tsx @@ -0,0 +1,224 @@ +// web/src/App.tsx — the M1 Overview page: 4 stat cards, the SSE live feed +// table, and a raw-traceability drawer. Same-origin /api only (dev: Vite +// proxy, prod: Caddy) — the gateway ships no CORS, and the dashboard is +// air-gapped: system fonts, zero external requests. +import { useCallback, useEffect, useState } from 'react' +import { getRaw, getStats } from './api' +import type { EventRow, OcsfEndpoint, RawTrace, Stats } from './types' +import { useEventStream } from './useEventStream' +import './App.css' + +const STATS_REFRESH_MS = 5000 + +/** parsed / (parsed + unparsed + parse_error + quarantined); "—" before any + * traffic has been seen (divide-by-zero guard). */ +function parsedPercent(byStatus: Stats['by_status']): string { + const total = + byStatus.parsed + byStatus.unparsed + byStatus.parse_error + byStatus.quarantined + if (total === 0) return '—' + return `${Math.round((byStatus.parsed / total) * 100)}%` +} + +function endpointIp(ep: OcsfEndpoint | null): string { + return ep?.ip ?? '—' +} + +function formatTime(iso: string): string { + return new Date(iso).toLocaleTimeString() +} + +interface StatCard { + label: string + value: string | number +} + +function statCards(stats: Stats | null): StatCard[] { + if (stats === null) { + return [ + { label: 'Events / min', value: '…' }, + { label: 'Parsed', value: '…' }, + { label: 'Unparsed', value: '…' }, + { label: 'Parse errors', value: '…' }, + ] + } + const { by_status: byStatus } = stats + return [ + { label: 'Events / min', value: stats.events_last_minute }, + { label: 'Parsed', value: parsedPercent(byStatus) }, + { label: 'Unparsed', value: byStatus.unparsed }, + { label: 'Parse errors', value: byStatus.parse_error }, + ] +} + +/** Side drawer: full OCSF document plus the raw line behind the event + * (fetched by event_id). Fetch is cancelled if the drawer closes first. */ +function TraceDrawer({ row, onClose }: { row: EventRow; onClose: () => void }) { + const [trace, setTrace] = useState(null) + const [error, setError] = useState(null) + + useEffect(() => { + let cancelled = false + getRaw(row.event_id) + .then((t) => { + if (!cancelled) setTrace(t) + }) + .catch((err: unknown) => { + if (!cancelled) setError(err instanceof Error ? err.message : String(err)) + }) + return () => { + cancelled = true + } + }, [row.event_id]) + + useEffect(() => { + const onKey = (ev: KeyboardEvent) => { + if (ev.key === 'Escape') onClose() + } + window.addEventListener('keydown', onKey) + return () => window.removeEventListener('keydown', onKey) + }, [onClose]) + + return ( + + ) +} + +export default function App() { + const [stats, setStats] = useState(null) + const [statsError, setStatsError] = useState(null) + const [selected, setSelected] = useState(null) + const rows = useEventStream() + + const refreshStats = useCallback(() => { + getStats() + .then((s) => { + setStats(s) + setStatsError(null) + }) + .catch((err: unknown) => { + setStatsError(err instanceof Error ? err.message : String(err)) + }) + }, []) + + useEffect(() => { + refreshStats() + const timer = setInterval(refreshStats, STATS_REFRESH_MS) + return () => clearInterval(timer) + }, [refreshStats]) + + return ( +
+
+

ULPF Overview

+ live feed · {rows.length} rows shown (cap 200) +
+ + {statsError !== null && ( +

+ stats unavailable: {statsError} +

+ )} + +
+ {statCards(stats).map((card) => ( +
+
{card.value}
+
{card.label}
+
+ ))} +
+ +
+ + + + + + + + + + + {rows.map((row) => ( + setSelected(row)} tabIndex={0} + onKeyDown={(ev) => { if (ev.key === 'Enter') setSelected(row) }}> + + + + + + ))} + {rows.length === 0 && ( + + + + )} + +
TimeFingerprintStatusSource → Destination
{formatTime(row.parsed_at)}{row.fingerprint_id} + {row.status} + + {endpointIp(row.ocsf?.src_endpoint ?? null)} →{' '} + {endpointIp(row.ocsf?.dst_endpoint ?? null)} +
+ Waiting for events on /api/stream/events… +
+
+ + {selected !== null && ( + setSelected(null)} + /> + )} +
+ ) +} diff --git a/web/src/api.ts b/web/src/api.ts new file mode 100644 index 0000000..6231249 --- /dev/null +++ b/web/src/api.ts @@ -0,0 +1,41 @@ +// web/src/api.ts — thin fetch wrappers over the same-origin /api gateway +// (services/gateway, Task 9). Every wrapper throws on non-2xx so callers get +// a real error instead of a resolved promise with an error body. +// +// getEvents is part of the client surface mirroring the gateway (status / +// fingerprint filters arrive in M2); the Overview page itself is fed by the +// SSE stream, which already snapshots the latest rows on connect. +import type { EventsResponse, RawTrace, Stats } from './types' + +async function getJson(url: string): Promise { + const res = await fetch(url) + if (!res.ok) { + throw new Error(`GET ${url} failed: ${res.status} ${res.statusText}`) + } + return (await res.json()) as T +} + +export function getStats(): Promise { + return getJson('/api/stats') +} + +export interface EventQuery { + status?: string + fingerprint?: string + limit?: number + before?: string +} + +export function getEvents(params: EventQuery = {}): Promise { + const q = new URLSearchParams() + if (params.status) q.set('status', params.status) + if (params.fingerprint) q.set('fingerprint', params.fingerprint) + if (params.limit !== undefined) q.set('limit', String(params.limit)) + if (params.before) q.set('before', params.before) + const qs = q.toString() + return getJson(`/api/events${qs ? `?${qs}` : ''}`) +} + +export function getRaw(eventId: string): Promise { + return getJson(`/api/events/${encodeURIComponent(eventId)}/raw`) +} diff --git a/web/src/main.tsx b/web/src/main.tsx new file mode 100644 index 0000000..1f7ba72 --- /dev/null +++ b/web/src/main.tsx @@ -0,0 +1,10 @@ +import { StrictMode } from 'react' +import { createRoot } from 'react-dom/client' +import './App.css' +import App from './App.tsx' + +createRoot(document.getElementById('root')!).render( + + + , +) diff --git a/web/src/types.ts b/web/src/types.ts new file mode 100644 index 0000000..17d9f5c --- /dev/null +++ b/web/src/types.ts @@ -0,0 +1,73 @@ +// web/src/types.ts — mirrors the frozen M1 gateway contract (Task 9, +// services/gateway/queries.py + app.py) key for key. EventRow carries EXACTLY +// these 7 keys; UUIDs arrive as strings (the gateway serializes UUIDs as JSON +// strings in REST and SSE alike), timestamps as ISO-8601 strings. + +export type EventStatus = 'parsed' | 'unparsed' | 'parse_error' | 'quarantined' + +/** Curated OCSF endpoint object (libs/ulpf-core parsing.py: {ip, port}); the + * document sets it to null when the rule mapped nothing into it. */ +export interface OcsfEndpoint { + ip?: string + port?: number | string | null +} + +/** The OCSF document produced by the pipeline parser (class_uid 4001 shape): + * src_endpoint/dst_endpoint are objects-or-null, unmapped carries captured + * fields with no OCSF mapping. Present only when status = "parsed". */ +export interface OcsfDocument { + class_uid: number + class_name: string + activity_id: number + severity_id: number | null + time: string | null + src_endpoint: OcsfEndpoint | null + dst_endpoint: OcsfEndpoint | null + action: string | null + message: string | null + metadata: { product: string } + unmapped: Record +} + +/** One normalized event (current view) — GET /api/events rows and the + * /api/stream/events SSE payload share this exact shape. */ +export interface EventRow { + event_id: string + raw_id: string + fingerprint_id: string + rule_version: number | null + status: EventStatus + parsed_at: string + ocsf: OcsfDocument | null +} + +/** GET /api/events response. */ +export interface EventsResponse { + events: EventRow[] +} + +/** One per-fingerprint aggregate from GET /api/stats. */ +export interface FingerprintStats { + fingerprint_id: string + total: number + parsed: number +} + +/** GET /api/stats response (dlq_total is always null in M1). */ +export interface Stats { + raw_total: number + by_status: Record + by_fingerprint: FingerprintStats[] + events_last_minute: number + dlq_total: number | null +} + +/** GET /api/events/{event_id}/raw response — the raw line behind an event. */ +export interface RawTrace { + raw_id: string + received_at: string + source_id: string + transport: string + content_hash: string + raw_text: string +} diff --git a/web/src/useEventStream.test.ts b/web/src/useEventStream.test.ts new file mode 100644 index 0000000..ee762b7 --- /dev/null +++ b/web/src/useEventStream.test.ts @@ -0,0 +1,134 @@ +// @vitest-environment jsdom +// web/src/useEventStream.test.ts — renderHook against a mock EventSource +// installed on globalThis (node/jsdom ships none). Asserts the feed contract: +// every SSE data line becomes a row, newest first, capped at FEED_CAP rows. +import { act, renderHook } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { FEED_CAP, useEventStream } from './useEventStream' +import type { EventRow } from './types' + +/** Minimal EventSource double: records instances, delivers data lines, and + * can simulate a socket that fully closed (readyState CLOSED + onerror). */ +class MockEventSource { + static instances: MockEventSource[] = [] + static readonly CONNECTING = 0 + static readonly OPEN = 1 + static readonly CLOSED = 2 + + url: string + readyState: number = MockEventSource.OPEN + onmessage: ((ev: MessageEvent) => void) | null = null + onerror: ((ev: Event) => void) | null = null + + constructor(url: string) { + this.url = url + MockEventSource.instances.push(this) + } + + close(): void { + this.readyState = MockEventSource.CLOSED + } + + emit(row: unknown): void { + this.onmessage?.({ data: JSON.stringify(row) } as MessageEvent) + } + + failAsClosed(): void { + this.readyState = MockEventSource.CLOSED + this.onerror?.(new Event('error')) + } +} + +function makeRow(seq: number): EventRow { + return { + event_id: `00000000-0000-0000-0000-${String(seq).padStart(12, '0')}`, + raw_id: `10000000-0000-0000-0000-${String(seq).padStart(12, '0')}`, + fingerprint_id: `fp-${seq}`, + rule_version: 1, + status: seq % 2 === 0 ? 'parsed' : 'unparsed', + parsed_at: new Date(Date.UTC(2026, 8, 19, 12, 0, seq)).toISOString(), + ocsf: null, + } +} + +beforeEach(() => { + MockEventSource.instances = [] + vi.stubGlobal('EventSource', MockEventSource as unknown as typeof EventSource) +}) + +afterEach(() => { + vi.unstubAllGlobals() + vi.useRealTimers() +}) + +describe('useEventStream', () => { + it('subscribes to the gateway SSE stream', () => { + renderHook(() => useEventStream()) + expect(MockEventSource.instances).toHaveLength(1) + expect(MockEventSource.instances[0]?.url).toBe('/api/stream/events') + }) + + it('prepends each event, newest first', () => { + const { result } = renderHook(() => useEventStream()) + const es = MockEventSource.instances[0] + act(() => { + for (const seq of [1, 2, 3]) es?.emit(makeRow(seq)) + }) + expect(result.current).toHaveLength(3) + // row[0] is the last-received event + expect(result.current[0]?.fingerprint_id).toBe('fp-3') + expect(result.current[1]?.fingerprint_id).toBe('fp-2') + expect(result.current[2]?.fingerprint_id).toBe('fp-1') + }) + + it(`caps the feed at ${FEED_CAP} rows`, () => { + const { result } = renderHook(() => useEventStream()) + const es = MockEventSource.instances[0] + act(() => { + for (let seq = 1; seq <= FEED_CAP + 5; seq++) es?.emit(makeRow(seq)) + }) + expect(result.current).toHaveLength(FEED_CAP) + // newest survives at the head... + expect(result.current[0]?.fingerprint_id).toBe(`fp-${FEED_CAP + 5}`) + // ...and the FEED_CAP-th oldest is the first to fall off the tail + expect(result.current[FEED_CAP - 1]?.fingerprint_id).toBe('fp-6') + }) + + it('drops malformed data lines instead of poisoning the feed', () => { + const { result } = renderHook(() => useEventStream()) + const es = MockEventSource.instances[0] + act(() => { + es?.emit(makeRow(1)) + es?.onmessage?.({ data: 'not-json' } as MessageEvent) + es?.emit(makeRow(2)) + }) + expect(result.current).toHaveLength(2) + expect(result.current[0]?.fingerprint_id).toBe('fp-2') + }) + + it('reconnects once after a 2s backoff when the socket fully closes', () => { + vi.useFakeTimers() + renderHook(() => useEventStream()) + expect(MockEventSource.instances).toHaveLength(1) + + act(() => { + MockEventSource.instances[0]?.failAsClosed() + }) + // no immediate reconnect — the 2s backoff guards a flapping gateway + expect(MockEventSource.instances).toHaveLength(1) + + act(() => { + vi.advanceTimersByTime(2000) + }) + expect(MockEventSource.instances).toHaveLength(2) + expect(MockEventSource.instances[1]?.url).toBe('/api/stream/events') + }) + + it('closes the socket on unmount', () => { + const { unmount } = renderHook(() => useEventStream()) + const es = MockEventSource.instances[0] + expect(es?.readyState).toBe(MockEventSource.OPEN) + unmount() + expect(es?.readyState).toBe(MockEventSource.CLOSED) + }) +}) diff --git a/web/src/useEventStream.ts b/web/src/useEventStream.ts new file mode 100644 index 0000000..63e7e3f --- /dev/null +++ b/web/src/useEventStream.ts @@ -0,0 +1,68 @@ +// web/src/useEventStream.ts — SSE subscription feeding the live table. +// +// One EventSource on /api/stream/events; every `data:` line is an EventRow +// JSON payload (gateway serializes UUIDs as strings) which is prepended and +// the feed capped at FEED_CAP rows client-side. Newest ends up first because +// the server snapshot arrives oldest-first and later rows are newer. +// +// Reconnect policy: while the browser keeps retrying (readyState CONNECTING) +// EventSource's built-in retry handles it; once the socket is fully CLOSED we +// close it for good and come back after a 2s backoff so a down gateway can't +// flap the feed hot. +import { useEffect, useState } from 'react' +import type { EventRow } from './types' + +export const FEED_CAP = 200 + +const RECONNECT_BACKOFF_MS = 2000 + +export function useEventStream(url: string = '/api/stream/events'): EventRow[] { + const [rows, setRows] = useState([]) + + useEffect(() => { + let source: EventSource | null = null + let reconnectTimer: ReturnType | null = null + let disposed = false + + const connect = () => { + source = new EventSource(url) + source.onmessage = (event: MessageEvent) => { + let row: EventRow + try { + row = JSON.parse(event.data) as EventRow + } catch { + console.warn('useEventStream: dropped malformed SSE data line') + return + } + setRows((prev) => [row, ...prev].slice(0, FEED_CAP)) + } + source.onerror = () => { + if (source !== null && source.readyState === EventSource.CLOSED) { + scheduleReconnect() + } + } + } + + const scheduleReconnect = () => { + source?.close() + source = null + if (!disposed && reconnectTimer === null) { + reconnectTimer = setTimeout(() => { + reconnectTimer = null + connect() + }, RECONNECT_BACKOFF_MS) + } + } + + connect() + + return () => { + disposed = true + if (reconnectTimer !== null) clearTimeout(reconnectTimer) + source?.close() + source = null + } + }, [url]) + + return rows +} diff --git a/web/tsconfig.app.json b/web/tsconfig.app.json new file mode 100644 index 0000000..6830b6f --- /dev/null +++ b/web/tsconfig.app.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", + "target": "es2023", + "lib": ["ES2023", "DOM"], + "module": "esnext", + "types": ["vite/client"], + "allowArbitraryExtensions": true, + "skipLibCheck": true, + + /* Bundler mode */ + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + + /* Linting */ + "noUnusedLocals": true, + "noUnusedParameters": true, + "erasableSyntaxOnly": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["src"] +} diff --git a/web/tsconfig.json b/web/tsconfig.json new file mode 100644 index 0000000..1ffef60 --- /dev/null +++ b/web/tsconfig.json @@ -0,0 +1,7 @@ +{ + "files": [], + "references": [ + { "path": "./tsconfig.app.json" }, + { "path": "./tsconfig.node.json" } + ] +} diff --git a/web/tsconfig.node.json b/web/tsconfig.node.json new file mode 100644 index 0000000..8455dcb --- /dev/null +++ b/web/tsconfig.node.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", + "target": "es2023", + "lib": ["ES2023"], + "types": ["node"], + "skipLibCheck": true, + + /* Bundler mode */ + "module": "nodenext", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "moduleDetection": "force", + "noEmit": true, + + /* Linting */ + "noUnusedLocals": true, + "noUnusedParameters": true, + "erasableSyntaxOnly": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["vite.config.ts"] +} diff --git a/web/vite.config.ts b/web/vite.config.ts new file mode 100644 index 0000000..0db5b80 --- /dev/null +++ b/web/vite.config.ts @@ -0,0 +1,19 @@ +import react from '@vitejs/plugin-react' +import { defineConfig } from 'vitest/config' + +// Dev proxy: same-origin /api in the browser, forwarded to the gateway +// (services/gateway, default :8000). In production Caddy does this instead — +// no CORS anywhere (the gateway deliberately ships none). +// test.environment stays "node"; the one DOM-dependent test opts into jsdom +// via a `@vitest-environment jsdom` docblock. +export default defineConfig({ + plugins: [react()], + server: { + proxy: { + '/api': 'http://localhost:8000', + }, + }, + test: { + environment: 'node', + }, +}) From b5f9f4e0a705570390ead253a2e3e9818d7d8dba Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 16:36:31 +0530 Subject: [PATCH 13/18] feat: traffic simulator over golden corpora (steady mode) Co-Authored-By: Claude Code --- pyproject.toml | 4 +- simulator/Dockerfile | 16 ++ simulator/__init__.py | 1 + simulator/data/raw_logs_acmegw.txt | 10 + simulator/data/raw_logs_cef.txt | 16 ++ simulator/data/raw_logs_json.txt | 15 ++ simulator/data/raw_logs_syslog.txt | 15 ++ simulator/requirements.txt | 1 + simulator/simulate.py | 281 +++++++++++++++++++++++ simulator/tests/__init__.py | 0 simulator/tests/test_simulate.py | 354 +++++++++++++++++++++++++++++ 11 files changed, 711 insertions(+), 2 deletions(-) create mode 100644 simulator/Dockerfile create mode 100644 simulator/__init__.py create mode 100644 simulator/data/raw_logs_acmegw.txt create mode 100644 simulator/data/raw_logs_cef.txt create mode 100644 simulator/data/raw_logs_json.txt create mode 100644 simulator/data/raw_logs_syslog.txt create mode 100644 simulator/requirements.txt create mode 100644 simulator/simulate.py create mode 100644 simulator/tests/__init__.py create mode 100644 simulator/tests/test_simulate.py diff --git a/pyproject.toml b/pyproject.toml index d44d6f0..5f1e850 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.pytest.ini_options] -testpaths = ["libs", "services"] -pythonpath = [".", "services/collector", "services/pipeline", "services/gateway"] +testpaths = ["libs", "services", "simulator"] +pythonpath = [".", "services/collector", "services/pipeline", "services/gateway", "simulator"] [tool.ruff] line-length = 100 diff --git a/simulator/Dockerfile b/simulator/Dockerfile new file mode 100644 index 0000000..6f14c30 --- /dev/null +++ b/simulator/Dockerfile @@ -0,0 +1,16 @@ +# simulator/Dockerfile — multi-vendor traffic simulator over the golden corpora. +# Build context is the repo root (compose build.context: ..; see /.dockerignore). +FROM python:3.12-slim + +WORKDIR /app/simulator + +# Runtime deps only: stdlib (asyncio, socket, argparse) plus httpx. +COPY simulator/requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +COPY simulator/simulate.py /app/simulator/simulate.py +COPY simulator/data/ /app/simulator/data/ + +# `docker compose run --rm simulator --eps 25 --duration 20` passes the flags +# straight through as argv (ENTRYPOINT, not CMD). +ENTRYPOINT ["python", "simulate.py"] diff --git a/simulator/__init__.py b/simulator/__init__.py new file mode 100644 index 0000000..43a846f --- /dev/null +++ b/simulator/__init__.py @@ -0,0 +1 @@ +# simulator — multi-vendor traffic simulator over the frozen golden corpora (M1 walking skeleton). diff --git a/simulator/data/raw_logs_acmegw.txt b/simulator/data/raw_logs_acmegw.txt new file mode 100644 index 0000000..650b74a --- /dev/null +++ b/simulator/data/raw_logs_acmegw.txt @@ -0,0 +1,10 @@ +ACMEGW|v3|2026-08-27T14:32:07+05:30|type:conn-deny|source=203.0.113.45:51422|target=192.168.1.10:22|proto=tcp|reason=blacklist-match|sev=high +ACMEGW|v3|2026-08-27T14:34:12+05:30|type:conn-allow|source=192.168.1.60:53211|target=8.8.8.8:53|proto=udp|reason=policy-ok|sev=info +ACMEGW|v3|2026-08-27T14:36:47+05:30|type:conn-deny|source=198.51.100.22:44521|target=192.168.1.14:80|proto=tcp|reason=known-bad-ip|sev=high +ACMEGW|v3|2026-08-27T14:38:30+05:30|type:conn-deny|source=203.0.113.90:51500|target=192.168.1.20:3389|proto=tcp|reason=brute-force|sev=critical +ACMEGW|v3|2026-08-27T14:40:05+05:30|type:conn-allow|source=192.168.1.70:52890|target=203.0.113.220:443|proto=tcp|reason=policy-ok|sev=info +ACMEGW|v3|2026-08-27T14:41:41+05:30|type:conn-deny|source=192.0.2.201:51900|target=192.168.1.15:3389|proto=tcp|reason=brute-force|sev=critical +ACMEGW|v3|2026-08-27T14:43:02+05:30|type:conn-deny|source=203.0.113.33:49213|target=192.168.1.50:80|proto=tcp|reason=malware-signature|sev=high +ACMEGW|v3|2026-08-27T14:45:15+05:30|type:conn-deny|source=198.51.100.90:44012|target=192.168.1.12:445|proto=tcp|reason=exploit-attempt|sev=critical +ACMEGW|v3|2026-08-27T14:46:29+05:30|type:conn-allow|source=192.168.1.18:60011|target=8.8.4.4:53|proto=udp|reason=policy-ok|sev=info +ACMEGW|v3|2026-08-27T14:47:03+05:30|type:conn-deny|source=203.0.113.150:59012|target=192.168.1.99:8080|proto=tcp|reason=proxy-abuse|sev=medium diff --git a/simulator/data/raw_logs_cef.txt b/simulator/data/raw_logs_cef.txt new file mode 100644 index 0000000..094a783 --- /dev/null +++ b/simulator/data/raw_logs_cef.txt @@ -0,0 +1,16 @@ +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|spyware|5|src=203.0.113.45 dst=192.168.1.10 spt=51422 dpt=443 act=block cat=spyware msg=Suspicious DNS Query +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|malware|7|src=198.51.100.22 dst=192.168.1.14 spt=44521 dpt=80 act=alert cat=malware msg=Known bad hash +CEF:0|CiscoASA|ASA|9.1|106023|Deny|4|src=203.0.113.90 dst=192.168.1.20 spt=51500 dpt=22 act=deny +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|spyware|3|src=192.0.2.15 dst=192.168.1.22 spt=52011 dpt=443 act=allow cat=spyware msg=Low confidence match +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|exploit|9|src=203.0.113.77 dst=192.168.1.10 spt=33210 dpt=3389 act=block cat=exploit msg=RDP brute force attempt +CEF:0|CiscoASA|ASA|9.1|106023|Deny|4|src=198.51.100.5 dst=192.168.1.30 spt=41022 dpt=25 act=deny +CEF:0|CiscoASA|ASA|9.1|106100|Permit|2|src=203.0.113.12 dst=192.168.1.40 spt=60011 dpt=443 act=allow +CEF:0|CiscoASA|ASA|9.1|106023|Deny|6|src=192.0.2.201 dst=192.168.1.15 spt=51900 dpt=3389 act=deny +CEF:0|FortiGate|FortiOS|7.2|virus|eicar|8|src=203.0.113.33 dst=192.168.1.50 spt=49213 dpt=80 act=block cat=virus msg=EICAR test file detected +CEF:0|FortiGate|FortiOS|7.2|traffic|forward|1|src=192.168.1.60 dst=8.8.8.8 spt=53211 dpt=53 act=allow cat=dns msg=Normal DNS lookup +CEF:0|FortiGate|FortiOS|7.2|ips|signature|7|src=198.51.100.90 dst=192.168.1.12 spt=44012 dpt=445 act=block cat=exploit msg=SMB exploit attempt CVE-2020-0796 +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|c2|10|src=192.168.1.18 dst=203.0.113.201 spt=51555 dpt=8443 act=block cat=command-and-control msg=Beaconing to known C2 domain +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|scan|5|src=203.0.113.44 dst=192.168.1.0 spt=40000 dpt=21 act=alert cat=scan msg=Port scan detected +CEF:0|CiscoASA|ASA|9.1|106023|Deny|3|src=203.0.113.150 dst=192.168.1.99 spt=59012 dpt=8080 act=deny +CEF:0|FortiGate|FortiOS|7.2|webfilter|blocked|4|src=192.168.1.70 dst=203.0.113.220 spt=52890 dpt=443 act=block cat=web-filter msg=Blocked category gambling +CEF:0|PaloAlto|PAN-OS|10.1|THREAT|vulnerability|6|src=198.51.100.61 dst=192.168.1.5 spt=48822 dpt=443 act=alert cat=vulnerability msg=SSL certificate anomaly detected diff --git a/simulator/data/raw_logs_json.txt b/simulator/data/raw_logs_json.txt new file mode 100644 index 0000000..26bfaca --- /dev/null +++ b/simulator/data/raw_logs_json.txt @@ -0,0 +1,15 @@ +{"timestamp":"2026-08-27T14:32:07+0530","event_type":"alert","src_ip":"203.0.113.45","src_port":51422,"dest_ip":"192.168.1.10","dest_port":22,"proto":"TCP","alert":{"signature":"ET SCAN Potential SSH Scan","severity":2}} +{"timestamp":"2026-08-27T14:34:00+0530","event_type":"alert","src_ip":"198.51.100.22","src_port":44521,"dest_ip":"192.168.1.14","dest_port":443,"proto":"TCP","alert":{"signature":"ET TROJAN Suspicious C2 Beacon","severity":1}} +{"timestamp":"2026-08-27T14:35:12+0530","event_type":"alert","src_ip":"203.0.113.90","src_port":51500,"dest_ip":"192.168.1.20","dest_port":3389,"proto":"TCP","alert":{"signature":"ET SCAN RDP Brute Force","severity":2}} +{"timestamp":"2026-08-27T14:36:47+0530","event_type":"flow","src_ip":"192.168.1.60","src_port":53211,"dest_ip":"8.8.8.8","dest_port":53,"proto":"UDP","alert":{"signature":"Normal DNS Lookup","severity":3}} +{"timestamp":"2026-08-27T14:37:59+0530","event_type":"alert","src_ip":"192.0.2.15","src_port":52011,"dest_ip":"192.168.1.22","dest_port":443,"proto":"TCP","alert":{"signature":"ET POLICY Low Confidence Spyware Match","severity":3}} +{"timestamp":"2026-08-27T14:38:34+0530","event_type":"alert","src_ip":"203.0.113.77","src_port":33210,"dest_ip":"192.168.1.10","dest_port":3389,"proto":"TCP","alert":{"signature":"ET EXPLOIT RDP Brute Force Attempt","severity":1}} +{"timestamp":"2026-08-27T14:39:20+0530","event_type":"alert","src_ip":"192.168.1.18","src_port":51555,"dest_ip":"203.0.113.201","dest_port":8443,"proto":"TCP","alert":{"signature":"ET TROJAN Known C2 Beacon Domain","severity":1}} +{"timestamp":"2026-08-27T14:40:05+0530","event_type":"alert","src_ip":"198.51.100.5","src_port":41022,"dest_ip":"192.168.1.30","dest_port":25,"proto":"TCP","alert":{"signature":"ET SCAN SMTP Probe","severity":2}} +{"timestamp":"2026-08-27T14:41:41+0530","event_type":"alert","src_ip":"203.0.113.150","src_port":59012,"dest_ip":"192.168.1.99","dest_port":8080,"proto":"TCP","alert":{"signature":"ET WEB_SERVER Suspicious Proxy Access","severity":2}} +{"timestamp":"2026-08-27T14:42:18+0530","event_type":"alert","src_ip":"192.0.2.201","src_port":51900,"dest_ip":"192.168.1.15","dest_port":3389,"proto":"TCP","alert":{"signature":"ET SCAN RDP Brute Force","severity":2}} +{"timestamp":"2026-08-27T14:43:02+0530","event_type":"alert","src_ip":"203.0.113.33","src_port":49213,"dest_ip":"192.168.1.50","dest_port":80,"proto":"TCP","alert":{"signature":"ET MALWARE EICAR Test File Detected","severity":1}} +{"timestamp":"2026-08-27T14:44:37+0530","event_type":"flow","src_ip":"192.168.1.70","src_port":52890,"dest_ip":"203.0.113.220","dest_port":443,"proto":"TCP","alert":{"signature":"Normal HTTPS Session","severity":3}} +{"timestamp":"2026-08-27T14:45:15+0530","event_type":"alert","src_ip":"198.51.100.90","src_port":44012,"dest_ip":"192.168.1.12","dest_port":445,"proto":"TCP","alert":{"signature":"ET EXPLOIT SMB CVE-2020-0796","severity":1}} +{"timestamp":"2026-08-27T14:45:53+0530","event_type":"alert","src_ip":"203.0.113.44","src_port":40000,"dest_ip":"192.168.1.1","dest_port":21,"proto":"TCP","alert":{"signature":"ET SCAN FTP Port Scan","severity":2}} +{"timestamp":"2026-08-27T14:46:29+0530","event_type":"alert","src_ip":"198.51.100.61","src_port":48822,"dest_ip":"192.168.1.5","dest_port":443,"proto":"TCP","alert":{"signature":"ET POLICY SSL Certificate Anomaly","severity":2}} diff --git a/simulator/data/raw_logs_syslog.txt b/simulator/data/raw_logs_syslog.txt new file mode 100644 index 0000000..0ee65fb --- /dev/null +++ b/simulator/data/raw_logs_syslog.txt @@ -0,0 +1,15 @@ +Aug 27 14:32:07 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=203.0.113.45 DST=192.168.1.10 LEN=60 PROTO=TCP SPT=51422 DPT=22 FLAGS=SYN +Aug 27 14:33:12 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=198.51.100.22 DST=192.168.1.14 LEN=52 PROTO=UDP SPT=33211 DPT=53 FLAGS= +Aug 27 14:34:01 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=203.0.113.90 DST=192.168.1.20 LEN=60 PROTO=TCP SPT=51500 DPT=3389 FLAGS=SYN +Aug 27 14:35:44 fw01 kernel: IPTABLES-ACCEPT: IN=eth0 OUT=eth1 SRC=192.168.1.60 DST=8.8.8.8 LEN=68 PROTO=UDP SPT=53211 DPT=53 FLAGS= +Aug 27 14:36:19 vpn-gw01 kernel: IPTABLES-DROP: IN=eth1 OUT= SRC=192.0.2.15 DST=192.168.1.22 LEN=60 PROTO=TCP SPT=52011 DPT=443 FLAGS=SYN +Aug 27 14:37:52 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=203.0.113.77 DST=192.168.1.10 LEN=60 PROTO=TCP SPT=33210 DPT=3389 FLAGS=SYN +Aug 27 14:38:30 vpn-gw01 kernel: IPTABLES-ACCEPT: IN=eth1 OUT=eth0 SRC=192.168.1.18 DST=203.0.113.201 LEN=1420 PROTO=TCP SPT=51555 DPT=8443 FLAGS=ACK +Aug 27 14:39:07 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=198.51.100.5 DST=192.168.1.30 LEN=60 PROTO=TCP SPT=41022 DPT=25 FLAGS=SYN +Aug 27 14:40:21 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=203.0.113.150 DST=192.168.1.99 LEN=60 PROTO=TCP SPT=59012 DPT=8080 FLAGS=SYN +Aug 27 14:41:55 vpn-gw01 kernel: IPTABLES-DROP: IN=eth1 OUT= SRC=192.0.2.201 DST=192.168.1.15 LEN=60 PROTO=TCP SPT=51900 DPT=3389 FLAGS=SYN +Aug 27 14:42:38 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=203.0.113.33 DST=192.168.1.50 LEN=60 PROTO=TCP SPT=49213 DPT=80 FLAGS=SYN +Aug 27 14:43:12 fw01 kernel: IPTABLES-ACCEPT: IN=eth0 OUT=eth1 SRC=192.168.1.70 DST=203.0.113.220 LEN=580 PROTO=TCP SPT=52890 DPT=443 FLAGS=ACK +Aug 27 14:44:49 vpn-gw01 kernel: IPTABLES-DROP: IN=eth1 OUT= SRC=198.51.100.90 DST=192.168.1.12 LEN=60 PROTO=TCP SPT=44012 DPT=445 FLAGS=SYN +Aug 27 14:45:26 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=203.0.113.44 DST=192.168.1.1 LEN=40 PROTO=TCP SPT=40000 DPT=21 FLAGS=SYN +Aug 27 14:46:03 fw01 kernel: IPTABLES-DROP: IN=eth0 OUT= SRC=198.51.100.61 DST=192.168.1.5 LEN=60 PROTO=TCP SPT=48822 DPT=443 FLAGS=SYN diff --git a/simulator/requirements.txt b/simulator/requirements.txt new file mode 100644 index 0000000..cee3237 --- /dev/null +++ b/simulator/requirements.txt @@ -0,0 +1 @@ +httpx==0.28.1 diff --git a/simulator/simulate.py b/simulator/simulate.py new file mode 100644 index 0000000..6bdd9e6 --- /dev/null +++ b/simulator/simulate.py @@ -0,0 +1,281 @@ +# simulator/simulate.py — multi-vendor traffic simulator over the frozen golden corpora. +# +# Routes each corpus to the transport the collector expects (R7): +# cef/syslog/acmegw -> UDP datagrams, one log line each, no envelope (the +# collector derives source_id from the peer address, so +# there is nothing to tag on the wire); +# json -> batched HTTP POSTs to /v1/ingest, source_id "ids01" +# (that field IS the raw.logs message key). +# +# Pacing: --eps is the TOTAL event rate, distributed evenly across the corpora +# found on disk; corpora are interleaved round-robin, one line per corpus per +# tick. --loop repeats the corpora until --duration elapses; without it, each +# corpus is sent once through (still paced), then the run ends. +# +# Stdout contract (parsed by the M1 smoke test): exactly one line per corpus, +# sent= +# then a final "total sent=". Everything else goes to stderr. Exit 0 unless +# the run was a total failure (no corpora, bad config, or every send failed). + +from __future__ import annotations + +import argparse +import asyncio +import inspect +import os +import socket +import sys +import time +from dataclasses import dataclass +from pathlib import Path + +import httpx + +DEFAULT_UDP_ADDR = ("localhost", 5514) +DEFAULT_HTTP_BASE = "http://localhost:8080" +HTTP_SOURCE_ID = "ids01" +HTTP_BATCH_SIZE = 10 +HTTP_TIMEOUT_S = 5.0 + +UDP_KEYS = frozenset({"cef", "syslog", "acmegw"}) +HTTP_KEYS = frozenset({"json"}) +CORPUS_KEYS = sorted(UDP_KEYS | HTTP_KEYS) + + +def transport_for(key: str) -> str: + if key in UDP_KEYS: + return "udp" + if key in HTTP_KEYS: + return "http" + raise ValueError(f"unknown corpus key {key!r} (expected one of {CORPUS_KEYS})") + + +def parse_udp_addr(raw: str) -> tuple[str, int]: + host, sep, port = raw.rpartition(":") + if not sep or not host: + raise ValueError(f"invalid COLLECTOR_UDP {raw!r} (expected host:port)") + try: + return host, int(port) + except ValueError: + raise ValueError(f"invalid COLLECTOR_UDP {raw!r} (bad port)") from None + + +class UdpSender: + """One log line per datagram. Socket is created lazily (nothing sent, nothing bound).""" + + def __init__(self, addr: tuple[str, int], sock: socket.socket | None = None): + self.addr = addr + self._sock = sock + + def _ensure_sock(self) -> socket.socket: + if self._sock is None: + self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + return self._sock + + async def send(self, line: str) -> int: + self._ensure_sock().sendto(line.encode("utf-8"), self.addr) + return 1 + + async def flush(self) -> int: + return 0 + + def close(self) -> None: + if self._sock is not None: + self._sock.close() + self._sock = None + + +class HttpSender: + """Buffers lines and POSTs batches to /v1/ingest; events count only on a 202.""" + + def __init__(self, base_url: str, source_id: str = HTTP_SOURCE_ID, + batch_size: int = HTTP_BATCH_SIZE, + client: httpx.AsyncClient | None = None): + self.url = base_url.rstrip("/") + "/v1/ingest" + self.source_id = source_id + self.batch_size = batch_size + self._client = client if client is not None else httpx.AsyncClient(timeout=HTTP_TIMEOUT_S) + self._buf: list[str] = [] + + async def send(self, line: str) -> int: + self._buf.append(line) + if len(self._buf) >= self.batch_size: + return await self.flush() + return 0 + + async def flush(self) -> int: + if not self._buf: + return 0 + lines, self._buf = self._buf, [] + resp = await self._client.post(self.url, json={"source_id": self.source_id, "lines": lines}) + if resp.status_code != 202: + print(f"error: http {resp.status_code} from {self.url} " + f"({len(lines)} lines dropped)", file=sys.stderr) + return 0 + try: + return int(resp.json().get("accepted", len(lines))) + except Exception: + return len(lines) # 202 but unparseable body: trust the transport + + async def close(self) -> None: + await self._client.aclose() + + +@dataclass +class Corpus: + name: str + lines: list[str] + sender: object # UdpSender / HttpSender / test double: async send(line)->int, flush()->int + + +def load_lines(path: Path) -> list[str]: + lines = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + lines.append(line) + return lines + + +def load_corpora(corpus_dir: Path | str, senders: dict) -> list[Corpus]: + corpus_dir = Path(corpus_dir) + corpora = [] + for key in CORPUS_KEYS: + path = corpus_dir / f"raw_logs_{key}.txt" + if not path.is_file(): + print(f"warning: corpus file missing, skipping: {path}", file=sys.stderr) + continue + sender = senders.get(key) + if sender is None: + print(f"warning: no sender wired for corpus {key!r}, skipping", file=sys.stderr) + continue + lines = load_lines(path) + if not lines: + print(f"warning: corpus {key!r} has no lines, skipping", file=sys.stderr) + continue + corpora.append(Corpus(name=key, lines=lines, sender=sender)) + return corpora + + +def default_senders() -> dict: + """Wire every corpus key to its transport per R7, from COLLECTOR_UDP / COLLECTOR_HTTP.""" + udp_addr = parse_udp_addr(os.environ.get("COLLECTOR_UDP", "localhost:5514")) + http_base = os.environ.get("COLLECTOR_HTTP", DEFAULT_HTTP_BASE) + udp = UdpSender(udp_addr) + senders = {key: udp for key in sorted(UDP_KEYS)} + senders["json"] = HttpSender(http_base) + return senders + + +async def run(corpora: list[Corpus], *, eps: float, duration: float, + loop: bool = False) -> tuple[dict, int]: + """Send the corpora round-robin at eps total events/sec. + + Returns (sent_per_corpus, error_count). A send counts only when the + transport accepted it (UDP sendto returned; HTTP got a 202). + """ + sent = {c.name: 0 for c in corpora} + errors = 0 + n = len(corpora) + if n == 0 or eps <= 0: + return sent, errors + + tick = n / eps # one round-robin pass per tick; n events per pass -> eps total + start = time.monotonic() + deadline = start + max(duration, 0.0) + cursors = [0] * n + + pass_no = 0 + while True: + if loop and time.monotonic() >= deadline: + break + if not loop and all(cursors[i] >= len(corpora[i].lines) for i in range(n)): + break + for i, corpus in enumerate(corpora): + if cursors[i] >= len(corpus.lines): + if not loop: + continue + cursors[i] = 0 + line = corpus.lines[cursors[i]] + cursors[i] += 1 + try: + sent[corpus.name] += await corpus.sender.send(line) + except Exception as exc: + errors += 1 + print(f"error: {corpus.name}: {exc}", file=sys.stderr) + pass_no += 1 + delay = start + pass_no * tick - time.monotonic() + if delay > 0: + await asyncio.sleep(delay) + + # Drain sender-side buffers (HTTP batches) so nothing simulated is silently dropped. + for corpus in corpora: + try: + sent[corpus.name] += await corpus.sender.flush() + except Exception as exc: + errors += 1 + print(f"error: {corpus.name} flush: {exc}", file=sys.stderr) + return sent, errors + + +async def _close_all(senders: dict) -> None: + for sender in senders.values(): + result = sender.close() + if inspect.isawaitable(result): + await result + + +def _parse_args(argv: list[str] | None) -> argparse.Namespace: + p = argparse.ArgumentParser( + prog="simulate.py", + description="Replay the golden corpora against the collector at a steady rate.") + p.add_argument("--eps", type=float, default=20.0, + help="total events/sec across all corpora (default 20)") + p.add_argument("--duration", type=float, default=60.0, + help="seconds to run when --loop is set (default 60)") + p.add_argument("--corpora", default=None, + help="directory holding raw_logs_*.txt (default: simulator/data)") + p.add_argument("--loop", action="store_true", + help="repeat the corpora until --duration elapses") + return p.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + if args.eps <= 0: + print(f"error: --eps must be > 0 (got {args.eps})", file=sys.stderr) + return 1 + corpus_dir = Path(args.corpora) if args.corpora else Path(__file__).resolve().parent / "data" + + try: + senders = default_senders() + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + corpora = load_corpora(corpus_dir, senders) + if not corpora: + print(f"error: no corpora found in {corpus_dir}", file=sys.stderr) + asyncio.run(_close_all(senders)) + return 1 + + async def drive(): + try: + return await run(corpora, eps=args.eps, duration=args.duration, loop=args.loop) + finally: + await _close_all(senders) + + sent, errors = asyncio.run(drive()) + + for name in sorted(sent): + print(f"{name} sent={sent[name]}") + total = sum(sent.values()) + print(f"total sent={total}") + + if total == 0 and errors > 0: + print(f"error: all {errors} sends failed; is the collector up?", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/simulator/tests/__init__.py b/simulator/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/simulator/tests/test_simulate.py b/simulator/tests/test_simulate.py new file mode 100644 index 0000000..595a8c7 --- /dev/null +++ b/simulator/tests/test_simulate.py @@ -0,0 +1,354 @@ +# simulator/tests/test_simulate.py — simulator unit tests (fake senders, no network). +# Routing, source tagging (R7: HTTP only), pacing floor, round-robin order, exit codes. +import asyncio +import json as jsonlib +import re +import time + +import httpx +import pytest + +from simulator.simulate import ( + HTTP_SOURCE_ID, + Corpus, + HttpSender, + UdpSender, + default_senders, + load_corpora, + main, + run, + transport_for, +) + +# --- fakes (capture destination + payload; no sockets, no collector) ----------- + +class FakeSock: + def __init__(self): + self.sent = [] # (payload_bytes, addr) + + def sendto(self, data, addr): + self.sent.append((data, addr)) + + +class FakeUdp: + """Stands in for UdpSender; records every line in a shared event log.""" + + def __init__(self, name, log, fail_every=0): + self.name = name + self.log = log + self.fail_every = fail_every + self.lines = [] + self.calls = 0 + + async def send(self, line): + self.calls += 1 + if self.fail_every and self.calls % self.fail_every == 0: + raise OSError("udp send failed") + self.lines.append(line) + self.log.append((self.name, line)) + return 1 + + async def flush(self): + return 0 + + def close(self): + pass + + +class FakeHttp: + """Stands in for HttpSender; behaves like it (buffer, flush on demand).""" + + def __init__(self, name, log, batch_size=10, status=202): + self.name = name + self.log = log + self.batch_size = batch_size + self.status = status + self.batches = [] # list of posted line-lists + self.buf = [] + + async def send(self, line): + self.buf.append(line) + if len(self.buf) >= self.batch_size: + return await self.flush() + return 0 + + async def flush(self): + if not self.buf: + return 0 + lines, self.buf = self.buf, [] + if self.status != 202: + self.log.append((self.name, f"ERROR:{self.status}")) + return 0 + self.batches.append(lines) + for line in lines: + self.log.append((self.name, line)) + return len(lines) + + async def close(self): + pass + + +def make_corpora(dir, contents): + for name, text in contents.items(): + (dir / f"raw_logs_{name}.txt").write_text(text, encoding="utf-8") + + +def do_run(corpora, **kw): + return asyncio.run(run(corpora, **kw)) + + +# --- routing table ------------------------------------------------------------- + +def test_transport_routing_table(): + assert transport_for("cef") == "udp" + assert transport_for("syslog") == "udp" + assert transport_for("acmegw") == "udp" + assert transport_for("json") == "http" + with pytest.raises(ValueError): + transport_for("zenwall") + + +# --- UDP sender: raw datagram, no envelope (R7: no source tagging on UDP) ------ + +def test_udp_sender_sends_raw_line_to_destination(): + sock = FakeSock() + sender = UdpSender(("10.9.9.9", 5514), sock=sock) + line = "CEF:0|PaloAlto|PAN-OS|10.1|THREAT|spyware|5|src=203.0.113.45" + sent = asyncio.run(sender.send(line)) + assert sent == 1 + payload, addr = sock.sent[0] + assert addr == ("10.9.9.9", 5514) + # R7: the datagram IS the log line — no JSON envelope, no source_id field. + assert payload == line.encode("utf-8") + assert b"source_id" not in payload + assert b"fw01" not in payload + + +def test_udp_sender_socket_is_lazy_and_close_is_idempotent(): + sender = UdpSender(("localhost", 5514)) # no socket created yet + sender.close() # must not raise + assert sender._sock is None + + +# --- HTTP sender: source tagging ids01, batching, 202-gated counting ----------- + +def test_http_sender_posts_batches_tagged_ids01_in_order(): + reqs = [] + + def handler(request: httpx.Request) -> httpx.Response: + reqs.append(request) + body = jsonlib.loads(request.read()) + return httpx.Response(202, json={"accepted": len(body["lines"])}) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + sender = HttpSender("http://collector:8080/", batch_size=2, client=client) + lines = ['{"a":1}', '{"a":2}', '{"a":3}'] + + async def drive(): + n1 = await sender.send(lines[0]) + n2 = await sender.send(lines[1]) # batch_size=2 -> flush here + await sender.send(lines[2]) # buffers the third line + n3 = await sender.flush() # drains it as a second POST + await sender.close() + return n1, n2, n3 + + n1, n2, n3 = asyncio.run(drive()) + assert (n1, n2, n3) == (0, 2, 1) + assert len(reqs) == 2 + first = reqs[0] + assert str(first.url) == "http://collector:8080/v1/ingest" + assert first.method == "POST" + body = jsonlib.loads(first.read()) + assert body["source_id"] == HTTP_SOURCE_ID == "ids01" + assert body["lines"] == ['{"a":1}', '{"a":2}'] # order preserved + assert jsonlib.loads(reqs[1].read())["lines"] == ['{"a":3}'] + + +def test_http_sender_counts_only_on_202(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(500, text="boom") + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + sender = HttpSender("http://collector:8080", batch_size=1, client=client) + + async def drive(): + n = 0 + n += await sender.send('{"a":1}') + n += await sender.send('{"a":2}') + await sender.close() + return n + + assert asyncio.run(drive()) == 0 # failures counted nowhere + + +def test_http_sender_default_endpoint_and_batch_size(): + sender = HttpSender("http://collector:8080") + assert sender.url == "http://collector:8080/v1/ingest" + assert sender.batch_size == 10 + assert sender.source_id == "ids01" + + +# --- load_corpora ---------------------------------------------------------------- + +def test_load_corpora_reads_files_binds_senders_skips_missing(tmp_path, capsys): + make_corpora(tmp_path, {"cef": "c1\nc2\n", "json": 'j1\nj2\n'}) + senders = {"cef": FakeUdp("cef", []), "json": FakeHttp("json", [])} + corpora = load_corpora(tmp_path, senders) + assert [c.name for c in corpora] == ["cef", "json"] + assert corpora[0].lines == ["c1", "c2"] + assert corpora[1].sender is senders["json"] + # no syslog/acmegw files -> skipped with a stderr note, not an error + assert "syslog" in capsys.readouterr().err + + +def test_load_corpora_skips_blank_lines(tmp_path): + make_corpora(tmp_path, {"cef": "c1\n\n \nc2\n"}) + corpora = load_corpora(tmp_path, {"cef": FakeUdp("cef", [])}) + assert corpora[0].lines == ["c1", "c2"] + + +def test_load_corpora_empty_dir_is_no_corpora(tmp_path): + assert load_corpora(tmp_path, {}) == [] + + +# --- run(): pacing floor, round-robin, loop -------------------------------------- + +def udp_corpus(name, lines, log): + return Corpus(name=name, lines=list(lines), sender=FakeUdp(name, log)) + + +def test_pacing_floor_eps4_duration1_at_least_4_sends(): + log = [] + corpora = [ + udp_corpus("acmegw", ["a1", "a2", "a3"], log), + udp_corpus("cef", ["c1", "c2", "c3"], log), + udp_corpus("json", ["j1", "j2", "j3"], log), + udp_corpus("syslog", ["s1", "s2", "s3"], log), + ] + t0 = time.monotonic() + sent, errors = do_run(corpora, eps=4, duration=1, loop=True) + elapsed = time.monotonic() - t0 + assert sum(sent.values()) >= 4 # brief: eps=4, duration=1 -> >=4 sends + assert errors == 0 + assert elapsed < 5 # sanity: did not hang + + +def test_round_robin_interleaves_corpora_until_exhausted(): + log = [] + corpora = [ + udp_corpus("cef", ["c1", "c2", "c3"], log), + udp_corpus("syslog", ["s1", "s2"], log), + udp_corpus("acmegw", ["a1"], log), + ] + sent, errors = do_run(corpora, eps=100, duration=1, loop=False) + assert errors == 0 + # one pass sends one line per corpus in list order; shorter corpora drop out + assert [name for name, _ in log] == [ + "cef", "syslog", "acmegw", # pass 0 + "cef", "syslog", # pass 1 + "cef", # pass 2 + ] + assert [line for _, line in log] == ["c1", "s1", "a1", "c2", "s2", "c3"] + assert sent == {"cef": 3, "syslog": 2, "acmegw": 1} + + +def test_loop_repeats_corpora_until_duration_elapses(): + log = [] + corpora = [udp_corpus("cef", ["c1", "c2"], log)] + sent, errors = do_run(corpora, eps=50, duration=0.25, loop=True) + assert errors == 0 + assert sent["cef"] > 2 # wrapped past the end at least once + assert log[0] == ("cef", "c1") and log[1] == ("cef", "c2") + assert log[2] == ("cef", "c1") # restarts from the top + + +def test_run_counts_send_failures_as_errors_not_sends(): + log = [] + flaky = FakeUdp("cef", log, fail_every=2) # every 2nd send raises + sent, errors = do_run([Corpus("cef", ["c1", "c2", "c3", "c4"], flaky)], + eps=100, duration=1, loop=False) + assert sent == {"cef": 2} + assert errors == 2 + + +def test_run_json_goes_to_http_sender_only(): + log = [] + http = FakeHttp("json", log, batch_size=100) + udp = FakeUdp("cef", log) + sent, errors = do_run( + [Corpus("cef", ["c1"], udp), Corpus("json", ["j1", "j2"], http)], + eps=100, duration=1, loop=False, + ) + assert errors == 0 + assert sent == {"cef": 1, "json": 2} + assert http.batches == [["j1", "j2"]] # one POST at final flush + assert [name for name, _ in log] == ["cef", "json", "json"] + + +# --- main(): stdout contract, exit codes, env wiring ------------------------------ + +def test_main_prints_one_summary_line_per_corpus_plus_total(tmp_path, capsys, monkeypatch): + make_corpora(tmp_path, {"cef": "c1\nc2\n", "json": "j1\n"}) + log = [] + monkeypatch.setattr( + "simulator.simulate.default_senders", + lambda: {"cef": FakeUdp("cef", log), "json": FakeHttp("json", log, batch_size=10)}, + ) + rc = main(["--eps", "100", "--duration", "1", "--corpora", str(tmp_path)]) + out = capsys.readouterr().out.strip().splitlines() + assert rc == 0 + assert out == ["cef sent=2", "json sent=1", "total sent=3"] + for line in out: + assert re.fullmatch(r"(?:\S+ sent=\d+|total sent=\d+)", line) + + +def test_main_exit_1_when_no_corpora(tmp_path, capsys): + rc = main(["--eps", "10", "--duration", "1", "--corpora", str(tmp_path)]) + assert rc == 1 + assert "no corpora" in capsys.readouterr().err + + +def test_main_exit_1_on_total_send_failure_but_still_prints_summary(tmp_path, capsys, monkeypatch): + make_corpora(tmp_path, {"cef": "c1\n"}) + + class DeadUdp(FakeUdp): + async def send(self, line): + raise OSError("cannot resolve collector") + + monkeypatch.setattr("simulator.simulate.default_senders", + lambda: {"cef": DeadUdp("cef", [])}) + rc = main(["--eps", "10", "--duration", "1", "--corpora", str(tmp_path)]) + out = capsys.readouterr() + assert rc == 1 + assert "cef sent=0" in out.out and "total sent=0" in out.out + assert "cannot resolve collector" in out.err + + +def test_main_exit_0_on_partial_send_failure(tmp_path, capsys, monkeypatch): + make_corpora(tmp_path, {"cef": "c1\nc2\n"}) + monkeypatch.setattr("simulator.simulate.default_senders", + lambda: {"cef": FakeUdp("cef", [], fail_every=2)}) + rc = main(["--eps", "100", "--duration", "1", "--corpora", str(tmp_path)]) + out = capsys.readouterr() + assert rc == 0 + assert "cef sent=1" in out.out and "total sent=1" in out.out + + +def test_default_senders_reads_env(monkeypatch): + monkeypatch.setenv("COLLECTOR_UDP", "collector-host:5999") + monkeypatch.setenv("COLLECTOR_HTTP", "http://collector-host:9999") + senders = default_senders() + assert set(senders) == {"cef", "syslog", "acmegw", "json"} + assert senders["cef"].addr == ("collector-host", 5999) + assert senders["json"].url == "http://collector-host:9999/v1/ingest" + for s in senders.values(): + if isinstance(s, UdpSender): + s.close() # UDP sockets are lazy; HTTP clients close via gc (no request made) + + +def test_default_senders_env_defaults(monkeypatch): + monkeypatch.delenv("COLLECTOR_UDP", raising=False) + monkeypatch.delenv("COLLECTOR_HTTP", raising=False) + senders = default_senders() + assert senders["cef"].addr == ("localhost", 5514) + assert senders["json"].url == "http://localhost:8080/v1/ingest" From c40d260ff7a2b5fc9254b67168af4cece10e0553 Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 17:22:02 +0530 Subject: [PATCH 14/18] =?UTF-8?q?feat:=20M1=20acceptance=20=E2=80=94=20e2e?= =?UTF-8?q?=20smoke=20with=20replay=20idempotency,=20CI,=20quickstart?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/smoke.py drives the live stack: lossless ingest (delta-based, DB never reset), per-fingerprint parsed/unparsed split via /api/stats, raw traceability (HTTP /raw + SQL join with recomputed sha256), forced Kafka replay (group deleted after the consumer stops, pipeline restarted: raw_events unchanged, raw_batches grows), and an independent recomputation of every raw_batches merkle root plus full per-partition chain linkage from genesis. Also: R19 hardening — _is_xml catches RecursionError alongside ParseError so a deeply-nested hostile line falls through to the shape hash instead of wedging a pipeline batch; redpanda-init.sh creates missing topics via list-then-create (v24.1.7 rpk has no --if-not-exists flag); Postgres publishes on host 5433 so host connections cannot land on a native Postgres owning 5432; ruff made repo-clean for CI (frozen legacy-demo excluded; import-order, blind-except and small dead-variable fixes); CI workflow (py3.12 ruff+pytest, node 22 web). Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 42 ++ README.md | 34 +- bench/fingerprint_fragmentation.py | 8 +- deploy/docker-compose.yml | 5 +- deploy/migrate.py | 41 +- deploy/redpanda-init.sh | 21 +- libs/ulpf-core/tests/test_fingerprint.py | 22 + libs/ulpf-core/tests/test_ids.py | 4 +- libs/ulpf-core/tests/test_models.py | 6 +- libs/ulpf-core/tests/test_parsing.py | 4 +- libs/ulpf-core/ulpf_core/fingerprint.py | 15 +- libs/ulpf-core/ulpf_core/models.py | 1 + pyproject.toml | 3 + scripts/smoke.py | 474 +++++++++++++++++++++ services/collector/app.py | 6 +- services/collector/tests/test_collector.py | 6 +- simulator/simulate.py | 6 +- 17 files changed, 650 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 scripts/smoke.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..88dcfba --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +# M1 CI (Task 12). Unit tests only — no container services; the live E2E gate +# is `python scripts/smoke.py`, run locally against the compose stack. +name: ci + +on: [push, pull_request] + +jobs: + python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install libs, services, and test deps + run: | + pip install -e libs/ulpf-core -e libs/ocsf-schema + pip install -r services/collector/requirements.txt \ + -r services/pipeline/requirements.txt \ + -r services/gateway/requirements.txt + pip install pytest ruff + - name: Lint + run: ruff check . + - name: Unit tests (libs + services + simulator) + run: pytest + + web: + runs-on: ubuntu-latest + defaults: + run: + working-directory: web + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Install + run: npm ci || npm i + - name: Test + run: npm run test --if-present + - name: Build + run: npm run build diff --git a/README.md b/README.md index ed8ff9d..904e6f5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # ULPF — Universal Log Pre-processing Framework -SIH 2026 PS 26156 MVP. Status: M1 walking skeleton in development. +SIH 2026 PS 26156 MVP. Status: M1 walking skeleton complete — the milestone +exit is `python scripts/smoke.py` (all 5 checks PASS). - `services/` — collector, pipeline, gateway (+ onboarding, drift in M2/M3) - `libs/ulpf-core` — fingerprinting, parsing, IDs, hash chain (single implementation) @@ -8,4 +9,35 @@ SIH 2026 PS 26156 MVP. Status: M1 walking skeleton in development. - `web/` — React dashboard - `simulator/` — multi-vendor traffic player - `deploy/` — docker-compose, migrations +- `scripts/smoke.py` — M1 acceptance gate (live E2E incl. forced replay) - `legacy-demo/` — the original hackathon demo, frozen. See its README. + +## Quickstart (M1) + +```bash +cd deploy && docker compose --profile sim up -d --build +python scripts/smoke.py # end-to-end acceptance incl. forced replay +open http://localhost:3000 # Overview: live feed + raw traceability +``` + +The smoke runs the simulator over the golden corpora, then verifies: lossless +ingest (raw delta == sent), the parsed/unparsed split per fingerprint, raw +traceability (API + SQL join with recomputed sha256), forced replay +idempotency (Kafka consumer-group delete + pipeline restart: raw events +unchanged, raw_batches grown), and an independent recomputation of every +raw_batches merkle root plus full per-partition chain linkage. + +Smoke host deps (Python 3.13 tested): `confluent-kafka~=2.6`, `psycopg[binary]`. +Do NOT install the services' `confluent-kafka==2.5.0` pin host-side — no cp313 +wheels; the images themselves run it fine on Python 3.12. The smoke is +delta-based: the database is never reset, so re-runs accumulate and re-verify +the full accumulated chain every time. + +Ports: web 3000, gateway 8000, Postgres published on host **5433** — 5432 is +deliberately left to any native Postgres on the dev box (host connections to +5432 would silently hit that one). Inside the compose network everything still +uses `postgres:5432`. + +CI (`.github/workflows/ci.yml`): `ruff check .` + `pytest` (libs, services, +simulator) on Python 3.12; web `npm ci`, vitest, `npm run build` on Node 22. +The dashboard is React 19.2 + Vite 8.3 + TypeScript 6. diff --git a/bench/fingerprint_fragmentation.py b/bench/fingerprint_fragmentation.py index f0be2ad..57d4ce1 100644 --- a/bench/fingerprint_fragmentation.py +++ b/bench/fingerprint_fragmentation.py @@ -197,9 +197,11 @@ def write_doc(report: dict) -> None: out: list[str] = [ "# M1 fragmentation bench (spec §9)", "", - f"Generated by `python bench/fingerprint_fragmentation.py` " - f"(seed {report['seed']}, {report['mutations']} mutants per corpus). " - "Do not edit by hand; re-run to regenerate.", + ( + f"Generated by `python bench/fingerprint_fragmentation.py` " + f"(seed {report['seed']}, {report['mutations']} mutants per corpus). " + "Do not edit by hand; re-run to regenerate." + ), "", "## Method", "", diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 9bac225..c3f1372 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -4,7 +4,10 @@ services: postgres: image: postgres:16 environment: { POSTGRES_USER: postgres, POSTGRES_PASSWORD: postgres, POSTGRES_DB: ulpf } - ports: ["5432:5432"] + # Host port 5433, NOT 5432: dev boxes with a native Postgres service own + # 5432, and host connections would silently hit that one instead. Only the + # published port changes; the compose network keeps postgres:5432. + ports: ["5433:5432"] volumes: [pgdata:/var/lib/postgresql/data] healthcheck: { test: ["CMD-SHELL", "pg_isready -U postgres -d ulpf"], interval: 3s, retries: 20 } migrate: diff --git a/deploy/migrate.py b/deploy/migrate.py index a8f6cd4..56b1fae 100644 --- a/deploy/migrate.py +++ b/deploy/migrate.py @@ -37,29 +37,28 @@ def main() -> int: # autocommit=True so each conn.transaction() below is a real top-level # transaction that commits per file, independently of later files. - with psycopg.connect(admin_url, autocommit=True) as conn: - with conn.cursor() as cur: - cur.execute( - "CREATE TABLE IF NOT EXISTS schema_migrations (" - "name TEXT PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())" - ) - cur.execute("SELECT name FROM schema_migrations") - applied = {row[0] for row in cur.fetchall()} + with psycopg.connect(admin_url, autocommit=True) as conn, conn.cursor() as cur: + cur.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + "name TEXT PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())" + ) + cur.execute("SELECT name FROM schema_migrations") + applied = {row[0] for row in cur.fetchall()} - pending = [f for f in files if f.name not in applied] - if not pending: - print("migrations: up to date") - return 0 + pending = [f for f in files if f.name not in applied] + if not pending: + print("migrations: up to date") + return 0 - for path in pending: - print(f"applying {path.name}") - with conn.transaction(): - cur.execute(path.read_text(encoding="utf-8")) - cur.execute( - "INSERT INTO schema_migrations (name) VALUES (%s)", - (path.name,), - ) - print(f"applied {path.name}") + for path in pending: + print(f"applying {path.name}") + with conn.transaction(): + cur.execute(path.read_text(encoding="utf-8")) + cur.execute( + "INSERT INTO schema_migrations (name) VALUES (%s)", + (path.name,), + ) + print(f"applied {path.name}") return 0 diff --git a/deploy/redpanda-init.sh b/deploy/redpanda-init.sh index 115c51e..723cfc7 100644 --- a/deploy/redpanda-init.sh +++ b/deploy/redpanda-init.sh @@ -1,7 +1,18 @@ #!/bin/sh set -e -# --if-not-exists (controller R10): any stack re-run against a surviving broker -# must not fail the topics service and block depends_on consumers. -rpk topic create --if-not-exists raw.logs -p 6 -rpk topic create --if-not-exists normalized.events -p 6 -rpk topic create --if-not-exists pipeline.dlq -p 3 +# Idempotent (controller R10): any stack re-run against a surviving broker must +# not fail the topics service and block depends_on consumers. This image's rpk +# (redpanda v24.1.7) has no `topic create --if-not-exists` flag, so list the +# existing topics and create only the missing ones. +existing=$(rpk topic list | tail -n +2 | awk '{print $1}' | tr '\n' ' ') +create() { + topic=$1 + partitions=$2 + case " $existing " in + *" $topic "*) ;; + *) rpk topic create "$topic" -p "$partitions" ;; + esac +} +create raw.logs 6 +create normalized.events 6 +create pipeline.dlq 3 diff --git a/libs/ulpf-core/tests/test_fingerprint.py b/libs/ulpf-core/tests/test_fingerprint.py index a553b1b..237e19f 100644 --- a/libs/ulpf-core/tests/test_fingerprint.py +++ b/libs/ulpf-core/tests/test_fingerprint.py @@ -1,6 +1,7 @@ # libs/ulpf-core/tests/test_fingerprint.py import re from pathlib import Path + from ulpf_core.fingerprint import fingerprint_id, shape_hash GOLDEN = Path(__file__).parent / "data" / "golden" @@ -38,3 +39,24 @@ def test_mutated_hostnames_and_numbers_stable(): def test_csv_and_xml_detected(): assert fingerprint_id("a,b,c,1,2,3") == "csv" assert fingerprint_id("1.2.3.4") == "xml" + +def test_deeply_nested_xml_cannot_wedge_fingerprint(monkeypatch): + # R19 regression: on recursion-limited ElementTree builds the probe over a + # deeply-nested hostile line raises RecursionError instead of ParseError. + # RecursionError is not a ParseError, so an uncaught one escapes + # fingerprint_id and wedges the pipeline batch (redelivery loop). It must + # classify as "not xml" and fall through to the shape-hash path. + import xml.etree.ElementTree as ET + + def recurse_forever(text): + raise RecursionError("maximum recursion depth exceeded while parsing") + + monkeypatch.setattr(ET, "fromstring", recurse_forever) + assert re.fullmatch(r"auto_[0-9a-f]{8}", fingerprint_id("" * 100000)) + +def test_hostile_nested_xml_line_is_stable(): + # The literal hostile input: never raises, and the same line always gets + # the same auto_ id (the truncation probe stays intact). + line = "" * 100000 + assert fingerprint_id(line).startswith("auto_") + assert fingerprint_id(line) == fingerprint_id(line) diff --git a/libs/ulpf-core/tests/test_ids.py b/libs/ulpf-core/tests/test_ids.py index cdf2843..deb632f 100644 --- a/libs/ulpf-core/tests/test_ids.py +++ b/libs/ulpf-core/tests/test_ids.py @@ -1,5 +1,7 @@ import uuid -from ulpf_core.ids import raw_id_for, event_id_for + +from ulpf_core.ids import event_id_for, raw_id_for + def test_raw_id_deterministic(): assert raw_id_for("raw.logs", 3, 9182) == raw_id_for("raw.logs", 3, 9182) diff --git a/libs/ulpf-core/tests/test_models.py b/libs/ulpf-core/tests/test_models.py index 510b2ca..8c252e0 100644 --- a/libs/ulpf-core/tests/test_models.py +++ b/libs/ulpf-core/tests/test_models.py @@ -1,13 +1,15 @@ import pytest +from pydantic import ValidationError from ulpf_core.models import RawEnvelope + def test_envelope_roundtrip(): e = RawEnvelope(source_id="fw01", transport="syslog-udp", received_at="2026-09-19T10:00:00+00:00", raw="Aug 27 14:32:07 fw01 kernel: X") assert RawEnvelope.model_validate_json(e.model_dump_json()) == e def test_envelope_rejects_empty_and_oversized(): - with pytest.raises(Exception): + with pytest.raises(ValidationError): RawEnvelope(source_id="s", transport="http", received_at="t", raw="") - with pytest.raises(Exception): + with pytest.raises(ValidationError): RawEnvelope(source_id="s", transport="http", received_at="t", raw="x" * 70000) diff --git a/libs/ulpf-core/tests/test_parsing.py b/libs/ulpf-core/tests/test_parsing.py index f182c59..251ca9e 100644 --- a/libs/ulpf-core/tests/test_parsing.py +++ b/libs/ulpf-core/tests/test_parsing.py @@ -1,8 +1,8 @@ # libs/ulpf-core/tests/test_parsing.py — key cases, all with concrete data: import sys -from ulpf_core.parsing import parse, validate_rule_output, OCSF_TARGETS -from ulpf_core.models import Rule, Mapping +from ulpf_core.models import Mapping, Rule +from ulpf_core.parsing import OCSF_TARGETS, parse, validate_rule_output CEF_RULE = Rule(fingerprint_id="cef_paloalto", version=1, pattern=r"^.*\|(?P.*)$", diff --git a/libs/ulpf-core/ulpf_core/fingerprint.py b/libs/ulpf-core/ulpf_core/fingerprint.py index 09d5a88..b5d870a 100644 --- a/libs/ulpf-core/ulpf_core/fingerprint.py +++ b/libs/ulpf-core/ulpf_core/fingerprint.py @@ -74,11 +74,18 @@ def _first_element_end(text: str) -> int | None: def _is_xml(text: str) -> bool: - """XML heuristic: parses as XML, or its truncated first element does.""" + """XML heuristic: parses as XML, or its truncated first element does. + + RecursionError is caught alongside ParseError: a deeply-nested hostile + line can exhaust the parser's recursion before it yields a ParseError + (recursion-limited ElementTree builds), and an escaped RecursionError + would wedge the consuming pipeline batch in a redelivery loop. Either + failure classifies as "not xml" and falls through to the shape hash. + """ try: ET.fromstring(text) return True - except ET.ParseError: + except (ET.ParseError, RecursionError): pass end = _first_element_end(text) if end is None: @@ -86,7 +93,7 @@ def _is_xml(text: str) -> bool: try: ET.fromstring(text[:end]) return True - except ET.ParseError: + except (ET.ParseError, RecursionError): return False @@ -152,7 +159,7 @@ def _known_format(line: str) -> str | None: def _classify(token: str) -> str: if "=" in token: - key, _, value = token.partition("=") + _, _, value = token.partition("=") # An empty value ("OUT=", "FLAGS=") classifies like a word value so a # field that is present-but-empty hashes the same as one with a value. return f"KV:{_classify(value)}" diff --git a/libs/ulpf-core/ulpf_core/models.py b/libs/ulpf-core/ulpf_core/models.py index 6b0e135..ee25be7 100644 --- a/libs/ulpf-core/ulpf_core/models.py +++ b/libs/ulpf-core/ulpf_core/models.py @@ -1,5 +1,6 @@ from pydantic import BaseModel, Field + class RawEnvelope(BaseModel): source_id: str transport: str # "syslog-udp" | "syslog-tcp" | "http" diff --git a/pyproject.toml b/pyproject.toml index 5f1e850..80cfa3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,6 @@ pythonpath = [".", "services/collector", "services/pipeline", "services/gateway" [tool.ruff] line-length = 100 target-version = "py312" +# legacy-demo/ is the frozen hackathon demo (out of M1 scope; its suite must +# never even be collected) — not held to this repo's lint rules. +extend-exclude = ["legacy-demo"] diff --git a/scripts/smoke.py b/scripts/smoke.py new file mode 100644 index 0000000..c94452c --- /dev/null +++ b/scripts/smoke.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python3 +"""ULPF M1 acceptance smoke (Task 12, spec section 15) — the milestone exit gate. + +End-to-end against the live compose stack: drives the simulator over the golden +corpora, then verifies lossless ingest, the parsed/unparsed split, raw +traceability, forced replay idempotency (consumer-group delete + pipeline +restart), and independently re-verifies EVERY raw_batches merkle root plus the +per-partition hash chain linkage. + +Host requirements (Python 3.13 tested): confluent-kafka~=2.6, psycopg[binary]. +Everything else is stdlib (urllib for HTTP, subprocess for docker compose). +Docker commands run from deploy/; the DB is NOT reset — every count assertion +is delta-based (snapshot before, diff after), so re-runs accumulate. + +Usage: python scripts/smoke.py (from the repo root) +Exit: 0 all checks PASS, 1 any FAIL. +""" + +from __future__ import annotations + +import hashlib +import json +import random +import re +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +import psycopg + +# pipeline.hashchain lives at services/pipeline (package dir named `pipeline`); +# services/ on sys.path is what makes `import pipeline` resolve. +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "services")) +from pipeline.hashchain import GENESIS_PREV_HASH, merkle_root + +ROOT = Path(__file__).resolve().parents[1] +DEPLOY = ROOT / "deploy" +DSN = "postgresql://postgres:postgres@localhost:5433/ulpf" # admin, host port (5433: compose publish; 5432 often owned by a native Postgres) +GATEWAY = "http://localhost:8000" +WEB = "http://localhost:3000" + +SIM_EPS = "25" +SIM_DURATION = "20" # ~500 events: corpora total 57 lines -> several loops +SEEDED_FPS = ("cef_paloalto", "cef_ciscoasa", "cef_fortigate", "syslog", "json") +TRACE_SAMPLE = 20 + +BUILD_TIMEOUT_S = 900 +READY_TIMEOUT_S = 240 +SIM_TIMEOUT_S = 300 +DRAIN_TIMEOUT_S = 150 +LAG_TIMEOUT_S = 180 +POLL_INTERVAL_S = 2.0 + +_SIM_LINE = re.compile(r"^(\w+) sent=(\d+)$", re.MULTILINE) +_LAG_LINE = re.compile(r"^TOTAL-LAG\s+(\S+)\s*$", re.MULTILINE) + +results: list[tuple[str, bool, str]] = [] + + +def check(name: str, ok: bool, detail: str) -> bool: + results.append((name, ok, detail)) + print(f"[{'PASS' if ok else 'FAIL'}] {name}: {detail}") + return ok + + +def compose(*args: str, timeout: float = 300) -> subprocess.CompletedProcess: + return subprocess.run( + ["docker", "compose", *args], cwd=DEPLOY, capture_output=True, + text=True, timeout=timeout, check=False, + ) + + +def http_get(url: str, timeout: float = 10) -> tuple[int, bytes]: + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status, resp.read() + except urllib.error.HTTPError as exc: + return exc.code, b"" + + +def http_json(url: str, timeout: float = 10) -> tuple[int, object]: + status, raw = http_get(url, timeout=timeout) + if not raw: + return status, None + try: + return status, json.loads(raw.decode("utf-8")) + except json.JSONDecodeError: + return status, None + + +def q1(cur, sql: str, params=()): + cur.execute(sql, params) + return cur.fetchone()[0] + + +def db_count(sql: str) -> int: + with psycopg.connect(DSN) as conn, conn.cursor() as cur: + return q1(cur, sql) + + +def stats_by_fingerprint() -> dict[str, dict]: + """{fingerprint_id: {total, parsed}} from GET /api/stats (delta base).""" + status, body = http_json(f"{GATEWAY}/api/stats") + if status != 200: + raise RuntimeError(f"GET /api/stats -> {status}") + return {row["fingerprint_id"]: {"total": row["total"], "parsed": row["parsed"]} + for row in body["by_fingerprint"]} + + +def rpk(*args: str, timeout: float = 60) -> subprocess.CompletedProcess: + return compose("exec", "-T", "redpanda", "rpk", *args, timeout=timeout) + + +def group_listed(listing: subprocess.CompletedProcess) -> bool: + """True iff `rpk group list` output has a row whose GROUP column is exactly + 'pipeline' (a regex word-match would also hit e.g. 'pipeline-x').""" + return listing.returncode == 0 and any( + line.split()[-1:] == ["pipeline"] for line in listing.stdout.splitlines() if line.split() + ) + + +def wait_group_present(timeout_s: float) -> bool: + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if group_listed(rpk("group", "list")): + return True + time.sleep(POLL_INTERVAL_S) + return False + + +def wait_group_lag_zero(timeout_s: float) -> tuple[bool, str]: + """Poll `rpk group describe pipeline` until TOTAL-LAG is 0.""" + deadline = time.monotonic() + timeout_s + last = "no successful describe" + while time.monotonic() < deadline: + out = rpk("group", "describe", "pipeline") + if out.returncode == 0: + match = _LAG_LINE.search(out.stdout) + if match: + lag = match.group(1) + last = f"TOTAL-LAG={lag}" + if lag.isdigit() and int(lag) == 0: + return True, last + else: + last = f"no TOTAL-LAG line in describe output: {out.stdout[:200]!r}" + else: + last = f"describe rc={out.returncode}: {out.stderr.strip()[:120]}" + time.sleep(POLL_INTERVAL_S) + return False, last + + +def wait_drain(before_raw: int, before_norm: int, total_sent: int) -> tuple[bool, int, int]: + """Poll until this run's raw and normalized deltas both equal total_sent.""" + deadline = time.monotonic() + DRAIN_TIMEOUT_S + raw = norm = -1 + while time.monotonic() < deadline: + raw = db_count("SELECT count(*) FROM raw_events") - before_raw + norm = db_count("SELECT count(*) FROM normalized_events") - before_norm + if raw == total_sent and norm == total_sent: + return True, raw, norm + time.sleep(POLL_INTERVAL_S) + return False, raw, norm + + +def wait_http_ready(url: str, name: str, timeout_s: float) -> bool: + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + try: + status, _ = http_json(url, timeout=5) + if status == 200: + print(f" {name} ready: {url} -> 200") + return True + except (urllib.error.URLError, OSError, json.JSONDecodeError): + pass + time.sleep(2.0) + print(f" {name} NOT ready within {timeout_s:.0f}s: {url}") + return False + + +def fingerprint_split_table(before: dict, after: dict) -> str: + lines = [f" {'fingerprint':<20} {'delta':>6} {'total':>6} {'parsed':>7} {'unparsed':>9}"] + for fp in sorted(set(before) | set(after)): + total = after.get(fp, {}).get("total", 0) + parsed = after.get(fp, {}).get("parsed", 0) + delta = total - before.get(fp, {}).get("total", 0) + lines.append(f" {fp:<20} {delta:>6} {total:>6} {parsed:>7} {total - parsed:>9}") + return "\n" + "\n".join(lines) + + +def step1_up() -> None: + print("== Step 1: docker compose up -d --build + health wait ==") + out = compose("up", "-d", "--build", timeout=BUILD_TIMEOUT_S) + if out.returncode != 0: + print(out.stdout[-2000:]) + print(out.stderr[-2000:]) + sys.exit("FATAL: compose up failed; cannot run the smoke.") + ok_gw = wait_http_ready(f"{GATEWAY}/api/stats", "gateway", READY_TIMEOUT_S) + ok_web = wait_http_ready(f"{WEB}/", "web", READY_TIMEOUT_S) + ok_group = wait_group_present(READY_TIMEOUT_S) + if not (ok_gw and ok_web and ok_group): + sys.exit("FATAL: stack not ready (gateway/web/kafka group 'pipeline').") + + +def run_simulator() -> dict[str, int]: + print(f"== Step 2: simulator --eps {SIM_EPS} --duration {SIM_DURATION} ==") + out = compose("run", "--rm", "-T", "simulator", + "--eps", SIM_EPS, "--duration", SIM_DURATION, timeout=SIM_TIMEOUT_S) + sent = {name: int(n) for name, n in _SIM_LINE.findall(out.stdout)} + if out.returncode != 0 or "total" not in sent: + print("stdout:", out.stdout[-2000:]) + print("stderr:", out.stderr[-2000:]) + sys.exit("FATAL: simulator run failed or stdout contract broken.") + corpora_total = sum(n for name, n in sent.items() if name != "total") + missing = {"acmegw", "cef", "json", "syslog"} - set(sent) + if missing or corpora_total != sent["total"]: + sys.exit(f"FATAL: simulator stdout contract broken: {sent} (missing={missing})") + return sent + + +def assert_a(before_raw: int, total_sent: int) -> None: + print("== Assert A: lossless (raw_events delta == sent) ==") + raw_after = db_count("SELECT count(*) FROM raw_events") + delta = raw_after - before_raw + check("A lossless", delta == total_sent, + f"sent={total_sent} raw_events delta={delta} (before={before_raw} after={raw_after})") + + +def assert_b(before_fps: dict) -> None: + print("== Assert B: parsed split via /api/stats by_fingerprint ==") + after = stats_by_fingerprint() + details = [fingerprint_split_table(before_fps, after)] + ok = True + problems = [] + seeded_delta = 0 + for fp in SEEDED_FPS: + row = after.get(fp) + if row is None or row["total"] == 0: + ok = False + problems.append(f"{fp}: absent/empty") + continue + seeded_delta += row["total"] - before_fps.get(fp, {}).get("total", 0) + if row["parsed"] != row["total"]: + ok = False + problems.append(f"{fp}: parsed {row['parsed']}/{row['total']}") + autos = {fp: row for fp, row in after.items() if fp.startswith("auto_")} + auto_delta = sum(row["total"] - before_fps.get(fp, {}).get("total", 0) + for fp, row in autos.items()) + for fp, row in autos.items(): + if row["parsed"] != 0: + ok = False + problems.append(f"{fp}: parsed={row['parsed']} (must be 0)") + if not autos or auto_delta == 0: + ok = False + problems.append("no auto_* fingerprint saw traffic (AcmeGW unparsed path unexercised)") + total_sent_b = seeded_delta + auto_delta + details.append(f" seeded delta={seeded_delta}, auto delta={auto_delta}, sum={total_sent_b}") + check("B parsed split", ok, "; ".join(problems) if problems else + f"5 seeded fingerprints fully parsed, auto_* fully unparsed{details[0]}\n{details[1]}") + + +def assert_c() -> None: + print(f"== Assert C: traceability ({TRACE_SAMPLE} random parsed events) ==") + status, body = http_json(f"{GATEWAY}/api/events?status=parsed&limit=500") + if status != 200 or not body or len(body.get("events", [])) < TRACE_SAMPLE: + check("C traceability", False, f"GET /api/events -> {status}, " + f"{0 if not body else len(body['events'])} parsed events available") + return + events = random.sample(body["events"], TRACE_SAMPLE) + http_ok = sql_ok = 0 + problems = [] + with psycopg.connect(DSN) as conn, conn.cursor() as cur: + for ev in events: + event_id = ev["event_id"] + rstatus, raw = http_json(f"{GATEWAY}/api/events/{event_id}/raw") + if rstatus != 200: + problems.append(f"{event_id}: /raw -> {rstatus}") + continue + digest = hashlib.sha256(raw["raw_text"].encode("utf-8")).hexdigest() + if digest != raw["content_hash"]: + problems.append(f"{event_id}: HTTP content_hash mismatch") + continue + http_ok += 1 + cur.execute( + "SELECT r.content_hash, r.raw_text FROM normalized_events ne " + "JOIN raw_events r ON r.raw_id = ne.raw_id " + "AND r.received_at = ne.raw_received_at " + "WHERE ne.event_id = %s::uuid AND ne.superseded_by_event_id IS NULL", + (event_id,), + ) + row = cur.fetchone() + if row is None: + problems.append(f"{event_id}: SQL join found no raw row") + continue + if hashlib.sha256(row[1].encode("utf-8")).hexdigest() != row[0]: + problems.append(f"{event_id}: SQL content_hash mismatch") + continue + sql_ok += 1 + ok = http_ok == TRACE_SAMPLE and sql_ok == TRACE_SAMPLE + check("C traceability", ok, + f"HTTP /raw 200+hash {http_ok}/{TRACE_SAMPLE}, SQL join+hash {sql_ok}/{TRACE_SAMPLE}" + + ("" if ok else "; problems: " + "; ".join(problems[:5]))) + + +def assert_d_replay() -> None: + print("== Assert D: forced replay (group delete + pipeline restart) ==") + pre_raw = db_count("SELECT count(*) FROM raw_events") + pre_batches = db_count("SELECT count(*) FROM raw_batches") + pre_fps = stats_by_fingerprint() + + # Order matters: the pipeline is stopped BEFORE the group delete. Deleted + # while running, the live consumer's auto-rejoin recreates the group and + # re-commits its offsets within seconds, so a restart afterwards would have + # nothing to redeliver (vacuous replay, observed as raw_batches unchanged). + stop = compose("stop", "pipeline", timeout=180) + if stop.returncode != 0: + check("D replay idempotency", False, + f"compose stop pipeline failed: {stop.stderr.strip()[:200]}") + return + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: # consumer must be fully out of the group + out = rpk("group", "describe", "pipeline") + if out.returncode != 0 or re.search(r"^MEMBERS\s+0\s*$", out.stdout, re.MULTILINE): + break + time.sleep(POLL_INTERVAL_S) + print(" pipeline stopped; consumer has left the group") + + listing = rpk("group", "list") + group_present = listing.returncode == 0 and any( + line.split()[-1:] == ["pipeline"] + for line in listing.stdout.splitlines() if line.split()) + if not group_present: + check("D replay idempotency", False, + f"kafka group 'pipeline' absent before delete (replay would be vacuous): " + f"{listing.stdout.strip()!r}") + return + delete = rpk("group", "delete", "pipeline") + if delete.returncode != 0: + check("D replay idempotency", False, + f"rpk group delete failed rc={delete.returncode}: {delete.stderr.strip()[:200]}") + return + print(" group 'pipeline' deleted (committed offsets dropped); starting pipeline") + start = compose("start", "pipeline", timeout=180) + if start.returncode != 0: + check("D replay idempotency", False, f"compose start pipeline failed: " + f"{start.stderr.strip()[:200]}") + return + lag_ok, lag_detail = wait_group_lag_zero(LAG_TIMEOUT_S) + if not lag_ok: + check("D replay idempotency", False, f"group lag never reached 0: {lag_detail}") + return + print(f" replay drained ({lag_detail})") + + post_raw = db_count("SELECT count(*) FROM raw_events") + post_batches = db_count("SELECT count(*) FROM raw_batches") + post_fps = stats_by_fingerprint() + summary = ( + f"raw_events {pre_raw}->{post_raw}, raw_batches {pre_batches}->{post_batches}, " + f"split unchanged={pre_fps == post_fps}" + ) + details = [summary] + ok = (post_raw == pre_raw and post_batches > pre_batches and pre_fps == post_fps) + if pre_fps != post_fps: + details.append(fingerprint_split_table(pre_fps, post_fps)) + check("D replay idempotency", ok, + f"raw_events unchanged={post_raw == pre_raw}, raw_batches grew " + f"{pre_batches}->{post_batches}, by_fingerprint unchanged={pre_fps == post_fps}; " + + "; ".join(details)) + + +def assert_e_chain() -> None: + print("== Assert E: full per-partition chain walk + independent root recompute ==") + with psycopg.connect(DSN) as conn, conn.cursor() as cur: + cur.execute( + "SELECT partition_id, batch_seq, prev_hash, merkle_root, row_hashes, count " + "FROM raw_batches ORDER BY partition_id, batch_seq" + ) + rows = cur.fetchall() + if not rows: + check("E chain continuity", False, "raw_batches is empty") + return + chains: dict[int, list] = {} + for partition_id, batch_seq, prev_hash, root, row_hashes, count in rows: + chains.setdefault(partition_id, []).append( + (batch_seq, prev_hash, root, row_hashes, count)) + problems = [] + n_verified = 0 + for partition_id in sorted(chains): + chain = sorted(chains[partition_id]) + for position, (batch_seq, prev_hash, root, row_hashes, count) in enumerate(chain): + if position == 0: + if batch_seq != 1 or prev_hash != GENESIS_PREV_HASH: + problems.append(f"p{partition_id}: bad genesis " + f"(seq={batch_seq}, prev_hash={prev_hash[:12]}...)") + else: + if batch_seq != chain[position - 1][0] + 1: + problems.append(f"p{partition_id}: batch_seq gap at {batch_seq}") + if prev_hash != chain[position - 1][2]: + problems.append(f"p{partition_id}: prev_hash link broken at seq {batch_seq}") + if count != len(row_hashes): + problems.append(f"p{partition_id} seq {batch_seq}: count {count} != " + f"len(row_hashes) {len(row_hashes)}") + recomputed = merkle_root(list(row_hashes)) + if recomputed != root: + problems.append(f"p{partition_id} seq {batch_seq}: stored root != " + f"merkle_root(row_hashes)") + n_verified += 1 + ok = not problems + parts = ", ".join(f"p{p}={len(chains[p])} batches" for p in sorted(chains)) + check("E chain continuity", ok, + f"{n_verified} batches across {len(chains)} partition(s) " + f"({parts}); every prev_hash link + every merkle root recomputed & matched" + + ("" if ok else "; problems: " + "; ".join(problems[:5]))) + + +def wait_quiet(window_s: float = 4.0, timeout_s: float = 45.0) -> int: + """Wait until raw_events stops moving, then return its count. + + The delta base is taken only when the DB is quiescent, so stray traffic + (e.g. the quickstart's `--profile sim` one-shot still draining) cannot be + charged to this run's asserts. + """ + deadline = time.monotonic() + timeout_s + last = db_count("SELECT count(*) FROM raw_events") + last_change = time.monotonic() + while time.monotonic() < deadline: + time.sleep(2.0) + current = db_count("SELECT count(*) FROM raw_events") + if current != last: + last, last_change = current, time.monotonic() + elif time.monotonic() - last_change >= window_s: + return current + return last + + +def main() -> int: + print(f"ULPF M1 smoke - {time.strftime('%Y-%m-%d %H:%M:%S')}") + step1_up() + + print("== Snapshot before (delta base) ==") + before_raw = wait_quiet() + before_norm = db_count("SELECT count(*) FROM normalized_events") + before_fps = stats_by_fingerprint() + print(f" raw_events={before_raw} normalized_events={before_norm} " + f"fingerprints={len(before_fps)}") + + sent = run_simulator() + print(" " + ", ".join(f"{name}={n}" for name, n in sorted(sent.items()))) + + drained, raw_delta, norm_delta = wait_drain(before_raw, before_norm, sent["total"]) + if not drained: + print(f"[FAIL] drain: pipeline did not drain within {DRAIN_TIMEOUT_S:.0f}s " + f"(raw delta={raw_delta}, normalized delta={norm_delta}, sent={sent['total']})") + return 1 + print(f" drained: raw delta={raw_delta}, normalized delta={norm_delta}") + + assert_a(before_raw, sent["total"]) + assert_b(before_fps) + assert_c() + assert_d_replay() + assert_e_chain() + + failed = [name for name, ok, _ in results if not ok] + print(f"\nSMOKE RESULT: {'PASS' if not failed else 'FAIL'} " + f"({len(results) - len(failed)}/{len(results)} checks)" + + (f" failed: {', '.join(failed)}" if failed else "")) + return 0 if not failed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/services/collector/app.py b/services/collector/app.py index 8f923e3..ab14e6b 100644 --- a/services/collector/app.py +++ b/services/collector/app.py @@ -7,14 +7,14 @@ import contextlib import logging import os -from datetime import datetime, timezone +from datetime import UTC, datetime import uvicorn from fastapi import FastAPI, HTTPException from pydantic import BaseModel, ValidationError +from ulpf_core.models import RawEnvelope from collector.producer import KafkaProducer -from ulpf_core.models import RawEnvelope log = logging.getLogger("collector") @@ -28,7 +28,7 @@ def make_envelope(transport: str, source_id: str, raw: str) -> dict: return { "transport": transport, "source_id": source_id, - "received_at": datetime.now(timezone.utc).isoformat(), + "received_at": datetime.now(UTC).isoformat(), "raw": raw, } diff --git a/services/collector/tests/test_collector.py b/services/collector/tests/test_collector.py index 9148389..3895576 100644 --- a/services/collector/tests/test_collector.py +++ b/services/collector/tests/test_collector.py @@ -1,6 +1,8 @@ # services/collector/tests/test_collector.py — collector unit tests (fake producer, no Kafka). import pytest -from collector.app import build_app # FastAPI app factory; producer injected + +from collector.app import build_app # FastAPI app factory; producer injected + class FakeProducer: def __init__(self): self.produced = [] @@ -23,7 +25,7 @@ def test_http_ingest_builds_envelopes(client): assert env["received_at"] # ISO8601 present def test_http_rejects_oversized_line(client): - c, prod = client + c, _prod = client r = c.post("/v1/ingest", json={"source_id": "x", "lines": ["y" * 70000]}) assert r.status_code == 422 diff --git a/simulator/simulate.py b/simulator/simulate.py index 6bdd9e6..d39b0db 100644 --- a/simulator/simulate.py +++ b/simulator/simulate.py @@ -114,7 +114,7 @@ async def flush(self) -> int: return 0 try: return int(resp.json().get("accepted", len(lines))) - except Exception: + except Exception: # noqa: BLE001 - any 202-body failure still trusts the transport return len(lines) # 202 but unparseable body: trust the transport async def close(self) -> None: @@ -199,7 +199,7 @@ async def run(corpora: list[Corpus], *, eps: float, duration: float, cursors[i] += 1 try: sent[corpus.name] += await corpus.sender.send(line) - except Exception as exc: + except Exception as exc: # noqa: BLE001 - one failed send counts and moves on errors += 1 print(f"error: {corpus.name}: {exc}", file=sys.stderr) pass_no += 1 @@ -211,7 +211,7 @@ async def run(corpora: list[Corpus], *, eps: float, duration: float, for corpus in corpora: try: sent[corpus.name] += await corpus.sender.flush() - except Exception as exc: + except Exception as exc: # noqa: BLE001 - a failed flush counts and moves on errors += 1 print(f"error: {corpus.name} flush: {exc}", file=sys.stderr) return sent, errors From fb69da20ff463c65b0c76c3a6aabf644a82a2995 Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 17:47:48 +0530 Subject: [PATCH 15/18] fix: iterative _classify (KV recursion wedge), smoke/CI hardening per review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _classify recursed once per '=' in a token, so a hostile single token ('a=' * 1200) raised RecursionError out of shape_hash and wedged a pipeline batch — the failure class R19 fixed in _is_xml. Classification now loops partition('=') with identical semantics (KV prefixes, empty value -> W); golden + mutation-stability suites green unchanged. Also per review: smoke host deps stated as psycopg[binary] only (Kafka interaction is rpk via docker compose exec, not confluent_kafka); the MEMBERS-0 wait tracks saw_members_zero and reports timeouts truthfully; group-exists reuses group_listed(); CI pins ruff==0.16.8 and installs simulator requirements; corpora-total comment corrected to 56. Plus: compose() decodes docker output as UTF-8 explicitly — the Windows ANSI codepage crashed a subprocess reader thread on build output. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 5 ++-- README.md | 7 +++--- libs/ulpf-core/tests/test_fingerprint.py | 11 +++++++++ libs/ulpf-core/ulpf_core/fingerprint.py | 15 ++++++++---- scripts/smoke.py | 29 ++++++++++++++++-------- 5 files changed, 47 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88dcfba..e1f788e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,8 +17,9 @@ jobs: pip install -e libs/ulpf-core -e libs/ocsf-schema pip install -r services/collector/requirements.txt \ -r services/pipeline/requirements.txt \ - -r services/gateway/requirements.txt - pip install pytest ruff + -r services/gateway/requirements.txt \ + -r simulator/requirements.txt + pip install pytest ruff==0.16.8 - name: Lint run: ruff check . - name: Unit tests (libs + services + simulator) diff --git a/README.md b/README.md index 904e6f5..ba9b708 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,10 @@ idempotency (Kafka consumer-group delete + pipeline restart: raw events unchanged, raw_batches grown), and an independent recomputation of every raw_batches merkle root plus full per-partition chain linkage. -Smoke host deps (Python 3.13 tested): `confluent-kafka~=2.6`, `psycopg[binary]`. -Do NOT install the services' `confluent-kafka==2.5.0` pin host-side — no cp313 -wheels; the images themselves run it fine on Python 3.12. The smoke is +Smoke host deps (Python 3.13 tested): `psycopg[binary]` only. No confluent-kafka +on the host: Kafka interaction goes through `docker compose exec redpanda rpk`, +and the simulator runs in its own container (the services' +`confluent-kafka==2.5.0` pin is in-image only, py3.12). The smoke is delta-based: the database is never reset, so re-runs accumulate and re-verify the full accumulated chain every time. diff --git a/libs/ulpf-core/tests/test_fingerprint.py b/libs/ulpf-core/tests/test_fingerprint.py index 237e19f..fcaea3e 100644 --- a/libs/ulpf-core/tests/test_fingerprint.py +++ b/libs/ulpf-core/tests/test_fingerprint.py @@ -60,3 +60,14 @@ def test_hostile_nested_xml_line_is_stable(): line = "" * 100000 assert fingerprint_id(line).startswith("auto_") assert fingerprint_id(line) == fingerprint_id(line) + +def test_hostile_kv_token_cannot_wedge_classify(): + # Same failure class as the XML probe (R19), one layer down: _classify + # recursed once per "=" in a token, so a single token with thousands of + # key=value layers raised RecursionError out of shape_hash and wedged the + # pipeline batch. Classification must be iterative: no exception, and the + # same line always gets the same shape hash / auto_ id. + line = "a=" * 1200 # one token: no spaces or [|;,] separators + assert shape_hash(line) == shape_hash(line) + assert fingerprint_id(line) == fingerprint_id(line) + assert fingerprint_id(line).startswith("auto_") diff --git a/libs/ulpf-core/ulpf_core/fingerprint.py b/libs/ulpf-core/ulpf_core/fingerprint.py index b5d870a..6294f93 100644 --- a/libs/ulpf-core/ulpf_core/fingerprint.py +++ b/libs/ulpf-core/ulpf_core/fingerprint.py @@ -158,15 +158,20 @@ def _known_format(line: str) -> str | None: def _classify(token: str) -> str: - if "=" in token: - _, _, value = token.partition("=") + # Iterative over nested key=value layers ("a=b=c" -> "KV:KV:"): the + # recursive form drove one stack frame per "=" and a hostile single token + # with thousands of layers wedged the pipeline batch with RecursionError + # (same failure class as _is_xml's RecursionError guard). + prefixes: list[str] = [] + while "=" in token: # An empty value ("OUT=", "FLAGS=") classifies like a word value so a # field that is present-but-empty hashes the same as one with a value. - return f"KV:{_classify(value)}" + _, _, token = token.partition("=") + prefixes.append("KV:") for rx, name in _SHAPE_RULES: if rx.match(token): - return name - return "W" + return "".join(prefixes) + name + return "".join(prefixes) + "W" def shape_sequence(line: str) -> list[str]: diff --git a/scripts/smoke.py b/scripts/smoke.py index c94452c..7610126 100644 --- a/scripts/smoke.py +++ b/scripts/smoke.py @@ -7,8 +7,10 @@ restart), and independently re-verifies EVERY raw_batches merkle root plus the per-partition hash chain linkage. -Host requirements (Python 3.13 tested): confluent-kafka~=2.6, psycopg[binary]. -Everything else is stdlib (urllib for HTTP, subprocess for docker compose). +Host requirements (Python 3.13 tested): psycopg[binary] only. Everything else +is stdlib: urllib for HTTP, subprocess for docker compose; all Kafka +interaction is `docker compose exec redpanda rpk` (no confluent-kafka +host-side). Docker commands run from deploy/; the DB is NOT reset — every count assertion is delta-based (snapshot before, diff after), so re-runs accumulate. @@ -43,7 +45,7 @@ WEB = "http://localhost:3000" SIM_EPS = "25" -SIM_DURATION = "20" # ~500 events: corpora total 57 lines -> several loops +SIM_DURATION = "20" # ~500 events: corpora total 56 lines (10+16+15+15) -> several loops SEEDED_FPS = ("cef_paloalto", "cef_ciscoasa", "cef_fortigate", "syslog", "json") TRACE_SAMPLE = 20 @@ -67,9 +69,12 @@ def check(name: str, ok: bool, detail: str) -> bool: def compose(*args: str, timeout: float = 300) -> subprocess.CompletedProcess: + # Explicit UTF-8 decoding: the default on Windows is the ANSI codepage + # (cp1252 here), and docker output is UTF-8 — a stray byte crashed a + # subprocess reader thread with UnicodeDecodeError. return subprocess.run( ["docker", "compose", *args], cwd=DEPLOY, capture_output=True, - text=True, timeout=timeout, check=False, + text=True, encoding="utf-8", errors="replace", timeout=timeout, check=False, ) @@ -321,18 +326,22 @@ def assert_d_replay() -> None: f"compose stop pipeline failed: {stop.stderr.strip()[:200]}") return deadline = time.monotonic() + 60 + saw_members_zero = False + last_detail = "group describe never succeeded" while time.monotonic() < deadline: # consumer must be fully out of the group out = rpk("group", "describe", "pipeline") - if out.returncode != 0 or re.search(r"^MEMBERS\s+0\s*$", out.stdout, re.MULTILINE): + if out.returncode == 0 and re.search(r"^MEMBERS\s+0\s*$", out.stdout, re.MULTILINE): + saw_members_zero = True break + last_detail = (out.stderr.strip() or out.stdout.strip())[:120] time.sleep(POLL_INTERVAL_S) - print(" pipeline stopped; consumer has left the group") + if saw_members_zero: + print(" pipeline stopped; consumer has left the group") + else: + print(f" timeout waiting for members=0, last={last_detail!r} (continuing)") listing = rpk("group", "list") - group_present = listing.returncode == 0 and any( - line.split()[-1:] == ["pipeline"] - for line in listing.stdout.splitlines() if line.split()) - if not group_present: + if not group_listed(listing): check("D replay idempotency", False, f"kafka group 'pipeline' absent before delete (replay would be vacuous): " f"{listing.stdout.strip()!r}") From b2f6e1b1599715c5e9dc226dbe6e7139d39a674d Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 18:19:21 +0530 Subject: [PATCH 16/18] fix: seek-back failed batches (lossless), guard hostile nested JSON per final review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CRITICAL-1: consume() advances the consumer position regardless of processing success and no-arg commit() commits that position, so a failed batch (rollback, no commit) was silently covered by the next successful batch's commit — its events never redelivered. Every failure path (persist_batch, persist_normalized, the R16 catch-all) now seeks each affected partition back to the failed batch's first offset before the loop continues; fake-consumer unit tests assert the exact seeks and zero commits on both persist-failure paths and the run()-level crash. CRITICAL-2: _known_format's JSON probe caught only JSONDecodeError, but deeply-nested hostile text (17,995 chars, under the 65,536 cap) raises RecursionError out of json.loads, escaping build_rows via fingerprint_id and blowing up the batch — silent loss before C1, infinite redelivery wedge after; landed together. Regression tests at the fingerprint and build_rows layers, red first. Also: worker comments/docstrings now describe the actual seek-back mechanism; compose notes the single-replica requirement for M1 batch_seq safety (scale-out = M3 with fencing/epochs); README quickstart points at --loop sustained traffic for the dashboard. Co-Authored-By: Claude Code --- README.md | 3 + deploy/docker-compose.yml | 3 + libs/ulpf-core/tests/test_fingerprint.py | 10 +++ libs/ulpf-core/ulpf_core/fingerprint.py | 5 +- services/pipeline/tests/test_batch_builder.py | 14 +++ services/pipeline/tests/test_worker.py | 86 ++++++++++++++++--- services/pipeline/worker.py | 62 +++++++++++-- 7 files changed, 162 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index ba9b708..6109919 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,9 @@ exit is `python scripts/smoke.py` (all 5 checks PASS). cd deploy && docker compose --profile sim up -d --build python scripts/smoke.py # end-to-end acceptance incl. forced replay open http://localhost:3000 # Overview: live feed + raw traceability +# sustained live traffic for the dashboard (the --profile sim one-shot above +# exits after its single ~56-line pass): +docker compose run --rm simulator --eps 20 --duration 300 --loop ``` The smoke runs the simulator over the golden corpora, then verifies: lossless diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index c3f1372..9509a5e 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -40,6 +40,9 @@ services: environment: { KAFKA_BOOTSTRAP: redpanda:9092 } ports: ["5514:5514/udp", "5514:5514", "8080:8080"] depends_on: { topics: { condition: service_completed_successfully } } + # M1: exactly ONE pipeline replica — batch_seq = MAX+1 per partition is only + # safe with a single consumer per partition (spec §4). Scale-out needs zombie + # fencing and epoch-based batch_seq before it is correctness-safe (M3). pipeline: build: { context: .., dockerfile: services/pipeline/Dockerfile } environment: diff --git a/libs/ulpf-core/tests/test_fingerprint.py b/libs/ulpf-core/tests/test_fingerprint.py index fcaea3e..84b2d88 100644 --- a/libs/ulpf-core/tests/test_fingerprint.py +++ b/libs/ulpf-core/tests/test_fingerprint.py @@ -71,3 +71,13 @@ def test_hostile_kv_token_cannot_wedge_classify(): assert shape_hash(line) == shape_hash(line) assert fingerprint_id(line) == fingerprint_id(line) assert fingerprint_id(line).startswith("auto_") + +def test_deeply_nested_json_cannot_wedge_fingerprint(): + # Same failure class again (final-review critical): json.loads raises + # RecursionError — not JSONDecodeError — for deeply-nested hostile text, + # and the 65,536-char line cap does not bound nesting depth. This exact + # line is 17,995 chars, so it reaches fingerprint_id via UDP/HTTP ingest. + line = '{"a":' * 3000 + "1" + "}" * 3000 + fp = fingerprint_id(line) + assert re.fullmatch(r"auto_[0-9a-f]{8}", fp) + assert fp == fingerprint_id(line) diff --git a/libs/ulpf-core/ulpf_core/fingerprint.py b/libs/ulpf-core/ulpf_core/fingerprint.py index 6294f93..30b3cdb 100644 --- a/libs/ulpf-core/ulpf_core/fingerprint.py +++ b/libs/ulpf-core/ulpf_core/fingerprint.py @@ -124,7 +124,10 @@ def _known_format(line: str) -> str | None: try: json.loads(text) return "json" - except json.JSONDecodeError: + except (json.JSONDecodeError, RecursionError): + # RecursionError, like _is_xml's guard (R19): the json decoder + # raises it — not JSONDecodeError — for deeply-nested hostile + # text, and an escaped one would wedge the pipeline batch. pass # 4. Syslog Format diff --git a/services/pipeline/tests/test_batch_builder.py b/services/pipeline/tests/test_batch_builder.py index 6c4499b..2141bc6 100644 --- a/services/pipeline/tests/test_batch_builder.py +++ b/services/pipeline/tests/test_batch_builder.py @@ -112,3 +112,17 @@ def test_poison_received_at_skipped_not_fatal(): assert len(rows) == 1 assert rows[0].raw_id == raw_id_for(TOPIC, 0, 2) assert parts[0].first_offset == 2 + + +def test_hostile_nested_json_line_becomes_a_row_not_a_crash(): + # Final-review critical at the batch layer: fingerprint_id is called + # OUTSIDE build_rows' envelope guard, so a RecursionError escaping the + # JSON probe used to blow up the whole batch. The hostile line (17,995 + # chars — under the 65,536 cap, so it passes envelope validation) must + # yield an ordinary fingerprinted raw row, not an exception. + raw = '{"a":' * 3000 + "1" + "}" * 3000 + rows, parts = build_rows([FakeMsg(0, 9, envelope(raw=raw))]) + assert len(rows) == 1 + assert rows[0].fingerprint_id.startswith("auto_") + assert rows[0].raw_text == raw + assert parts[0].content_hashes == [rows[0].content_hash] diff --git a/services/pipeline/tests/test_worker.py b/services/pipeline/tests/test_worker.py index eb81ea8..e22f2fe 100644 --- a/services/pipeline/tests/test_worker.py +++ b/services/pipeline/tests/test_worker.py @@ -1,22 +1,37 @@ -# services/pipeline/tests/test_worker.py — run-loop resilience (controller R16b). +# services/pipeline/tests/test_worker.py — run-loop resilience (controller +# R16b) + failed-batch seek-back (final-review critical). # -# A batch that blows up anywhere (here: the unguarded rules SELECT surfacing a -# transient DB drop) must degrade to retry-next-batch: run() logs and keeps -# consuming — never process death, never an offset commit for the failed batch -# (§10: Postgres down -> workers stop committing offsets -> replay on recovery). -# No Kafka, no real DB: the conn is a stub whose every cursor() raises. +# A batch that blows up anywhere must degrade to redelivery of THAT batch: +# run() logs and keeps consuming — never process death, never an offset commit +# for the failed batch. Because consume() advances the consumer position +# regardless of processing success and no-arg commit() commits that position, +# a failed batch must be seeked back to its first failed offsets, or a later +# successful batch would silently cover it (lossless guarantee, spec §10). +# No Kafka, no real DB: fakes duck-type the consumer; the conn is a stub. +import json + import pytest +import pipeline.worker as worker_mod from pipeline.worker import PipelineWorker -class InterruptingConsumer: +def envelope_bytes(raw="x"): + return json.dumps({"source_id": "s", "transport": "udp", + "received_at": "2026-09-19T12:00:00+00:00", + "raw": raw}).encode() + + +class RecordingConsumer: """First consume() returns one batch; the second raises to stop the loop - (the same way main() shuts down — KeyboardInterrupt).""" + (the same way main() shuts down — KeyboardInterrupt). Records every + seek() and flags any commit() via the assertion in commit itself.""" def __init__(self, batch): self._batch = batch self.calls = 0 + self.seeks: list[dict] = [] + self.commits = 0 def subscribe(self, topic): pass @@ -27,8 +42,11 @@ def consume(self, num_messages, timeout): return self._batch raise KeyboardInterrupt + def seek(self, partition_offsets): + self.seeks.append(dict(partition_offsets)) + def commit(self): - raise AssertionError("offsets must never be committed for a failed batch") + self.commits += 1 def close(self): pass @@ -75,11 +93,55 @@ def value(self): def test_run_survives_batch_crash_and_keeps_consuming(caplog): # Undecodable value -> build_rows skips it -> persist_batch is a no-op -> # the unguarded rules SELECT (load_active_rules) hits the broken conn and - # raises out of process_batch. run() must absorb that, log it, and come - # back around for the next batch (R16b: degrade to retry, not death). + # raises out of process_batch. run() must absorb that, seek the batch's + # partitions back to their first failed offsets, and come back around for + # the next batch (R16b: degrade to redelivery, not death). msgs = [FakeMsg(0, 7, b"not-json")] - worker = PipelineWorker(InterruptingConsumer(msgs), SilentProducer(), BrokenConn()) + worker = PipelineWorker(RecordingConsumer(msgs), SilentProducer(), BrokenConn()) with caplog.at_level("ERROR", logger="pipeline.worker"), pytest.raises(KeyboardInterrupt): worker.run() # must reach consume #2, i.e. survive batch 1 assert worker.consumer.calls == 2 + assert worker.consumer.seeks == [{0: 7}] # failed batch is the next thing consumed + assert worker.consumer.commits == 0 assert "offsets NOT committed" in caplog.text + + +def test_persist_batch_failure_seeks_back_to_first_failed_offsets(monkeypatch): + # CRITICAL (lossless guarantee): on a raw-persist failure every partition + # of the failed batch must be seeked back to its FIRST offset, and no + # offset commit may happen on that path — otherwise the next successful + # batch's commit would cover the failed one and its events would be gone. + def boom(conn, rows, partitions): + raise RuntimeError("transient db drop") + + monkeypatch.setattr(worker_mod, "persist_batch", boom) + msgs = [FakeMsg(0, 7, envelope_bytes()), FakeMsg(0, 8, envelope_bytes()), + FakeMsg(2, 41, envelope_bytes())] + consumer = RecordingConsumer(msgs) + worker = PipelineWorker(consumer, SilentProducer(), BrokenConn()) + assert worker.process_batch(msgs) is False + assert consumer.seeks == [{0: 7, 2: 41}] + assert consumer.commits == 0 + + +def test_persist_normalized_failure_seeks_back_and_never_commits(monkeypatch): + # Same contract on the normalized-persist failure path: the raw rows of + # this batch may already be durable, but offsets stay uncommitted and the + # partitions seek back so the batch is redelivered (raw ON CONFLICT dedups). + monkeypatch.setattr(worker_mod, "persist_batch", lambda conn, rows, parts: None) + monkeypatch.setattr(worker_mod, "load_active_rules", lambda conn: {}) + + def boom(conn, events): + raise RuntimeError("transient db drop") + + monkeypatch.setattr(worker_mod, "persist_normalized", boom) + + class RollbackOnlyConn: + def rollback(self): + pass + + msgs = [FakeMsg(0, 7, envelope_bytes()), FakeMsg(2, 41, envelope_bytes())] + consumer = RecordingConsumer(msgs) + worker = PipelineWorker(consumer, SilentProducer(), RollbackOnlyConn()) + assert worker.process_batch(msgs) is False + assert consumer.seeks == [{0: 7, 2: 41}] diff --git a/services/pipeline/worker.py b/services/pipeline/worker.py index 5c692bc..742f07a 100644 --- a/services/pipeline/worker.py +++ b/services/pipeline/worker.py @@ -8,8 +8,11 @@ # + pipeline.dlq; parsed docs -> normalized.events (key=fingerprint_id) # 5. persist_normalized — COMMITTED # 6. only then consumer.commit() -# Crash anywhere before step 6 and the batch is redelivered; every write is -# deterministic and ON CONFLICT-guarded, so redelivery is a no-op (spec §4). +# Crash/rollback anywhere before step 6 and the worker seeks each affected +# partition back to the failed batch's first offset (consume() had already +# advanced the position; the next no-arg commit() would otherwise cover the +# failed batch and its events would be silently lost). Redelivery is a no-op +# on the DB side: every write is deterministic and ON CONFLICT-guarded (§4). # # parsed_at is the raw event's received_at (not wall clock) so a replay mints # the same event_id and collides instead of duplicating. M1 never supersedes @@ -62,6 +65,16 @@ def subscribe(self, topic: str) -> None: def consume(self, num_messages: int, timeout: float) -> list: return self._consumer.consume(num_messages=num_messages, timeout=timeout) + def seek(self, partition_offsets: dict[int, int]) -> None: + """Seek each partition back to *offset* (failed-batch redelivery). + + confluent_kafka keeps the import deferred here, like __init__.""" + import confluent_kafka + + for partition, offset in sorted(partition_offsets.items()): + self._consumer.seek( + confluent_kafka.TopicPartition(TOPIC_RAW, partition, offset)) + def commit(self) -> None: self._consumer.commit() @@ -99,6 +112,28 @@ def __init__(self, consumer, producer, conn): self.canary_failures = 0 # spec §6.4: canary failures logged + counted self._parsed_total = 0 + def _seek_back(self, msgs) -> None: + """Seek every partition in *msgs* back to its first failed-batch offset. + + consume() advances the consumer position past a batch regardless of + processing success, and no-arg commit() commits that position — so a + failed batch whose offsets were merely 'not committed' would be + silently covered by the next successful batch's commit. Seeking back + makes the failed batch the next thing consumed (at-least-once, §4). + """ + first: dict[int, int] = {} + for msg in msgs: + partition, offset = msg.partition(), msg.offset() + if partition not in first or offset < first[partition]: + first[partition] = offset + try: + self.consumer.seek(first) + except Exception: + # Never let a seek failure kill the loop; the next failed batch + # re-attempts the seek, and rebalances reset positions to the + # (clean) committed offsets anyway. + log.exception("seek-back failed; offsets NOT committed; will re-seek on next failure") + def run(self) -> None: self.consumer.subscribe(TOPIC_RAW) log.info("pipeline consuming %s as group %r", TOPIC_RAW, GROUP_ID) @@ -112,12 +147,15 @@ def run(self) -> None: data.append(msg) if data: # The WHOLE iteration is guarded (R16): a transient DB/Kafka - # drop anywhere in the batch must degrade to retry-next-batch - # (offsets stay uncommitted -> redelivery), not process death. + # drop anywhere in the batch must degrade to redelivery of + # THAT batch (seek back + offsets stay uncommitted), not + # process death and not silent position advance. try: self.process_batch(data) except Exception: - log.exception("batch failed; offsets NOT committed; batch redelivered") + self._seek_back(data) + log.exception("batch failed; seeked back to first failed offsets; " + "offsets NOT committed; batch redelivered") def process_batch(self, msgs) -> bool: """One full batch; returns True iff offsets were committed. @@ -132,7 +170,9 @@ def process_batch(self, msgs) -> bool: persist_batch(self.conn, rows, partitions) except Exception: self.conn.rollback() - log.exception("raw persist failed; offsets NOT committed; batch redelivered") + self._seek_back(msgs) # redeliver THIS batch, not just hope for it + log.exception("raw persist failed; seeked back to first failed offsets; " + "offsets NOT committed; batch redelivered") return False rules = load_active_rules(self.conn) # refreshed every batch @@ -203,10 +243,16 @@ def process_batch(self, msgs) -> bool: persist_normalized(self.conn, events) except Exception: self.conn.rollback() - log.exception("normalized persist failed; offsets NOT committed; batch redelivered") + self._seek_back(msgs) # redeliver THIS batch (raw rows ON CONFLICT dedup) + log.exception("normalized persist failed; seeked back to first failed offsets; " + "offsets NOT committed; batch redelivered") return False - self.consumer.commit() # both DB txns durable -> safe to advance offsets + # Both DB txns durable -> safe to advance. commit() (no args) commits + # the consumer POSITION, which already covers every consumed batch — + # any batch that failed earlier must have seeked back above, or this + # commit would silently cover it. + self.consumer.commit() log.info("batch: %d consumed, %d parsed, %d unparsed, %d parse_error, " "%d skipped-envelope, canary_failures=%d", len(msgs), parsed, unparsed, parse_errors, len(msgs) - len(rows), From 2e189ed98c389f5a6b19d11cc36c9f7366d98e7c Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 19:17:14 +0530 Subject: [PATCH 17/18] fix: make shebang scripts executable (EXE001 on linux CI) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ruff 0.16.8 EXE001 fired on the linux runner for bench/fingerprint_fragmentation.py, deploy/migrate.py, scripts/smoke.py — each carries a deliberate #!/usr/bin/env python3 but was committed mode 644 (staged from Windows, where the exec bit is invisible to every local check). Set the executable bit via git; file content unchanged. Co-Authored-By: Claude Code --- bench/fingerprint_fragmentation.py | 0 deploy/migrate.py | 0 scripts/smoke.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bench/fingerprint_fragmentation.py mode change 100644 => 100755 deploy/migrate.py mode change 100644 => 100755 scripts/smoke.py diff --git a/bench/fingerprint_fragmentation.py b/bench/fingerprint_fragmentation.py old mode 100644 new mode 100755 diff --git a/deploy/migrate.py b/deploy/migrate.py old mode 100644 new mode 100755 diff --git a/scripts/smoke.py b/scripts/smoke.py old mode 100644 new mode 100755 From a5ff4484f5fef8ec085cbb11598ae53152397f70 Mon Sep 17 00:00:00 2001 From: Karthik Sethu Raman Date: Sat, 19 Sep 2026 19:23:06 +0530 Subject: [PATCH 18/18] test: deepen nested-JSON regression literal to trigger on py3.12 and py3.13 CI runs CPython 3.12, whose C json scanner parses depth-3000 nesting without error, so the Final-C2 regression tests (written against 3.13, which raises there) passed vacuously on the runner. Depth 10,000 raises RecursionError on both generations, and the 60,001-char line stays under the 65,536-char envelope cap, so it still exercises the ingest path. Co-Authored-By: Claude Code --- libs/ulpf-core/tests/test_fingerprint.py | 10 +++++++--- services/pipeline/tests/test_batch_builder.py | 7 ++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/libs/ulpf-core/tests/test_fingerprint.py b/libs/ulpf-core/tests/test_fingerprint.py index 84b2d88..12b5434 100644 --- a/libs/ulpf-core/tests/test_fingerprint.py +++ b/libs/ulpf-core/tests/test_fingerprint.py @@ -75,9 +75,13 @@ def test_hostile_kv_token_cannot_wedge_classify(): def test_deeply_nested_json_cannot_wedge_fingerprint(): # Same failure class again (final-review critical): json.loads raises # RecursionError — not JSONDecodeError — for deeply-nested hostile text, - # and the 65,536-char line cap does not bound nesting depth. This exact - # line is 17,995 chars, so it reaches fingerprint_id via UDP/HTTP ingest. - line = '{"a":' * 3000 + "1" + "}" * 3000 + # and the 65,536-char line cap does not bound nesting depth. The depth + # must trip the recursion limit on BOTH interpreter generations: CPython + # 3.12's C json scanner parses depth-3000 without error (CI runs 3.12), + # while 3.13 already raises there. Depth 10,000 raises on both and the + # line is 60,001 chars — under the cap, so it reaches fingerprint_id + # via UDP/HTTP ingest. + line = '{"a":' * 10000 + "1" + "}" * 10000 fp = fingerprint_id(line) assert re.fullmatch(r"auto_[0-9a-f]{8}", fp) assert fp == fingerprint_id(line) diff --git a/services/pipeline/tests/test_batch_builder.py b/services/pipeline/tests/test_batch_builder.py index 2141bc6..1ff177c 100644 --- a/services/pipeline/tests/test_batch_builder.py +++ b/services/pipeline/tests/test_batch_builder.py @@ -117,10 +117,11 @@ def test_poison_received_at_skipped_not_fatal(): def test_hostile_nested_json_line_becomes_a_row_not_a_crash(): # Final-review critical at the batch layer: fingerprint_id is called # OUTSIDE build_rows' envelope guard, so a RecursionError escaping the - # JSON probe used to blow up the whole batch. The hostile line (17,995 - # chars — under the 65,536 cap, so it passes envelope validation) must + # JSON probe used to blow up the whole batch. Depth 10,000 (60,001 chars + # — under the 65,536 cap, so it passes envelope validation, and deep + # enough to raise RecursionError on CPython 3.12 and 3.13 alike) must # yield an ordinary fingerprinted raw row, not an exception. - raw = '{"a":' * 3000 + "1" + "}" * 3000 + raw = '{"a":' * 10000 + "1" + "}" * 10000 rows, parts = build_rows([FakeMsg(0, 9, envelope(raw=raw))]) assert len(rows) == 1 assert rows[0].fingerprint_id.startswith("auto_")