diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d89723..fdf5495 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,29 @@ ### Added +- **Sources are flows.** A source's editor is a workspace: the step list on the left, the source's + steps placed and connected by hand, each clicked open for its settings and samples. After the + input (transport and codec) come Reject, Map, Filter, Entity links, Affiliation, Tracker and + Throttle steps, any number of each, branching to the source's one **Output** (system tracks, to + correlation) and any **NATS topics** (the mapped data, as JSON, without correlation). The spec + keeps the flow under `pipeline` (`nodes`, `edges`); a spec in the older fixed form still reads, + as the same stages in a line. Add source asks for a name, id and transport and opens a draft. - **Draft sources.** A source that is not complete yet saves as a draft, which cannot be enabled, - instead of being refused. Its row carries `draft` and `problems`, each with the pipeline stage it - belongs to. An enabled source is still saved only complete, and enabling re-checks it. + instead of being refused. Its row carries `draft` and `problems`, each on the step it belongs to. An enabled source is still saved only complete, and enabling re-checks it. - **Every problem with a source at once.** A refused save or enable is a 422 that lists all of the - source's `problems` (`{kind, stage, field?, message}`), not just the first, and an empty + source's `problems` (`{kind, node, field?, message}`), not just the first, and an empty transport setting (host, port, URL, bind address, topic, method, path) is now reported by name. The SQLite schema moves to version 20. +### Changed + +- **Raw output is a NATS topic step.** Its consent and the `/sources/{id}/raw-output` endpoints are + gone. A source saved with a raw-output subject gets a NATS topic step beside its Output, once, on + start (recorded as a revision by `op:migration`); a configuration file's raw output is read the + same way. A NATS topic's subject must lie outside the system tracks' subjects. +- **The add-source wizard, the source drawer and the pipeline designer** are replaced by the + source editor. + ## 1.0.0, 2026-10-06 The first open-source release, under the MIT License. 1.0 carries everything up to 0.4.7, plus the diff --git a/crates/ot-server/src/api.rs b/crates/ot-server/src/api.rs index 49c4642..6f17998 100644 --- a/crates/ot-server/src/api.rs +++ b/crates/ot-server/src/api.rs @@ -10,10 +10,11 @@ use axum::extract::{Path, Query, State}; use axum::http::{HeaderMap, StatusCode}; use axum::routing::{get, post, put}; use axum::{Extension, Json, Router}; +use ot_source::flow::{INPUT, Step}; use ot_source::frame::Frame; use ot_source::pipeline::Pipeline; use ot_source::schema::{ExtensionField, ExtensionSchema}; -use ot_source::source::{Problem, ProblemKind, SourceSpec, Stage}; +use ot_source::source::{Problem, ProblemKind, SourceSpec}; use ot_source::trace::Trace; use ot_store::{Entity, SourceRow, SourceWrite}; use serde::Deserialize; @@ -33,10 +34,6 @@ pub fn routes() -> Router { ) .route("/sources/{id}/enable", post(enable_source)) .route("/sources/{id}/disable", post(disable_source)) - .route( - "/sources/{id}/raw-output", - put(set_raw_output).delete(clear_raw_output), - ) .route("/sources/{id}/revisions", get(source_revisions)) .route("/sources/{id}/metrics", get(source_metrics)) .route("/registry", get(registry_resolve)) @@ -77,6 +74,7 @@ pub(crate) fn actor(headers: &HeaderMap) -> String { /// filled in), the published extension schema its mapping targets (if it /// is published) and everything that stops it running. A spec that does /// not parse, or is not allowed at all, is refused here. +#[derive(Clone)] struct Checked { spec: SourceSpec, normalised: Value, @@ -100,12 +98,58 @@ impl Checked { } } - /// Refuse it unless it is complete (it may run). - fn complete(self) -> Result<(SourceSpec, Value, ExtensionSchema), ApiError> { - match self.schema { - Some(schema) if self.problems.is_empty() => Ok((self.spec, self.normalised, schema)), - _ => Err(ApiError::source_problems(&self.problems)), - } + /// Its pipeline, to dry-run over samples: a draft runs too, as far as + /// its shape, maps, trackers and codec allow (its transport is not used, + /// and without a published schema its extension fields are not checked). + fn dry_run(&self) -> Result { + self.clone().savable()?; + let pipeline = + Pipeline::new(self.spec.id.clone(), self.spec.pipeline.clone()).map_err(|e| { + use ot_source::pipeline::PipelineError as E; + let flow = &self.spec.pipeline; + let blocks = |p: &&Problem| { + p.kind == ProblemKind::Invalid + || (p.node.as_deref() == Some(INPUT) && p.field.as_deref() == Some("codec")) + || p.node + .as_deref() + .and_then(|n| flow.node(n)) + .is_some_and(|n| matches!(n.step, Step::Map(_) | Step::Tracker(_))) + }; + let mut blocking: Vec = + self.problems.iter().filter(blocks).cloned().collect(); + // What the checks did not see (a tracker plugin not loaded, say), on its step. + if blocking.is_empty() { + let first = |want: fn(&Step) -> bool| { + flow.nodes + .iter() + .find(|n| want(&n.step)) + .map(|n| n.id.clone()) + }; + blocking.push(Problem { + kind: ProblemKind::Incomplete, + node: match e { + E::Mapping(_) => first(|s| matches!(s, Step::Map(_))), + E::Tracker(_) => first(|s| matches!(s, Step::Tracker(_))), + E::Codec(_) => Some(INPUT.to_owned()), + E::Flow(_) => None, + }, + field: None, + message: e.to_string(), + }); + } + // Then the rest, so every step shows what it needs. + let rest = self + .problems + .iter() + .filter(|p| !blocking.contains(p)) + .cloned(); + let all: Vec = blocking.iter().cloned().chain(rest).collect(); + ApiError::source_problems(&all) + })?; + Ok(match &self.schema { + Some(schema) => pipeline.with_schema(schema.clone()), + None => pipeline, + }) } } @@ -113,7 +157,7 @@ impl Checked { fn invalid(message: String) -> ApiError { ApiError::source_problems(&[Problem { kind: ProblemKind::Invalid, - stage: None, + node: None, field: None, message, }]) @@ -125,13 +169,27 @@ async fn check_spec(s: &AppState, body: Value) -> Result { crate::probe::file_allowed(&s.common, &spec.transport).map_err(|e| { ApiError::source_problems(&[Problem { kind: ProblemKind::Invalid, - stage: Some(Stage::Source), + node: Some(INPUT.to_owned()), field: Some("path".into()), message: e, }]) })?; - let schema = published_schema(s, spec.pipeline.mapping.schema_version).await?; - let problems = spec.problems(schema.as_ref()); + let schema = published_schema(s, spec.pipeline.schema_version()).await?; + let mut problems = spec.problems(schema.as_ref()); + // A NATS topic publishes outside the system tracks' subjects. + for n in &spec.pipeline.nodes { + if let Step::Nats(t) = &n.step + && ot_source::source::check_subject(&t.subject).is_ok() + && let Err(e) = check_raw_subject(&t.subject, &s.common.nats.tracks_subject) + { + problems.push(Problem { + kind: ProblemKind::Incomplete, + node: Some(n.id.clone()), + field: Some("subject".into()), + message: e, + }); + } + } let normalised = serde_json::to_value(&spec).map_err(|e| ApiError::internal(e.to_string()))?; Ok(Checked { spec, @@ -353,16 +411,6 @@ async fn disable_source( toggle(s, id, false, None, headers).await } -#[derive(Deserialize)] -struct RawOutput { - /// NATS subject for this source's raw tracks; `opentrack.raw.` - /// when omitted. - #[serde(default)] - subject: Option, - /// Must be true: the admin confirms this source's raw data is published. - consent: bool, -} - /// A concrete (wildcard-free) NATS subject outside the system tracks prefix. pub(crate) fn check_raw_subject(subject: &str, tracks_prefix: &str) -> Result<(), String> { let valid = !subject.is_empty() @@ -376,47 +424,12 @@ pub(crate) fn check_raw_subject(subject: &str, tracks_prefix: &str) -> Result<() } if subject == tracks_prefix || subject.starts_with(&format!("{tracks_prefix}.")) { return Err(format!( - "raw output needs its own subject, outside the system tracks subjects ({tracks_prefix}.>)" + "a NATS topic needs its own subject, outside the system tracks subjects ({tracks_prefix}.>)" )); } Ok(()) } -async fn set_raw_output( - State(s): State, - Path(id): Path, - headers: HeaderMap, - Json(body): Json, -) -> Result, ApiError> { - if !body.consent { - return Err(ApiError::unprocessable( - "raw output publishes this source's tracks as received; set consent to true to confirm", - )); - } - let c = match body.subject.as_deref().map(str::trim) { - Some(sub) if !sub.is_empty() => sub.to_owned(), - _ => format!("opentrack.raw.{id}"), - }; - check_raw_subject(&c, &s.common.nats.tracks_subject).map_err(ApiError::unprocessable)?; - let actor = actor(&headers); - let row = s - .with_db(move |db| db.set_raw_output(&id, Some(&c), &actor)) - .await?; - Ok(Json(with_status(&s, row).await)) -} - -async fn clear_raw_output( - State(s): State, - Path(id): Path, - headers: HeaderMap, -) -> Result, ApiError> { - let actor = actor(&headers); - let row = s - .with_db(move |db| db.set_raw_output(&id, None, &actor)) - .await?; - Ok(Json(with_status(&s, row).await)) -} - async fn source_revisions( State(s): State, u: Option>, @@ -473,13 +486,22 @@ struct ValidateBody { trace: Option, } -/// Validate a source spec and optionally dry-run sample frames through its -/// pipeline, using the live registry. Nothing is saved or published. +/// Check a source spec and optionally dry-run sample frames through its +/// pipeline, using the live registry. Nothing is saved or published. The +/// answer lists the spec's `problems` (`valid` when there are none); a draft +/// is dry-run as far as it can be. async fn validate_source( State(s): State, Json(body): Json, ) -> Result, ApiError> { - let (spec, normalised, schema) = check_spec(&s, body.spec).await?.complete()?; + let checked = check_spec(&s, body.spec).await?; + let mut pipeline = checked.dry_run()?; + let Checked { + normalised, + problems, + .. + } = checked; + let valid = problems.is_empty(); let mut frames: Vec = body .samples .iter() @@ -498,7 +520,9 @@ async fn validate_source( })); } if frames.is_empty() { - return Ok(Json(json!({ "valid": true, "spec": normalised }))); + return Ok(Json( + json!({ "valid": valid, "problems": problems, "spec": normalised }), + )); } let rows = s.with_db(|db| db.registry_rows()).await?; let registry: BTreeMap<(String, String), ot_source::registry::RegistryEntry> = rows @@ -515,9 +539,6 @@ async fn validate_source( ) }) .collect(); - let mut pipeline = Pipeline::new(spec.id.clone(), spec.pipeline.clone()) - .map_err(|e| ApiError::unprocessable(e.to_string()))? - .with_schema(schema); let mut trace = body.trace.filter(|n| *n > 0).map(Trace::new); let mut observations = Vec::new(); let mut errors = Vec::new(); @@ -536,13 +557,14 @@ async fn validate_source( .last() .map_or_else(chrono::Utc::now, |f| f.received_at); let flushed = match trace.as_mut() { - Some(t) => pipeline.flush_traced(last, true, t), - None => pipeline.flush(last, true), + Some(t) => pipeline.flush_traced(last, true, ®istry, t), + None => pipeline.flush(last, true, ®istry), }; observations.extend(flushed.observations); let counts: BTreeMap = pipeline.take_counts().pairs().into_iter().collect(); let mut body = json!({ - "valid": true, + "valid": valid, + "problems": problems, "counts": counts, "errors": errors, "observations": observations, @@ -999,8 +1021,8 @@ mod tests { assert_eq!(st, StatusCode::CREATED, "{body}"); let problem = &body["problems"][0]; assert_eq!( - (&problem["stage"], &problem["field"]), - (&json!("source"), &json!("unauthenticated")), + (&problem["node"], &problem["field"]), + (&json!("input"), &json!("unauthenticated")), "{body}" ); assert!( @@ -1061,7 +1083,7 @@ mod tests { assert_eq!(body["transport"], "http_poll"); assert_eq!(body["draft"], true, "{body}"); assert_eq!(body["problems"][0]["kind"], "incomplete", "{body}"); - assert_eq!(body["problems"][0]["stage"], "map", "{body}"); + assert_eq!(body["problems"][0]["node"], "map", "{body}"); assert!( body["problems"][0]["message"] .as_str() @@ -1075,7 +1097,7 @@ mod tests { body["error"].as_str().unwrap().contains("schema version 2"), "{body}" ); - assert_eq!(body["problems"][0]["stage"], "map", "{body}"); + assert_eq!(body["problems"][0]["node"], "map", "{body}"); // Once the schema is published, it may run as it is. publish_example_schema(&app).await; @@ -1120,7 +1142,7 @@ mod tests { .as_array() .unwrap() .iter() - .inspect(|p| assert_eq!(p["stage"], "map", "{body}")) + .inspect(|p| assert_eq!(p["node"], "map", "{body}")) .map(|p| p["message"].as_str().unwrap()) .collect(); assert!( @@ -1150,44 +1172,28 @@ mod tests { assert_eq!(body["sources"][0]["name"], "adsb.lol (edited)"); assert_eq!(body["sources"][0]["draft"], false); - // Raw output needs explicit consent and a subject outside `tracks.>`. - for bad in [ - json!({"consent": false}), - json!({"subject": "tracks.raw", "consent": true}), - json!({"subject": "tracks", "consent": true}), - json!({"subject": "raw.*", "consent": true}), - json!({"subject": "raw..adsb", "consent": true}), - ] { - let (st, _) = call( - &app, - "PUT", - "/api/v1/sources/adsb-lol/raw-output", - Some(bad.clone()), - ) - .await; - assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "{bad}"); - } - let (st, body) = call( - &app, - "PUT", - "/api/v1/sources/adsb-lol/raw-output", - Some(json!({"consent": true})), - ) - .await; - assert_eq!( - (st, body["raw_subject"].as_str()), - (StatusCode::OK, Some("opentrack.raw.adsb-lol")) - ); - let (st, body) = call( - &app, - "PUT", - "/api/v1/sources/adsb-lol/raw-output", - Some(json!({"subject": "raw.adsb", "consent": true})), - ) - .await; + // A NATS topic publishes outside the system tracks' subjects. + let mut topic = spec.clone(); + topic["pipeline"] = json!({ + "codec": spec["pipeline"]["codec"], + "nodes": [ + {"id": "map", "type": "map", "config": {"mapping": spec["pipeline"]["mapping"]}}, + {"id": "out", "type": "output"}, + {"id": "raw", "type": "nats", "config": {"subject": "tracks.raw"}} + ], + "edges": [{"source": "input", "target": "map"}, {"source": "map", "target": "out"}, + {"source": "map", "target": "raw"}] + }); + topic["name"] = json!("adsb.lol (edited)"); + let (st, body) = call(&app, "PUT", "/api/v1/sources/adsb-lol", Some(topic.clone())).await; + assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "enabled: {body}"); + assert_eq!(body["problems"][0]["node"], "raw", "{body}"); + topic["pipeline"]["nodes"][2]["config"]["subject"] = json!("raw.adsb"); + let (st, body) = call(&app, "PUT", "/api/v1/sources/adsb-lol", Some(topic)).await; + assert_eq!(st, StatusCode::OK, "{body}"); assert_eq!( - (st, body["raw_subject"].as_str()), - (StatusCode::OK, Some("raw.adsb")) + body["spec"]["pipeline"]["nodes"][3]["config"]["subject"], "raw.adsb", + "{body}" ); // Status reports NATS as down without blocking the API: 503, with @@ -1308,6 +1314,42 @@ mod tests { assert_eq!(o["ext"]["registry"]["scheme"], "elnot"); assert_eq!(o["ext"]["registry"]["grade"], "name"); assert_eq!(o["classification"]["cot_type"], "a-f-S-C-A"); + assert_eq!(body["valid"], true, "{body}"); + + // A draft runs as far as it can: its transport is not used. + let mut draft = spec.clone(); + draft["transport"] = json!({"type": "udp", "bind": ""}); + let (st, body) = call( + &app, + "POST", + "/api/v1/sources/validate", + Some(json!({"spec": draft, "samples": [frame]})), + ) + .await; + assert_eq!(st, StatusCode::OK, "{body}"); + assert_eq!(body["valid"], false, "{body}"); + assert_eq!(body["problems"][0]["field"], "bind", "{body}"); + assert_eq!(body["observations"].as_array().unwrap().len(), 1, "{body}"); + // One whose mapping cannot run says why, on its stage. + draft["pipeline"]["mapping"]["rules"] = json!([]); + let (st, body) = call( + &app, + "POST", + "/api/v1/sources/validate", + Some(json!({"spec": draft, "samples": [frame]})), + ) + .await; + assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["problems"][0]["node"], "map", "{body}"); + // The others follow, so each step shows what it needs. + assert!( + body["problems"] + .as_array() + .unwrap() + .iter() + .any(|p| p["field"] == "bind"), + "{body}" + ); redis.purge_namespace().await.unwrap(); } @@ -1374,13 +1416,17 @@ mod tests { .iter() .map(|s| s["id"].as_str().unwrap()) .collect(); - assert_eq!(ids, ["decode", "map", "registry", "filter", "publish"]); - let publish = &t[0]["stages"][4]["items"][0]; + assert_eq!( + ids, + ["input", "map", "registry", "filter", "output", "publish"] + ); + let publish = &t[0]["stages"][5]["items"][0]; assert_eq!(publish["name"], "ALPHA", "the published message"); assert_eq!(publish["op"], "upsert"); assert_eq!(publish["track_id"], "assigned by correlation"); assert_eq!(t[1]["stages"][3]["dropped"], json!(["drop_if matched"])); assert_eq!(t[1]["stages"][4]["items"], json!([])); + assert_eq!(t[1]["stages"][5]["items"], json!([])); assert_eq!( frames[2], json!({"format": "text", "bytes": 8, "content": "not json"}) diff --git a/crates/ot-server/src/config.rs b/crates/ot-server/src/config.rs index df46ae6..a557d46 100644 --- a/crates/ot-server/src/config.rs +++ b/crates/ot-server/src/config.rs @@ -219,7 +219,14 @@ impl Common { if let Some(dir) = self.sqlite.parent().filter(|d| !d.as_os_str().is_empty()) { std::fs::create_dir_all(dir)?; } - Ok(ot_store::Db::open(&self.sqlite)?) + let mut db = ot_store::Db::open(&self.sqlite)?; + // Once a process: sources from before flows move their raw output + // to a NATS topic step. + static MIGRATED: std::sync::Once = std::sync::Once::new(); + let mut result = Ok(()); + MIGRATED.call_once(|| result = crate::sources::migrate_raw_outputs(&mut db)); + result?; + Ok(db) } /// Keep one connection open for every later `open_db` of this (and diff --git a/crates/ot-server/src/config_backup.rs b/crates/ot-server/src/config_backup.rs index 0d7efd0..e3ebbff 100644 --- a/crates/ot-server/src/config_backup.rs +++ b/crates/ot-server/src/config_backup.rs @@ -281,7 +281,7 @@ fn check(common: &Common, f: &mut ConfigFile) -> Vec { for s in &mut f.sources { let at = format!("source {}", s.id); p.extend(unique(&mut seen, "source", s.id.clone())); - let spec: SourceSpec = match serde_json::from_value(s.spec.clone()) { + let mut spec: SourceSpec = match serde_json::from_value(s.spec.clone()) { Ok(spec) => spec, Err(e) => { p.push(format!("{at}: invalid source spec: {e}")); @@ -296,8 +296,21 @@ fn check(common: &Common, f: &mut ConfigFile) -> Vec { p.push(format!("{at}: {e}")); continue; } + // A raw output (from before flows) comes back as a NATS topic step. + if let Some(sub) = s.raw_subject.take() { + if let Err(e) = crate::api::check_raw_subject(&sub, &common.nats.tracks_subject) { + p.push(format!("{at}: raw output: {e}")); + continue; + } + if spec.pipeline.add_output_topic(&sub).is_none() { + p.push(format!( + "{at}: raw output: the source has no Output to publish beside" + )); + continue; + } + } // A draft comes back a draft; an enabled source must be complete. - let schema = schemas.get(&spec.pipeline.mapping.schema_version); + let schema = schemas.get(&spec.pipeline.schema_version()); let problems = spec.problems(schema); let refused: Vec<_> = problems .iter() @@ -311,11 +324,6 @@ fn check(common: &Common, f: &mut ConfigFile) -> Vec { Ok(v) => s.problems = v, Err(e) => p.push(format!("{at}: {e}")), } - if let Some(sub) = &s.raw_subject - && let Err(e) = crate::api::check_raw_subject(sub, &common.nats.tracks_subject) - { - p.push(format!("{at}: raw output: {e}")); - } s.name = spec.name.clone(); s.transport = spec.transport.kind().into(); s.codec = spec.pipeline.codec.name().into(); diff --git a/crates/ot-server/src/engine.rs b/crates/ot-server/src/engine.rs index 3f8fd4b..534fb3d 100644 --- a/crates/ot-server/src/engine.rs +++ b/crates/ot-server/src/engine.rs @@ -768,19 +768,20 @@ impl Engine { .filter(|s| s.enabled) .map(|s| { // A tracker stage turns detections into tracks before they get here. - let detections = s.spec.get("reports").and_then(|r| r.as_str()) - == Some("detections") - && s.spec - .pointer("/pipeline/tracker") - .is_none_or(Value::is_null); let spec = serde_json::from_value::(s.spec.clone()) .ok(); + // Plots reach correlation untracked: no Tracker before the Output. + let detections = s.spec.get("reports").and_then(|r| r.as_str()) + == Some("detections") + && !spec + .as_ref() + .is_some_and(|spec| spec.pipeline.output_tracked()); let alone = spec.as_ref().is_none_or(|spec| spec.publishes_alone()); let confirm = spec.as_ref().and_then(|spec| spec.confirms_after()); let motion = spec.as_ref().and_then(|spec| spec.emitter_motion); let stage = spec - .and_then(|spec| spec.pipeline.registry) + .map(|spec| spec.pipeline.output_registry()) .unwrap_or_default(); (s.id, s.priority, detections, alone, confirm, stage, motion) }) diff --git a/crates/ot-server/src/engine/bench.rs b/crates/ot-server/src/engine/bench.rs index 636e76c..c54711b 100644 --- a/crates/ot-server/src/engine/bench.rs +++ b/crates/ot-server/src/engine/bench.rs @@ -325,7 +325,10 @@ pub async fn run(common: &Common, opts: BenchOptions) -> anyhow::Result<()> { "observations": observations, "sources": specs.iter().map(|(s, _)| json!({ "id": s.id, "reports": s.reports, - "tracker": s.pipeline.tracker.as_ref().map(|t| serde_json::to_value(t).unwrap_or(Value::Null)), + "tracker": s.pipeline.nodes.iter().find_map(|n| match &n.step { + ot_source::flow::Step::Tracker(t) => Some(serde_json::to_value(t).unwrap_or(Value::Null)), + _ => None, + }), })).collect::>(), "correlation": correlation_used, "plugins": plugins, @@ -412,7 +415,7 @@ async fn replay( } let end = now > last; for p in pipelines.values_mut() { - let out = p.flush(now, end); + let out = p.flush(now, end, ®istry); if let Some(e) = &out.last_error { let entry = errors.entry(specs_id(p)).or_insert((0, String::new())); entry.0 += 1; diff --git a/crates/ot-server/src/sources.rs b/crates/ot-server/src/sources.rs index 6f20055..15edd87 100644 --- a/crates/ot-server/src/sources.rs +++ b/crates/ot-server/src/sources.rs @@ -5,8 +5,8 @@ //! backoff) feeding its pipeline, which appends observations to the source's //! Redis stream. The worker also persists the static-join cache, flushes //! per-source metrics, and publishes a live status the API reads. A source -//! with a raw-output subject (set with recorded consent) also publishes each -//! of its observations, before correlation, as JSON on that NATS subject. +//! with NATS topic steps also publishes what reaches each, as JSON on its +//! subject, without correlation. //! Entity fields a feed updates (a pipeline's track → entity links) are //! written to the registry every few seconds. @@ -109,7 +109,7 @@ pub async fn run( shutdown: impl std::future::Future, ) -> anyhow::Result<()> { let redis = common.open_redis().await?; - // Only raw output uses NATS here; the client connects in the background. + // Only NATS topic steps use NATS here; the client connects in the background. let nats = common.connect_nats().await?.client().clone(); let registry = Arc::new(Registry::default()); let mut running: BTreeMap = BTreeMap::new(); @@ -306,15 +306,12 @@ fn reconcile( ); continue; } - let Some(schema) = schemas.get(&spec.pipeline.mapping.schema_version).cloned() else { - tracing::error!(source = %id, version = spec.pipeline.mapping.schema_version, + let Some(schema) = schemas.get(&spec.pipeline.schema_version()).cloned() else { + tracing::error!(source = %id, version = spec.pipeline.schema_version(), "mapping targets an unpublished schema version; not started"); continue; }; - let raw = row.raw_subject.clone().map(|subject| RawOutput { - client: nats.clone(), - subject, - }); + let raw = Some(nats.clone()); let handle = tokio::spawn(run_source( spec, schema, @@ -336,20 +333,13 @@ fn reconcile( refused } -/// Where a source's raw output goes. -#[derive(Clone)] -struct RawOutput { - client: ot_nats::async_nats::Client, - subject: String, -} - /// Run one source forever: a supervised transport and its pipeline. async fn run_source( spec: SourceSpec, schema: ExtensionSchema, revision: i64, redis: RedisStore, - raw: Option, + raw: Option, registry: Arc, ) { let id = spec.id.clone(); @@ -431,10 +421,10 @@ async fn run_source( supervisor.abort(); } -/// Where a pipeline writes: its Redis stream, and its raw feed if it has one. +/// Where a pipeline writes: its Redis stream, and NATS for its topic steps. struct Outputs<'a> { redis: &'a RedisStore, - raw: Option<&'a RawOutput>, + raw: Option<&'a ot_nats::async_nats::Client>, } async fn pipeline_loop( @@ -451,7 +441,7 @@ async fn pipeline_loop( let mut pipeline = Pipeline::new(id, spec.pipeline.clone()) .context("invalid pipeline")? .with_schema(schema.clone()); - let static_ttl = Duration::from_secs(spec.pipeline.static_join.ttl_secs); + let static_ttl = pipeline.static_ttl(); // Warm the static cache so identity survives a restart. let mut loaded = 0; @@ -466,7 +456,7 @@ async fn pipeline_loop( let mut flush = tokio::time::interval(FLUSH_EVERY); let mut last_error: Option = None; let mut totals: HashMap = HashMap::new(); - let (mut raw_published, mut raw_errors) = (0u64, 0u64); + let mut publish_errors = 0u64; // Scans a tracker holds for late plots are run when their hold passes. let mut hold = tokio::time::interval(TRACKER_HOLD_POLL); loop { @@ -476,13 +466,11 @@ async fn pipeline_loop( tracing::info_span!("source.process", source = %id) .in_scope(|| pipeline.process(&frame, registry.as_ref())) } - _ = hold.tick(), if spec.pipeline.tracker.is_some() => pipeline.flush(Utc::now(), false), + _ = hold.tick(), if pipeline.has_tracker() => pipeline.flush(Utc::now(), false, registry.as_ref()), _ = flush.tick() => { let mut counts = pipeline.take_counts().pairs(); - for (k, n) in [("raw_published", &mut raw_published), ("raw_error", &mut raw_errors)] { - if *n > 0 { - counts.push((k.to_owned(), std::mem::take(n))); - } + if publish_errors > 0 { + counts.push(("publish_error".to_owned(), std::mem::take(&mut publish_errors))); } redis.incr_metrics(id, &counts).await?; for (k, n) in &counts { @@ -510,6 +498,12 @@ async fn pipeline_loop( for o in &mut out.observations { o.security = Some(label.clone()); } + let label = serde_json::to_value(label)?; + for (_, v) in &mut out.published { + if let Some(o) = v.as_object_mut() { + o.insert("security".into(), label.clone()); + } + } } redis .append_observations(id, &out.observations) @@ -519,15 +513,12 @@ async fn pipeline_loop( observations = out.observations.len() )) .await?; - if let Some(raw) = raw { - for obs in &out.observations { - let body = serde_json::to_vec(obs)?; - match raw.client.publish(raw.subject.clone(), body.into()).await { - Ok(()) => raw_published += 1, - Err(e) => { - raw_errors += 1; - last_error = Some(format!("raw output: {e}")); - } + if let Some(nats) = raw { + for (subject, v) in &out.published { + let body = serde_json::to_vec(v)?; + if let Err(e) = nats.publish(subject.clone(), body.into()).await { + publish_errors += 1; + last_error = Some(format!("NATS topic {subject}: {e}")); } } } @@ -559,10 +550,90 @@ fn rand_fraction() -> f64 { (h.finish() >> 11) as f64 / (1u64 << 53) as f64 } +/// Sources saved with a raw-output subject (before flows) publish it from a +/// NATS topic step instead: added once, beside the Output, and the subject +/// cleared. Each change is a revision, recorded as the migration's. +pub fn migrate_raw_outputs(db: &mut ot_store::Db) -> anyhow::Result<()> { + for row in db.list_sources()? { + let Some(subject) = row.raw_subject.clone() else { + continue; + }; + let Ok(mut spec) = serde_json::from_value::(row.spec.clone()) else { + tracing::warn!(source = %row.id, "raw output not moved to a NATS topic step: its spec does not read"); + continue; + }; + if spec.pipeline.add_output_topic(&subject).is_none() { + tracing::warn!(source = %row.id, "raw output not moved to a NATS topic step: it has no Output"); + continue; + } + let normalised = serde_json::to_value(&spec)?; + db.put_source( + &ot_store::SourceWrite { + id: &row.id, + name: &row.name, + transport: &row.transport, + codec: &row.codec, + priority: row.priority, + spec: &normalised, + problems: &row.problems, + }, + "op:migration", + )?; + db.set_raw_output(&row.id, None, "op:migration")?; + tracing::info!(source = %row.id, %subject, "raw output moved to a NATS topic step"); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn a_raw_output_moves_to_a_nats_topic_step_once() { + let mut db = ot_store::Db::open_in_memory().unwrap(); + let spec: serde_json::Value = + serde_json::from_str(include_str!("../../../docs/examples/gps-udp.json")).unwrap(); + let none = json!([]); + let write = ot_store::SourceWrite { + id: spec["id"].as_str().unwrap(), + name: "GPS", + transport: "udp", + codec: "json", + priority: 100, + spec: &spec, + problems: &none, + }; + db.put_source(&write, "op:test").unwrap(); + db.set_raw_output(write.id, Some("raw.gps"), "op:test") + .unwrap(); + migrate_raw_outputs(&mut db).unwrap(); + let row = db.get_source(write.id).unwrap().unwrap(); + assert!(row.raw_subject.is_none()); + let moved: SourceSpec = serde_json::from_value(row.spec.clone()).unwrap(); + let topic = moved.pipeline.node("raw").expect("a NATS topic step"); + assert!(matches!(&topic.step, ot_source::flow::Step::Nats(t) if t.subject == "raw.gps")); + // Beside the Output: fed by the same step. + let feeds = |id: &str| { + moved + .pipeline + .edges + .iter() + .find(|e| e.target == id) + .unwrap() + .source + .clone() + }; + assert_eq!(feeds("raw"), feeds("output")); + assert!(moved.pipeline.shape_problems().is_empty()); + // Again: nothing more to move. + migrate_raw_outputs(&mut db).unwrap(); + assert_eq!( + db.get_source(write.id).unwrap().unwrap().revision, + row.revision + ); + } + /// A UDP listener saved before 0.4.5, without the risk acceptance, is /// not started, and its status says why; with the acceptance it runs. #[tokio::test] @@ -586,7 +657,7 @@ mod tests { redis_namespace: ns.clone(), site: ot_core::SiteCode::new("TST").unwrap(), nats: crate::config::NatsArgs { - // Nothing listens here; only raw output would use it. + // Nothing listens here; only NATS topic steps would use it. url: "nats://127.0.0.1:9".into(), creds: None, token: None, diff --git a/crates/ot-source/src/flow.rs b/crates/ot-source/src/flow.rs new file mode 100644 index 0000000..561c7c7 --- /dev/null +++ b/crates/ot-source/src/flow.rs @@ -0,0 +1,755 @@ +//! A source's processing as a flow of steps (kineticlogic-io/OpenTrack#104): +//! one input (the transport and its codec), then steps the user connects, +//! each taking one input and feeding any number of others, ending at the +//! source's one Output (system tracks, to correlation) and any NATS topics +//! (its mapped data, published as JSON without correlation). +//! +//! Records flow from the input through Reject steps to a Map, which turns +//! them into observations; Filter, Entity links, Affiliation, Tracker and +//! Throttle steps work on observations; Output and NATS topics take them. +//! +//! A spec in the older fixed form (`mapping`, `filter`, `tracker`… under +//! `pipeline`) still reads: it becomes the same steps in a line. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; + +use crate::codec::CodecConfig; +use crate::mapping::{MappingSpec, RejectRule}; +use crate::pipeline::{AffiliationStage, FilterSpec, StaticJoinSpec, ThrottleSpec}; +use crate::registry::RegistryStage; +use crate::tracker::TrackerSpec; + +/// The input step's id: the transport and codec, where every flow starts. +pub const INPUT: &str = "input"; + +/// The most steps a flow may have. +pub const MAX_NODES: usize = 200; + +/// Everything between the transport and the source's outputs. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PipelineSpec { + /// How the input's frames become records. + pub codec: CodecConfig, + pub nodes: Vec, + pub edges: Vec, +} + +/// One step of a flow. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct FlowNode { + /// 1-64 of A-Z, a-z, 0-9, `-` and `_`; the input is `input`. + pub id: String, + #[serde(flatten)] + pub step: Step, + /// Where the editor draws it; nothing runs on it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub position: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Position { + pub x: f64, + pub y: f64, +} + +/// A connection: `target` takes what `source` passes on. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FlowEdge { + pub source: String, + pub target: String, +} + +/// What a step does, with its settings. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", content = "config", rename_all = "snake_case")] +pub enum Step { + /// The transport and codec (set on the source); only `input`. + Input, + /// Drop records by condition, each rule with a reason. + Reject(Vec), + /// Turn records into observations. + Map(MapStep), + /// Keep or drop observations. + Filter(FilterSpec), + /// Find each track's entity by its identifiers (Entity links). + Registry(RegistryStage), + Affiliation(AffiliationStage), + /// Form tracks from detections. + Tracker(TrackerSpec), + Throttle(ThrottleSpec), + /// The source's system tracks, to correlation. + Output, + /// Publish what reaches it, as JSON, on a NATS subject, without + /// correlation. + Nats(NatsStep), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MapStep { + pub mapping: MappingSpec, + /// How long identity fields from static rules are kept, to fill the + /// same track's later reports. + #[serde(default)] + pub static_join: StaticJoinSpec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NatsStep { + pub subject: String, +} + +/// What passes along a connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Data { + /// Decoded records, before a Map. + Records, + /// Observations, from a Map on. + Observations, +} + +impl Step { + /// What it takes; `None`: nothing (the input). + pub fn takes(&self) -> Option { + match self { + Step::Input => None, + Step::Reject(_) | Step::Map(_) => Some(Data::Records), + _ => Some(Data::Observations), + } + } + + /// What it passes on; `None`: nothing (an output). + pub fn gives(&self) -> Option { + match self { + Step::Input | Step::Reject(_) => Some(Data::Records), + Step::Output | Step::Nats(_) => None, + _ => Some(Data::Observations), + } + } + + /// The word for it in messages. + pub fn name(&self) -> &'static str { + match self { + Step::Input => "input", + Step::Reject(_) => "reject", + Step::Map(_) => "map", + Step::Filter(_) => "filter", + Step::Registry(_) => "entity links", + Step::Affiliation(_) => "affiliation", + Step::Tracker(_) => "tracker", + Step::Throttle(_) => "throttle", + Step::Output => "output", + Step::Nats(_) => "NATS topic", + } + } +} + +/// A problem with a flow's shape, on the step it concerns (`None`: the +/// flow as a whole). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FlowProblem { + /// It cannot be saved (rather than only not run yet). + pub invalid: bool, + pub node: Option, + pub message: String, +} + +impl PipelineSpec { + /// A step by id. + pub fn node(&self, id: &str) -> Option<&FlowNode> { + self.nodes.iter().find(|n| n.id == id) + } + + /// Every Map's mapping, in step order. + pub fn mappings(&self) -> impl Iterator { + self.nodes.iter().filter_map(|n| match &n.step { + Step::Map(m) => Some(&m.mapping), + _ => None, + }) + } + + /// The extension schema version the flow's mappings target (they share + /// one; see [`Self::shape_problems`]); 1 without a Map. + pub fn schema_version(&self) -> u32 { + self.mappings().next().map_or(1, |m| m.schema_version) + } + + /// The Output step, if the flow has one. + pub fn output(&self) -> Option<&FlowNode> { + self.nodes.iter().find(|n| matches!(n.step, Step::Output)) + } + + /// The steps from the input to `id`, input first (empty if `id` is not + /// reached from the input). + pub fn path_to(&self, id: &str) -> Vec<&FlowNode> { + let parent: HashMap<&str, &str> = self + .edges + .iter() + .map(|e| (e.target.as_str(), e.source.as_str())) + .collect(); + let mut out = Vec::new(); + let mut at = id; + let mut seen = HashSet::new(); + loop { + if !seen.insert(at) { + return Vec::new(); + } + match self.node(at) { + Some(n) => out.push(n), + None if at == INPUT => {} + None => return Vec::new(), + } + if at == INPUT { + break; + } + match parent.get(at) { + Some(p) => at = p, + None => return Vec::new(), + } + } + out.reverse(); + out + } + + /// The Entity links settings the source's system tracks pass through: + /// the last such step before the Output, else the defaults. + pub fn output_registry(&self) -> RegistryStage { + self.output() + .map(|o| self.path_to(&o.id)) + .unwrap_or_default() + .into_iter() + .rev() + .find_map(|n| match &n.step { + Step::Registry(r) => Some(r.clone()), + _ => None, + }) + .unwrap_or_default() + } + + /// Whether the source's system tracks come through a Tracker. + pub fn output_tracked(&self) -> bool { + self.output().is_some_and(|o| { + self.path_to(&o.id) + .iter() + .any(|n| matches!(n.step, Step::Tracker(_))) + }) + } + + /// The steps' children, by id. + pub fn children(&self) -> HashMap<&str, Vec<&str>> { + let mut out: HashMap<&str, Vec<&str>> = HashMap::new(); + for e in &self.edges { + out.entry(e.source.as_str()) + .or_default() + .push(e.target.as_str()); + } + out + } + + /// What is wrong with the flow's shape: ids, connections, what each + /// step takes and gives, and whether it leads to an output. Problems + /// with a step's own settings are the step's (see `SourceSpec::problems`). + pub fn shape_problems(&self) -> Vec { + let mut out = Vec::new(); + let invalid = |node: Option<&str>, m: String| FlowProblem { + invalid: true, + node: node.map(str::to_owned), + message: m, + }; + let incomplete = |node: Option<&str>, m: String| FlowProblem { + invalid: false, + node: node.map(str::to_owned), + message: m, + }; + if self.nodes.len() > MAX_NODES { + out.push(invalid( + None, + format!("a flow has at most {MAX_NODES} steps"), + )); + } + let mut ids = HashSet::new(); + for n in &self.nodes { + let ok = !n.id.is_empty() + && n.id.len() <= 64 + && n.id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_'); + if !ok { + out.push(invalid( + Some(&n.id), + format!("step id {:?}: 1-64 of letters, digits, - and _", n.id), + )); + } + if !ids.insert(n.id.as_str()) { + out.push(invalid( + Some(&n.id), + format!("two steps are called {:?}", n.id), + )); + } + if (n.id == INPUT) != matches!(n.step, Step::Input) { + out.push(invalid( + Some(&n.id), + "only the step called \"input\" is the input, and it is the input".into(), + )); + } + } + let outputs = self + .nodes + .iter() + .filter(|n| matches!(n.step, Step::Output)) + .count(); + if outputs > 1 { + out.push(invalid(None, "a source has one Output".into())); + } + let gives = |id: &str| match self.node(id) { + Some(n) => n.step.gives(), + None if id == INPUT => Some(Data::Records), + None => None, + }; + let mut incoming: HashMap<&str, usize> = HashMap::new(); + let mut pairs = HashSet::new(); + for e in &self.edges { + let (from, to) = (self.node(&e.source), self.node(&e.target)); + if from.is_none() && e.source != INPUT { + out.push(invalid( + None, + format!("a connection from {:?}, which is not a step", e.source), + )); + continue; + } + let Some(to) = to else { + out.push(invalid( + None, + format!("a connection to {:?}, which is not a step", e.target), + )); + continue; + }; + if !pairs.insert((e.source.as_str(), e.target.as_str())) { + out.push(invalid( + Some(&to.id), + "it is connected twice from one step".into(), + )); + continue; + } + *incoming.entry(e.target.as_str()).or_default() += 1; + match (gives(&e.source), to.step.takes()) { + (_, None) => out.push(invalid( + Some(&to.id), + "nothing connects into the input".into(), + )), + (None, _) => out.push(invalid( + Some(&e.source), + format!( + "a {} passes nothing on", + from.map_or("step", |f| f.step.name()) + ), + )), + (Some(Data::Records), Some(Data::Observations)) => out.push(invalid( + Some(&to.id), + format!( + "a {} takes observations: put a Map before it", + to.step.name() + ), + )), + (Some(Data::Observations), Some(Data::Records)) => out.push(invalid( + Some(&to.id), + format!("a {} takes records, before a Map", to.step.name()), + )), + _ => {} + } + } + for (id, n) in &incoming { + if *n > 1 { + out.push(invalid(Some(id), "a step takes one input".into())); + } + } + if !out.is_empty() { + return out; + } + // Every step is reached from the input, and leads to an output. + let children = self.children(); + let mut reached = HashSet::new(); + let mut stack = vec![INPUT]; + while let Some(at) = stack.pop() { + if reached.insert(at) { + stack.extend(children.get(at).into_iter().flatten().copied()); + } + } + // A loop is a step with one input that the input never reaches. + let mut sinks = 0; + for n in &self.nodes { + if n.id == INPUT { + continue; + } + if !reached.contains(n.id.as_str()) { + if incoming.contains_key(n.id.as_str()) { + let looped = self.in_loop(&n.id); + if looped { + out.push(invalid(Some(&n.id), "the flow loops back on itself".into())); + continue; + } + } + out.push(incomplete(Some(&n.id), "this step has no input".into())); + continue; + } + if n.step.gives().is_none() { + sinks += 1; + } else if !children.contains_key(n.id.as_str()) { + out.push(incomplete(Some(&n.id), "this step leads nowhere".into())); + } + } + if out.iter().any(|p| p.invalid) { + return out; + } + if !children.contains_key(INPUT) { + out.push(incomplete(Some(INPUT), "the input leads nowhere".into())); + } + if sinks == 0 { + out.push(incomplete( + None, + "a flow needs an Output or a NATS topic".into(), + )); + } + let versions: BTreeSet = self.mappings().map(|m| m.schema_version).collect(); + if versions.len() > 1 { + out.push(incomplete( + None, + format!("the Maps target different schema versions ({versions:?}); use one"), + )); + } + out + } + + /// Whether following inputs back from `id` comes round to it. + fn in_loop(&self, id: &str) -> bool { + let parent: HashMap<&str, &str> = self + .edges + .iter() + .map(|e| (e.target.as_str(), e.source.as_str())) + .collect(); + let mut at = id; + let mut seen = HashSet::new(); + while let Some(p) = parent.get(at) { + if *p == id { + return true; + } + if !seen.insert(*p) { + return false; + } + at = p; + } + false + } +} + +/// The older fixed form: one of each stage, in a fixed order. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct FixedPipeline { + codec: CodecConfig, + mapping: MappingSpec, + #[serde(default)] + static_join: StaticJoinSpec, + #[serde(default)] + registry: Option, + #[serde(default)] + affiliation: Option, + #[serde(default)] + filter: Option, + #[serde(default)] + tracker: Option, + #[serde(default)] + throttle: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct GraphPipeline { + codec: CodecConfig, + #[serde(default)] + nodes: Vec, + #[serde(default)] + edges: Vec, +} + +impl From for PipelineSpec { + /// The same stages, in a line, with the ids the stages had. The entity + /// stage, which always ran, is a step of its own. + fn from(p: FixedPipeline) -> Self { + let mut mapping = p.mapping; + let reject = std::mem::take(&mut mapping.reject); + let mut nodes = vec![node(INPUT, Step::Input)]; + if !reject.is_empty() { + nodes.push(node("reject", Step::Reject(reject))); + } + nodes.push(node( + "map", + Step::Map(MapStep { + mapping, + static_join: p.static_join, + }), + )); + nodes.push(node( + "registry", + Step::Registry(p.registry.unwrap_or_default()), + )); + if let Some(a) = p.affiliation { + nodes.push(node("affiliation", Step::Affiliation(a))); + } + if let Some(f) = p.filter { + nodes.push(node("filter", Step::Filter(f))); + } + if let Some(t) = p.tracker { + nodes.push(node("tracker", Step::Tracker(t))); + } + if let Some(t) = p.throttle { + nodes.push(node("throttle", Step::Throttle(t))); + } + nodes.push(node("output", Step::Output)); + let edges = nodes + .windows(2) + .map(|w| FlowEdge { + source: w[0].id.clone(), + target: w[1].id.clone(), + }) + .collect(); + PipelineSpec { + codec: p.codec, + nodes, + edges, + } + } +} + +fn node(id: &str, step: Step) -> FlowNode { + FlowNode { + id: id.to_owned(), + step, + position: None, + } +} + +impl<'de> Deserialize<'de> for PipelineSpec { + /// A flow (`nodes` and `edges`), or the older fixed form, read as the + /// same stages in a line. The input step is added if left out. + fn deserialize>(d: D) -> Result { + let v = Value::deserialize(d)?; + let graph = v.get("nodes").is_some() || v.get("edges").is_some(); + let mut spec: PipelineSpec = if graph { + let g: GraphPipeline = serde_json::from_value(v).map_err(serde::de::Error::custom)?; + PipelineSpec { + codec: g.codec, + nodes: g.nodes, + edges: g.edges, + } + } else { + serde_json::from_value::(v) + .map_err(serde::de::Error::custom)? + .into() + }; + if spec.node(INPUT).is_none() { + spec.nodes.insert(0, node(INPUT, Step::Input)); + } + Ok(spec) + } +} + +impl PipelineSpec { + /// A flow in a line through these steps: input, then each, ending at an + /// Output. For tests and simple sources. + pub fn line(codec: CodecConfig, steps: Vec<(&str, Step)>) -> Self { + let mut nodes = vec![node(INPUT, Step::Input)]; + nodes.extend(steps.into_iter().map(|(id, s)| node(id, s))); + let edges = nodes + .windows(2) + .map(|w| FlowEdge { + source: w[0].id.clone(), + target: w[1].id.clone(), + }) + .collect(); + PipelineSpec { + codec, + nodes, + edges, + } + } +} + +impl PipelineSpec { + /// Add a NATS topic step publishing on `subject` what the Output takes + /// (fed by the Output's step before it), as the raw output of a source + /// from before flows did. Returns its id; `None` if there is no Output + /// fed by a step. + pub fn add_output_topic(&mut self, subject: &str) -> Option { + let out = self.output()?.id.clone(); + let parent = self.edges.iter().find(|e| e.target == out)?.source.clone(); + let mut id = "raw".to_owned(); + let mut n = 1; + while self.node(&id).is_some() { + n += 1; + id = format!("raw-{n}"); + } + self.nodes.push(node( + &id, + Step::Nats(NatsStep { + subject: subject.to_owned(), + }), + )); + self.edges.push(FlowEdge { + source: parent, + target: id.clone(), + }); + Some(id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn spec(v: Value) -> PipelineSpec { + serde_json::from_value(v).unwrap() + } + + fn map() -> Value { + json!({"rules": [{"name": "r", "key": "id", + "fields": {"position.latitude": "lat", "position.longitude": "lon"}}]}) + } + + #[test] + fn the_fixed_form_reads_as_its_stages_in_a_line() { + let p = spec(json!({ + "codec": {"type": "json"}, + "mapping": {"reject": [{"reason": "x", "when": {"path": "a", "exists": false}}], "rules": map()["rules"]}, + "filter": {"drop_if": {"path": "name", "eq": "N"}}, + "throttle": {"min_interval_secs": 5} + })); + let ids: Vec<&str> = p.nodes.iter().map(|n| n.id.as_str()).collect(); + assert_eq!( + ids, + [ + "input", "reject", "map", "registry", "filter", "throttle", "output" + ] + ); + assert_eq!(p.edges.len(), 6); + assert!(p.shape_problems().is_empty(), "{:?}", p.shape_problems()); + let Step::Map(m) = &p.node("map").unwrap().step else { + panic!() + }; + assert!( + m.mapping.reject.is_empty(), + "the rejects moved to their own step" + ); + // Saved, it reads back the same. + let again: PipelineSpec = + serde_json::from_value(serde_json::to_value(&p).unwrap()).unwrap(); + assert_eq!(again, p); + assert!(p.output_registry() == RegistryStage::default() && !p.output_tracked()); + } + + #[test] + fn a_flow_branches_to_an_output_and_nats_topics() { + let p = spec(json!({ + "codec": {"type": "json"}, + "nodes": [ + {"id": "m", "type": "map", "config": {"mapping": map()}}, + {"id": "f1", "type": "filter", "config": {"keep_if": {"path": "name", "exists": true}}}, + {"id": "f2", "type": "filter", "config": {}}, + {"id": "out", "type": "output"}, + {"id": "raw", "type": "nats", "config": {"subject": "raw.feed"}} + ], + "edges": [ + {"source": "input", "target": "m"}, {"source": "m", "target": "f1"}, + {"source": "f1", "target": "f2"}, {"source": "f2", "target": "out"}, + {"source": "m", "target": "raw"} + ] + })); + assert_eq!(p.nodes[0].id, "input", "the input is added"); + assert!(p.shape_problems().is_empty(), "{:?}", p.shape_problems()); + let path: Vec<&str> = p.path_to("out").iter().map(|n| n.id.as_str()).collect(); + assert_eq!(path, ["input", "m", "f1", "f2", "out"]); + } + + #[test] + fn shape_problems_name_their_step() { + let base = || { + json!({"codec": {"type": "json"}, "nodes": [ + {"id": "m", "type": "map", "config": {"mapping": map()}}, + {"id": "f", "type": "filter", "config": {}}, + {"id": "out", "type": "output"} + ], "edges": [{"source": "input", "target": "m"}, {"source": "m", "target": "f"}, {"source": "f", "target": "out"}]}) + }; + let problems = |v: Value| -> Vec<(bool, Option)> { + spec(v) + .shape_problems() + .into_iter() + .map(|p| (p.invalid, p.node)) + .collect() + }; + // A filter before the map. + let mut v = base(); + v["edges"] = json!([{"source": "input", "target": "f"}, {"source": "f", "target": "m"}, {"source": "m", "target": "out"}]); + assert_eq!( + problems(v), + [(true, Some("f".into())), (true, Some("m".into()))] + ); + // Two inputs into one step. + let mut v = base(); + v["edges"] + .as_array_mut() + .unwrap() + .push(json!({"source": "input", "target": "f"})); + assert!(problems(v).contains(&(true, Some("f".into())))); + // A loop. + let mut v = base(); + v["nodes"] + .as_array_mut() + .unwrap() + .push(json!({"id": "g", "type": "filter", "config": {}})); + v["nodes"] + .as_array_mut() + .unwrap() + .push(json!({"id": "h", "type": "filter", "config": {}})); + v["edges"].as_array_mut().unwrap().extend([ + json!({"source": "g", "target": "h"}), + json!({"source": "h", "target": "g"}), + ]); + assert!(problems(v).contains(&(true, Some("g".into())))); + // Two outputs. + let mut v = base(); + v["nodes"] + .as_array_mut() + .unwrap() + .push(json!({"id": "out2", "type": "output"})); + assert!(problems(v).contains(&(true, None))); + // Loose ends are only incomplete: a draft. + let mut v = base(); + v["nodes"] + .as_array_mut() + .unwrap() + .push(json!({"id": "t", "type": "throttle", "config": {}})); + v["edges"] + .as_array_mut() + .unwrap() + .push(json!({"source": "f", "target": "t"})); + assert_eq!(problems(v), [(false, Some("t".into()))]); + let mut v = base(); + v["edges"] = json!([{"source": "input", "target": "m"}]); + assert_eq!( + problems(v), + [ + (false, Some("m".into())), + (false, Some("f".into())), + (false, Some("out".into())), + (false, None) + ] + ); + } +} diff --git a/crates/ot-source/src/lib.rs b/crates/ot-source/src/lib.rs index 7c17b5e..15154f6 100644 --- a/crates/ot-source/src/lib.rs +++ b/crates/ot-source/src/lib.rs @@ -6,6 +6,7 @@ pub mod codec; pub mod expr; +pub mod flow; pub mod frame; pub mod grpc; pub mod mapping; diff --git a/crates/ot-source/src/pipeline.rs b/crates/ot-source/src/pipeline.rs index c1c8c39..a1e9a0b 100644 --- a/crates/ot-source/src/pipeline.rs +++ b/crates/ot-source/src/pipeline.rs @@ -1,10 +1,9 @@ -//! The per-source processing pipeline, free of I/O: +//! The per-source processing pipeline, free of I/O: a frame is decoded into +//! records, and each record follows the source's flow of steps +//! ([`crate::flow`]) to its Output (system tracks) and NATS topics. //! -//! frame → codec → records → reject → mapping → static join → registry → -//! affiliation → filter → tracker → throttle → observation. -//! -//! The tracker stage is optional: it turns a detection feed's plots into -//! source tracks (see [`crate::tracker`]). +//! A Tracker step turns a detection feed's plots into source tracks (see +//! [`crate::tracker`]). //! //! The worker feeds frames in and ships observations out; everything here is //! deterministic given its inputs, so it can be replayed in tests and probes. @@ -17,37 +16,15 @@ use ot_core::{Identifier, Observation}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::codec::{Codec, CodecConfig}; +use crate::codec::Codec; use crate::expr::{Condition, ValueSpec, as_string}; use crate::frame::Frame; use crate::mapping::{self, MapOutcome, MappingSpec, RuleKind}; use crate::registry::{RegistryLookup, RegistryStage}; use crate::trace::{SampleTrace, Trace}; -/// Everything between the transport and the observation stream. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PipelineSpec { - pub codec: CodecConfig, - pub mapping: MappingSpec, - #[serde(default)] - pub static_join: StaticJoinSpec, - #[serde(default, skip_serializing_if = "Option::is_none")] - /// Registry resolution settings. Without them the registry still runs - /// with defaults: identifiers are resolved and graded under - /// `ext.registry`, and the entity's OTH-GOLD minimum populates the track - /// (see [`crate::registry::default_links`]). - pub registry: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub affiliation: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub filter: Option, - /// Form tracks from detections (GNN or MHT) before they leave the source. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tracker: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub throttle: Option, -} +pub use crate::flow::PipelineSpec; +use crate::flow::{INPUT, Step}; #[derive(Debug, Clone, PartialEq, thiserror::Error)] pub enum PipelineError { @@ -57,6 +34,9 @@ pub enum PipelineError { Tracker(String), #[error("{0}")] Codec(String), + /// The flow's shape cannot run (a loop, a step fed the wrong data). + #[error("{0}")] + Flow(String), } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -248,7 +228,9 @@ pub struct Counts { /// Runs of a tracker plugin that failed. pub tracker_errors: u64, pub throttled: u64, + /// Observations sent to the Output, and items published on NATS topics. pub emitted: u64, + pub published: u64, pub grades: HashMap<&'static str, u64>, } @@ -268,6 +250,7 @@ impl Counts { ("tracker_error", self.tracker_errors), ("throttled", self.throttled), ("emitted", self.emitted), + ("published", self.published), ] .into_iter() .filter(|(_, n)| *n > 0) @@ -298,30 +281,78 @@ pub struct Output { pub entity_updates: Vec<(String, Vec<(String, Value)>)>, /// The last decode or validation error seen, for status reporting. pub last_error: Option, - /// Plots given to the tracker stage, when [`Pipeline::keep_plots`] is on. + /// Plots given to a Tracker step, when [`Pipeline::keep_plots`] is on. pub plots: Vec, + /// What reached each NATS topic step: its subject and the item as JSON. + pub published: Vec<(String, Value)>, } pub struct Pipeline { source_id: String, spec: PipelineSpec, codec: Codec, - statics: HashMap, - throttle: HashMap, - /// Source tracks whose static details changed since their last write. - details_changed: HashSet, - /// Extension field types for the mapping's schema version. + /// Each step's state, in `spec.nodes` order. + steps: Vec, + /// Each step's children, by index. + children: Vec>, + /// Whether an Entity links step comes before each step. + linked: Vec, + input: usize, + /// The Map whose static cache is stored under the plain track key. + primary_map: Option, + /// The steps a trace records, input first, in the order they are met. + order: Vec, + /// Extension field types for the mappings' schema version. schema: Option, pub counts: Counts, - /// Used when the spec configures no registry stage. + /// Run before the Output when no Entity links step comes before it. default_registry: RegistryStage, /// The last value sent to each (entity, field), so a feed repeating a /// value writes it once, not until the registry snapshot catches up. entity_sent: HashMap<(String, String), Value>, - tracker: Option, keep_plots: bool, } +/// A step's settings and state. +enum Run { + Input, + Reject(Vec), + Map { + spec: MappingSpec, + ttl_secs: u64, + statics: HashMap, + /// Tracks whose static details changed since their last report. + changed: HashSet, + }, + Filter(FilterSpec), + Registry(RegistryStage), + Affiliation(AffiliationStage), + Tracker(Stage), + Throttle { + spec: ThrottleSpec, + last: HashMap, + /// Tracks whose details changed since they last passed. + pending: HashSet, + }, + Output, + Nats(String), +} + +/// What passes between steps: a record, or an observation's JSON form. +#[derive(Clone)] +struct Item { + value: Value, + /// The mapping rule's kind, for observations. + kind: RuleKind, + /// Its track's static details changed since its last report. + changed: bool, + /// The entity set its affiliation (the country lists then keep out). + entity_affiliation: bool, + received_at: DateTime, + /// The traced sample it belongs to. + sample: Option, +} + /// The tracker stage: OpenTrack's own tracker, or a plugin's. enum Stage { Builtin(Box), @@ -355,24 +386,95 @@ impl Stage { } } +/// Separates a Map's id from a track key in a stored static cache key; the +/// primary Map's keys are plain. +const STATIC_KEY_SEP: char = '\u{1f}'; + impl Pipeline { pub fn new(source_id: impl Into, spec: PipelineSpec) -> Result { - spec.mapping.validate()?; - let tracker = spec - .tracker - .clone() - .map(Stage::new) - .transpose() - .map_err(PipelineError::Tracker)?; + if let Some(p) = spec.shape_problems().into_iter().find(|p| p.invalid) { + return Err(PipelineError::Flow(match p.node { + Some(n) => format!("step {n}: {}", p.message), + None => p.message, + })); + } + let index: HashMap<&str, usize> = spec + .nodes + .iter() + .enumerate() + .map(|(i, n)| (n.id.as_str(), i)) + .collect(); + let mut steps = Vec::with_capacity(spec.nodes.len()); + for n in &spec.nodes { + steps.push(match &n.step { + Step::Input => Run::Input, + Step::Reject(r) => Run::Reject(r.clone()), + Step::Map(m) => { + m.mapping.validate()?; + Run::Map { + spec: m.mapping.clone(), + ttl_secs: m.static_join.ttl_secs, + statics: HashMap::new(), + changed: HashSet::new(), + } + } + Step::Filter(f) => Run::Filter(f.clone()), + Step::Registry(r) => Run::Registry(r.clone()), + Step::Affiliation(a) => Run::Affiliation(a.clone()), + Step::Tracker(t) => { + Run::Tracker(Stage::new(t.clone()).map_err(PipelineError::Tracker)?) + } + Step::Throttle(t) => Run::Throttle { + spec: t.clone(), + last: HashMap::new(), + pending: HashSet::new(), + }, + Step::Output => Run::Output, + Step::Nats(n) => Run::Nats(n.subject.clone()), + }); + } + let mut children = vec![Vec::new(); spec.nodes.len()]; + for e in &spec.edges { + if let (Some(&from), Some(&to)) = + (index.get(e.source.as_str()), index.get(e.target.as_str())) + { + children[from].push(to); + } + } + let input = index[INPUT]; + // Breadth first from the input: the order steps are met in, and + // whether an Entity links step comes before each. + let mut order = Vec::new(); + let mut linked = vec![false; spec.nodes.len()]; + let mut queue = std::collections::VecDeque::from([input]); + let mut seen = HashSet::new(); + while let Some(at) = queue.pop_front() { + if !seen.insert(at) { + continue; + } + order.push(spec.nodes[at].id.clone()); + let here = linked[at] || matches!(spec.nodes[at].step, Step::Registry(_)); + for &c in &children[at] { + linked[c] = here; + queue.push_back(c); + } + } + let primary_map = index + .get("map") + .copied() + .filter(|&i| matches!(steps[i], Run::Map { .. })) + .or_else(|| steps.iter().position(|s| matches!(s, Run::Map { .. }))); Ok(Self { - tracker, source_id: source_id.into(), codec: Codec::new(spec.codec.clone()) .map_err(|e| PipelineError::Codec(e.to_string()))?, spec, - statics: HashMap::new(), - throttle: HashMap::new(), - details_changed: HashSet::new(), + steps, + children, + linked, + input, + primary_map, + order, schema: None, counts: Counts::default(), default_registry: RegistryStage::default(), @@ -381,7 +483,7 @@ impl Pipeline { }) } - /// Also return the plots the tracker stage is given (benchmarks score + /// Also return the plots the Tracker steps are given (benchmarks score /// what the sensor saw as well as what the tracker made of it). pub fn keep_plots(mut self, on: bool) -> Self { self.keep_plots = on; @@ -402,9 +504,47 @@ impl Pipeline { &self.spec } - /// Seed the static cache (e.g. from Redis after a restart). + /// Whether it has a Tracker step (the worker then runs it on a timer). + pub fn has_tracker(&self) -> bool { + self.steps.iter().any(|s| matches!(s, Run::Tracker(_))) + } + + /// The key a Map's static cache entry is stored under. + fn static_key(&self, map: usize, key: &str) -> String { + if Some(map) == self.primary_map { + key.to_owned() + } else { + format!("{}{STATIC_KEY_SEP}{key}", self.spec.nodes[map].id) + } + } + + /// Seed a Map's static cache (e.g. from Redis after a restart), by the + /// key [`Output::statics`] gave it. pub fn load_static(&mut self, key: String, entry: StaticEntry) { - self.statics.insert(key, entry); + let (map, key) = match key.split_once(STATIC_KEY_SEP) { + Some((id, k)) => match self.spec.nodes.iter().position(|n| n.id == id) { + Some(i) => (Some(i), k.to_owned()), + None => return, + }, + None => (self.primary_map, key), + }; + if let Some(Run::Map { statics, .. }) = map.and_then(|i| self.steps.get_mut(i)) { + statics.insert(key, entry); + } + } + + /// The longest any Map keeps its static details. + pub fn static_ttl(&self) -> Duration { + let secs = self + .steps + .iter() + .filter_map(|s| match s { + Run::Map { ttl_secs, .. } => Some(*ttl_secs), + _ => None, + }) + .max() + .unwrap_or(0); + Duration::from_secs(secs) } /// Take and reset the counters. @@ -412,25 +552,32 @@ impl Pipeline { std::mem::take(&mut self.counts) } - /// Forget throttle and static state older than the static TTL. + /// Forget throttle and static state older than they are kept. pub fn prune(&mut self, now: DateTime) { - let ttl = chrono::Duration::from_std(Duration::from_secs(self.spec.static_join.ttl_secs)) - .unwrap_or(chrono::Duration::MAX); - self.statics.retain(|_, e| now - e.updated_at < ttl); - let heartbeat = self - .spec - .throttle - .as_ref() - .map_or(0.0, |t| t.heartbeat_secs); - let keep = chrono::Duration::milliseconds((heartbeat * 1000.0) as i64 * 2); - self.throttle.retain(|_, w| now - w.at < keep); + for s in &mut self.steps { + match s { + Run::Map { + ttl_secs, statics, .. + } => { + let ttl = chrono::Duration::from_std(Duration::from_secs(*ttl_secs)) + .unwrap_or(chrono::Duration::MAX); + statics.retain(|_, e| now - e.updated_at < ttl); + } + Run::Throttle { spec, last, .. } => { + let keep = + chrono::Duration::milliseconds((spec.heartbeat_secs * 1000.0) as i64 * 2); + last.retain(|_, w| now - w.at < keep); + } + _ => {} + } + } } pub fn process(&mut self, frame: &Frame, registry: &dyn RegistryLookup) -> Output { self.run_frame(frame, registry, None) } - /// [`Self::process`], recording in `trace` what each stage made of the + /// [`Self::process`], recording in `trace` what each step made of the /// frame's records while the trace has room (see [`crate::trace`]). pub fn process_traced( &mut self, @@ -441,41 +588,6 @@ impl Pipeline { self.run_frame(frame, registry, Some(trace)) } - /// The stages a trace records, after the transport, in the order they - /// run, with the ids the UI gives them. - fn trace_stages(&self) -> Vec<&'static str> { - let s = &self.spec; - let mut out = vec!["decode"]; - if !s.mapping.reject.is_empty() { - out.push("reject"); - } - out.push("map"); - if self.has_join() { - out.push("join"); - } - out.push("registry"); - for (id, on) in [ - ("affiliation", s.affiliation.is_some()), - ("filter", s.filter.is_some()), - ("tracker", s.tracker.is_some()), - ("throttle", s.throttle.is_some()), - ] { - if on { - out.push(id); - } - } - out - } - - /// Whether static rules feed an identity join. - fn has_join(&self) -> bool { - self.spec - .mapping - .rules - .iter() - .any(|r| r.kind == RuleKind::Static) - } - fn run_frame( &mut self, frame: &Frame, @@ -484,214 +596,345 @@ impl Pipeline { ) -> Output { let mut out = Output::default(); self.counts.frames += 1; - let stages = match trace.as_deref_mut() { - Some(t) => { - t.start_frame(); - self.trace_stages() - } - None => Vec::new(), - }; + if let Some(t) = trace.as_deref_mut() { + t.start_frame(); + } let records = match self.codec.decode(frame) { Ok(r) => r, Err(e) => { self.counts.decode_errors += 1; // A frame that does not decode is a sample of its own. if let Some(t) = trace.as_deref_mut() - && let Some(at) = t.begin(frame, &stages) + && let Some(at) = t.begin(frame, &self.order) { - t.samples[at].dropped("decode", format!("decode error: {e}")); + t.samples[at].dropped(INPUT, format!("decode error: {e}")); } out.last_error = Some(e.to_string()); return out; } }; // A tracker timed by the sensor follows what the codec has learnt. - if let Some(Stage::Builtin(t)) = self.tracker.as_mut() - && let Some(h) = self.codec.hints() - { - t.set_revisit(h.revisit_secs, h.revisit_source); + if let Some(h) = self.codec.hints() { + for s in &mut self.steps { + if let Run::Tracker(Stage::Builtin(t)) = s { + t.set_revisit(h.revisit_secs, h.revisit_source); + } + } } for record in records { - let mut f = match trace.as_deref_mut() { - Some(t) => t.begin(frame, &stages).map(|at| &mut t.samples[at]), - None => None, - }; - if let Some(f) = f.as_deref_mut() { - f.item("decode", || record.clone()); + self.counts.records += 1; + let sample = trace + .as_deref_mut() + .and_then(|t| t.begin(frame, &self.order)); + if let (Some(t), Some(at)) = (trace.as_deref_mut(), sample) { + t.samples[at].item(INPUT, || record.clone()); } - self.process_record(&record, frame.received_at, registry, &mut out, f); + let item = Item { + value: record, + kind: RuleKind::Observation, + changed: false, + entity_affiliation: false, + received_at: frame.received_at, + sample, + }; + self.pass_on(self.input, item, registry, &mut out, &mut trace); } - self.run_tracker(frame.received_at, false, &mut out, trace.as_deref_mut()); + self.run_trackers(frame.received_at, false, registry, &mut out, &mut trace); // Plots still waiting for the rest of their scan. - if let Some(Stage::Builtin(t)) = self.tracker.as_ref() - && let Some(tr) = trace - { - for f in &mut tr.samples { - if !f.plots.is_empty() - && t.waiting().any(|w| f.plots.contains(w)) - && let Some(s) = f.stage("tracker") - { - s.held = true; + if let Some(tr) = trace { + for (i, s) in self.steps.iter().enumerate() { + let Run::Tracker(Stage::Builtin(t)) = s else { + continue; + }; + let id = &self.spec.nodes[i].id; + for f in &mut tr.samples { + let mine: Vec<&Observation> = f + .plots + .iter() + .filter(|(n, _)| n == id) + .map(|(_, o)| o) + .collect(); + if !mine.is_empty() + && t.waiting().any(|w| mine.contains(&w)) + && let Some(st) = f.stage(id) + { + st.held = true; + } } } } out } - /// The windows a tracker with `auto_timing` is using, once it knows the - /// sensor's revisit period. - pub fn tracker_timing(&self) -> Option<&crate::tracker::Timing> { - match self.tracker.as_ref()? { - Stage::Builtin(t) => t.timing(), - Stage::Plugin { .. } => None, - } - } - - /// Run the tracker on scans still waiting (plots grouped by time wait - /// `scan_hold_secs` for more; `force` runs them regardless). The worker - /// calls this on a timer; a dry run calls it with `force` at the end. - pub fn flush(&mut self, now: DateTime, force: bool) -> Output { - let mut out = Output::default(); - self.run_tracker(now, force, &mut out, None); - out - } - - /// [`Self::flush`], adding the reports to the traced samples whose plots - /// they came from. With `force` (the end of a dry run), a sample whose - /// plots gave no report is marked so. - pub fn flush_traced(&mut self, now: DateTime, force: bool, trace: &mut Trace) -> Output { - let mut out = Output::default(); - self.run_tracker(now, force, &mut out, Some(trace)); - if force { - let why = match self.tracker { - Some(Stage::Plugin { .. }) => { - "a tracker plugin's reports are not traced back to the records behind them" - } - _ => { - "no track report: the tracker has not confirmed a track from these plots (or merged them into a nearby plot, or they came too late for their scan)" - } - }; - for f in &mut trace.samples { - let plots = !f.plots.is_empty(); - if let Some(s) = f.stage("tracker") - && plots - && s.items.is_empty() - && s.dropped.is_empty() - { - s.dropped.push(why.to_owned()); - } + /// Give `item` to each of `at`'s children. + fn pass_on( + &mut self, + at: usize, + item: Item, + registry: &dyn RegistryLookup, + out: &mut Output, + trace: &mut Option<&mut Trace>, + ) { + let n = self.children[at].len(); + for i in 0..n { + let c = self.children[at][i]; + if i + 1 < n { + self.step(c, item.clone(), registry, out, trace); + } else { + self.step(c, item, registry, out, trace); + return; } } - out } - /// Run the tracker stage. When tracing, each report of the built-in - /// tracker is added to the sample whose detection it came from. - fn run_tracker( + /// One step's work on one item. + fn step( &mut self, - now: DateTime, - force: bool, + at: usize, + mut item: Item, + registry: &dyn RegistryLookup, out: &mut Output, - mut trace: Option<&mut Trace>, + trace: &mut Option<&mut Trace>, ) { - // The detection behind each report, only when tracing. - let mut dets: Vec = Vec::new(); - let reports = match self.tracker.as_mut() { - None => return, - Some(Stage::Builtin(t)) => { - let force = force || t.spec().scans == crate::tracker::ScanGrouping::Frame; - let reports = t.run(now, force); - self.counts.late_plots += std::mem::take(&mut t.late); - if trace.is_some() { - dets = reports.iter().map(|(_, d)| d.clone()).collect(); + let id = self.spec.nodes[at].id.clone(); + let mut sample = match (trace.as_deref_mut(), item.sample) { + (Some(t), Some(s)) => t.samples.get_mut(s), + _ => None, + }; + match &mut self.steps[at] { + Run::Input => {} + Run::Reject(rules) => { + if let Some(r) = rules.iter().find(|r| r.when.eval(&item.value)) { + if let Some(f) = sample { + f.dropped(&id, format!("rejected: {}", r.reason)); + } + *self.counts.rejected.entry(r.reason.clone()).or_default() += 1; + return; + } + if let Some(f) = sample { + f.item(&id, || item.value.clone()); } - reports.into_iter().map(|(obs, _)| obs).collect() } - Some(Stage::Plugin { tracker, version }) => match tracker.run(now, force) { - Ok(tracks) => { - let mut tracks: Vec = tracks; - for t in &mut tracks { - t.source_id = self.source_id.clone(); - t.provenance.tracker = Some(version.clone()); + Run::Map { .. } => return self.map(at, &id, item, registry, out, trace), + Run::Filter(filter) => { + if let Some(why) = filter.verdict(&item.value) { + self.counts.filtered += 1; + if let Some(f) = sample { + f.dropped(&id, why); } - tracks + return; } - Err(e) => { - self.counts.tracker_errors += 1; - out.last_error = Some(format!("tracker plugin: {e}")); - Vec::new() + if let Some(f) = sample { + f.item(&id, || item.value.clone()); } - }, - }; - for (i, obs) in reports.into_iter().enumerate() { - let mut f = None; - if let Some(t) = trace.as_deref_mut() { - f = dets - .get(i) - .and_then(|d| t.samples.iter_mut().find(|f| f.plots.contains(d))); - if let Some(f) = f.as_deref_mut() { - f.obs("tracker", &obs); + } + Run::Registry(stage) => { + let stage = stage.clone(); + self.link(&stage, &mut item, registry, out); + if let Some(f) = sample { + f.item(&id, || item.value.clone()); } } - self.emit(obs, out, f); + Run::Affiliation(stage) => { + if !item.entity_affiliation { + stage.run(&mut item.value); + } + if let Some(f) = sample { + f.item(&id, || item.value.clone()); + if item.entity_affiliation { + f.note(&id, "kept the entity's affiliation"); + } + } + } + Run::Tracker(_) => { + let Some(obs) = self.typed(&id, &item, out, sample.as_deref_mut()) else { + return; + }; + if item.kind == RuleKind::Observation { + self.counts.plots += 1; + if self.keep_plots { + out.plots.push(obs.clone()); + } + if let Some(f) = sample { + f.plots.push((id.clone(), obs.clone())); + } + if let Run::Tracker(t) = &mut self.steps[at] { + t.push(obs, item.received_at); + } + return; + } + // A report with its own identity passes the tracker by. + if let Some(f) = sample { + f.obs(&id, &obs); + } + } + Run::Throttle { .. } => { + let Some(obs) = self.typed(&id, &item, out, sample.as_deref_mut()) else { + return; + }; + let Run::Throttle { + spec, + last, + pending, + } = &mut self.steps[at] + else { + unreachable!() + }; + let key = obs.source_track_key.clone(); + if obs.state == Some(ot_core::TrackState::Dropped) { + // A source track's end always passes. + last.remove(&key); + pending.remove(&key); + } else { + if item.changed { + pending.insert(key.clone()); + } + let prev = last.get(&key); + if !spec.due(prev, &obs, pending.contains(&key)) { + self.counts.throttled += 1; + if let Some(f) = sample { + f.dropped(&id, spec.why_not(prev, &obs)); + } + return; + } + pending.remove(&key); + last.insert( + key, + LastWrite { + at: obs.observed_at, + lat: obs.position.latitude, + lon: obs.position.longitude, + }, + ); + } + if let Some(f) = sample { + f.obs(&id, &obs); + } + } + Run::Output => { + // Tracks reach correlation with their entity looked up. + if !self.linked[at] { + let stage = self.default_registry.clone(); + self.link(&stage, &mut item, registry, out); + } + let Some(obs) = self.typed(&id, &item, out, sample.as_deref_mut()) else { + return; + }; + self.counts.emitted += 1; + if let Some(f) = sample { + f.obs(&id, &obs); + f.emitted.push(obs.clone()); + } + out.observations.push(obs); + return; + } + Run::Nats(subject) => { + self.counts.published += 1; + if let Some(f) = sample { + f.item(&id, || item.value.clone()); + } + out.published.push((subject.clone(), item.value)); + return; + } } + self.pass_on(at, item, registry, out, trace); } - /// Throttle, then ship. A source track's end always ships. - fn emit(&mut self, obs: Observation, out: &mut Output, trace: Option<&mut SampleTrace>) { - if obs.state == Some(ot_core::TrackState::Dropped) { - self.throttle.remove(&obs.source_track_key); - self.details_changed.remove(&obs.source_track_key); - } else if let Some(t) = &self.spec.throttle { - let key = obs.source_track_key.clone(); - let last = self.throttle.get(&key); - if !t.due(last, &obs, self.details_changed.contains(&key)) { - self.counts.throttled += 1; - if let Some(f) = trace { - f.dropped("throttle", t.why_not(last, &obs)); + /// An item as an observation, or why not (counted, and noted on the step). + fn typed( + &mut self, + id: &str, + item: &Item, + out: &mut Output, + sample: Option<&mut SampleTrace>, + ) -> Option { + match serde_json::from_value::(item.value.clone()) { + Ok(o) => Some(o), + Err(e) => { + self.counts.invalid += 1; + if let Some(f) = sample { + f.dropped(id, format!("invalid: {e}")); } - return; + out.last_error = Some(e.to_string()); + None } - self.details_changed.remove(&key); - self.throttle.insert( - key, - LastWrite { - at: obs.observed_at, - lat: obs.position.latitude, - lon: obs.position.longitude, - }, - ); } - self.counts.emitted += 1; - if let Some(f) = trace { - f.obs("throttle", &obs); - f.emitted.push(obs.clone()); + } + + /// Look up the item's entity with these Entity links: grades it, passes + /// fields between entity and track, and queues the entity's updates. + fn link( + &mut self, + stage: &RegistryStage, + item: &mut Item, + registry: &dyn RegistryLookup, + out: &mut Output, + ) { + let matched = stage.run(&mut item.value, registry); + // Values mapped to the entity go no further than here. + if let Some(ext) = item.value.get_mut("ext").and_then(Value::as_object_mut) { + ext.remove(crate::registry::ENTITY_EXT); + } + let grade = matched.as_ref().map_or("none", |m| m.grade.as_str()); + // An affiliation the entity set stands: the entity is the authority + // over the country lists. + item.entity_affiliation = matched + .as_ref() + .is_some_and(|m| m.set.iter().any(|f| f == "classification.affiliation")); + if let Some(m) = matched { + let fresh: Vec<(String, Value)> = m + .updates + .into_iter() + .filter(|(field, v)| { + self.entity_sent.get(&(m.entity_id.clone(), field.clone())) != Some(v) + }) + .collect(); + if !fresh.is_empty() { + if self.entity_sent.len() > 100_000 { + self.entity_sent.clear(); + } + for (field, v) in &fresh { + self.entity_sent + .insert((m.entity_id.clone(), field.clone()), v.clone()); + } + out.entity_updates.push((m.entity_id, fresh)); + } } - out.observations.push(obs); + *self.counts.grades.entry(grade).or_default() += 1; } - fn process_record( + /// A Map step: the record's rules, static details kept and joined, each + /// report finalised into an observation. + fn map( &mut self, - record: &Value, - received_at: DateTime, + at: usize, + id: &str, + item: Item, registry: &dyn RegistryLookup, out: &mut Output, - mut trace: Option<&mut SampleTrace>, + trace: &mut Option<&mut Trace>, ) { - self.counts.records += 1; - let mapped = match self.spec.mapping.apply(record) { + let Run::Map { spec, .. } = &self.steps[at] else { + return; + }; + let schema_version = spec.schema_version; + let outcome = spec.apply(&item.value); + let mut sample = match (trace.as_deref_mut(), item.sample) { + (Some(t), Some(s)) => t.samples.get_mut(s), + _ => None, + }; + let mapped = match outcome { MapOutcome::Rejected(reason) => { - if let Some(f) = trace { - f.dropped("reject", format!("rejected: {reason}")); + if let Some(f) = sample { + f.dropped(id, format!("rejected: {reason}")); } *self.counts.rejected.entry(reason).or_default() += 1; return; } MapOutcome::Unmatched => { - if let Some(f) = trace { - f.item("reject", || record.clone()); + if let Some(f) = sample { f.dropped( - "map", + id, "no mapping rule matched it (or the rule's key was empty)", ); } @@ -700,72 +943,62 @@ impl Pipeline { } MapOutcome::Mapped(m) => m, }; - // Where a mapped report is finalised into an observation. - let join = self.has_join(); - let made = if join { "join" } else { "map" }; - if let Some(f) = trace.as_deref_mut() { - f.item("reject", || record.clone()); - for m in &mapped { - if m.kind == RuleKind::Static { - f.item("map", || mapped_json(m)); - f.note( - "join", - format!( - "kept as the static identity of {}, to fill its later reports", - m.key - ), - ); - } else if join { - // Before the join fills it in. - let before = mapping::finalize( - &self.source_id, - self.spec.mapping.schema_version, - m, - received_at, - ); - match before { - Ok(o) => f.obs("map", &o), - Err(_) => f.item("map", || mapped_json(m)), - } - } - } - } // Static rules first, so a record carrying both (AIS msg 19) joins // its own identity fields. for m in mapped.iter().filter(|m| m.kind == RuleKind::Static) { self.counts.statics += 1; - let entry = self - .statics - .entry(m.key.clone()) - .or_insert_with(|| StaticEntry { - fields: Value::Object(Default::default()), - identifiers: Vec::new(), - updated_at: received_at, - }); + if let Some(f) = sample.as_deref_mut() { + f.item(id, || mapped_json(m)); + f.note( + id, + format!( + "kept the static identity of {}, to fill its later reports", + m.key + ), + ); + } + let stored = self.static_key(at, &m.key); + let Run::Map { + statics, changed, .. + } = &mut self.steps[at] + else { + return; + }; + let entry = statics.entry(m.key.clone()).or_insert_with(|| StaticEntry { + fields: Value::Object(Default::default()), + identifiers: Vec::new(), + updated_at: item.received_at, + }); let before = entry.fields.clone(); let ids_before = entry.identifiers.len(); merge_over(&mut entry.fields, &m.fields); - for id in &m.identifiers { - if !entry.identifiers.contains(id) { - entry.identifiers.push(id.clone()); + for i in &m.identifiers { + if !entry.identifiers.contains(i) { + entry.identifiers.push(i.clone()); } } - entry.updated_at = received_at; + entry.updated_at = item.received_at; if entry.fields != before || entry.identifiers.len() != ids_before { - self.details_changed.insert(m.key.clone()); + changed.insert(m.key.clone()); } - out.statics.push((m.key.clone(), entry.clone())); + out.statics.push((stored, entry.clone())); } - for m in mapped.into_iter().filter(|m| m.kind.reports()) { - let mut m = m; - if let Some(s) = self.statics.get(&m.key) { + for mut m in mapped.into_iter().filter(|m| m.kind.reports()) { + let Run::Map { + statics, changed, .. + } = &mut self.steps[at] + else { + return; + }; + if let Some(s) = statics.get(&m.key) { mapping::fill_missing(&mut m.fields, &s.fields); - for id in &s.identifiers { - if !m.identifiers.contains(id) { - m.identifiers.push(id.clone()); + for i in &s.identifiers { + if !m.identifiers.contains(i) { + m.identifiers.push(i.clone()); } } } + let details_changed = changed.remove(&m.key); if let Some(schema) = &self.schema { let ext = m .fields @@ -777,138 +1010,176 @@ impl Pipeline { && let Err(e) = schema.apply(ext) { self.counts.invalid += 1; - if let Some(f) = trace.as_deref_mut() { - f.dropped(made, format!("invalid: {e}")); + if let Some(f) = sample.as_deref_mut() { + f.dropped(id, format!("invalid: {e}")); } out.last_error = Some(e); continue; } } - // Stages below work on the observation's JSON form. - let mut obs = match mapping::finalize( - &self.source_id, - self.spec.mapping.schema_version, - &m, - received_at, - ) - .and_then(|o| { - serde_json::to_value(&o).map_err(|e| mapping::FinalizeError::Invalid(e.to_string())) - }) { + // Later steps work on the observation's JSON form. + let obs = match mapping::finalize(&self.source_id, schema_version, &m, item.received_at) + .and_then(|o| { + serde_json::to_value(&o) + .map_err(|e| mapping::FinalizeError::Invalid(e.to_string())) + }) { Ok(v) => v, Err(e) => { self.counts.invalid += 1; - if let Some(f) = trace.as_deref_mut() { - f.dropped(made, format!("invalid: {e}")); + if let Some(f) = sample.as_deref_mut() { + f.dropped(id, format!("invalid: {e}")); } out.last_error = Some(e.to_string()); continue; } }; - if let Some(f) = trace.as_deref_mut() { - f.item(made, || obs.clone()); + if let Some(f) = sample.as_deref_mut() { + f.item(id, || obs.clone()); } - let stage = self - .spec - .registry - .as_ref() - .unwrap_or(&self.default_registry); - let matched = stage.run(&mut obs, registry); - // Values mapped to the entity go no further than here. - if let Some(ext) = obs.get_mut("ext").and_then(Value::as_object_mut) { - ext.remove(crate::registry::ENTITY_EXT); - } - if let Some(f) = trace.as_deref_mut() { - f.item("registry", || obs.clone()); - } - let grade = matched.as_ref().map_or("none", |m| m.grade.as_str()); - // An affiliation the entity set stands: the entity is the authority - // over the country lists. - let entity_affiliation = matched - .as_ref() - .is_some_and(|m| m.set.iter().any(|f| f == "classification.affiliation")); - if let Some(m) = matched { - let fresh: Vec<(String, Value)> = m - .updates - .into_iter() - .filter(|(field, v)| { - self.entity_sent.get(&(m.entity_id.clone(), field.clone())) != Some(v) - }) - .collect(); - if !fresh.is_empty() { - if self.entity_sent.len() > 100_000 { - self.entity_sent.clear(); + let next = Item { + value: obs, + kind: m.kind, + changed: details_changed, + entity_affiliation: false, + received_at: item.received_at, + sample: item.sample, + }; + self.pass_on(at, next, registry, out, trace); + sample = match (trace.as_deref_mut(), item.sample) { + (Some(t), Some(s)) => t.samples.get_mut(s), + _ => None, + }; + } + } + + /// The windows a Tracker step with `auto_timing` is using, once it knows + /// the sensor's revisit period (the first such step's). + pub fn tracker_timing(&self) -> Option<&crate::tracker::Timing> { + self.steps.iter().find_map(|s| match s { + Run::Tracker(Stage::Builtin(t)) => t.timing(), + _ => None, + }) + } + + /// Run the Tracker steps on scans still waiting (plots grouped by time + /// wait `scan_hold_secs` for more; `force` runs them regardless). The + /// worker calls this on a timer; a dry run calls it with `force` at the + /// end. + pub fn flush( + &mut self, + now: DateTime, + force: bool, + registry: &dyn RegistryLookup, + ) -> Output { + let mut out = Output::default(); + self.run_trackers(now, force, registry, &mut out, &mut None); + out + } + + /// [`Self::flush`], adding the reports to the traced samples whose plots + /// they came from. With `force` (the end of a dry run), a sample whose + /// plots gave no report is marked so. + pub fn flush_traced( + &mut self, + now: DateTime, + force: bool, + registry: &dyn RegistryLookup, + trace: &mut Trace, + ) -> Output { + let mut out = Output::default(); + let mut t = Some(&mut *trace); + self.run_trackers(now, force, registry, &mut out, &mut t); + if force { + for (i, s) in self.steps.iter().enumerate() { + let Run::Tracker(stage) = s else { continue }; + let why = match stage { + Stage::Plugin { .. } => { + "a tracker plugin's reports are not traced back to the records behind them" } - for (field, v) in &fresh { - self.entity_sent - .insert((m.entity_id.clone(), field.clone()), v.clone()); + Stage::Builtin(_) => { + "no track report: the tracker has not confirmed a track from these plots (or merged them into a nearby plot, or they came too late for their scan)" } - out.entity_updates.push((m.entity_id, fresh)); - } - } - *self.counts.grades.entry(grade).or_default() += 1; - if let Some(stage) = &self.spec.affiliation { - if !entity_affiliation { - stage.run(&mut obs); - } - if let Some(f) = trace.as_deref_mut() { - f.item("affiliation", || obs.clone()); - if entity_affiliation { - f.note("affiliation", "kept the entity's affiliation"); + }; + let id = &self.spec.nodes[i].id; + for f in &mut trace.samples { + let plots = f.plots.iter().any(|(n, _)| n == id); + if let Some(st) = f.stage(id) + && plots + && st.items.is_empty() + && st.dropped.is_empty() + { + st.dropped.push(why.to_owned()); } } } - if let Some(filter) = &self.spec.filter { - if let Some(why) = filter.verdict(&obs) { - self.counts.filtered += 1; - if let Some(f) = trace.as_deref_mut() { - f.dropped("filter", why); + } + out + } + + /// Run each Tracker step, and pass its reports on. When tracing, each + /// report of the built-in tracker joins the sample its detection came + /// from. + fn run_trackers( + &mut self, + now: DateTime, + force: bool, + registry: &dyn RegistryLookup, + out: &mut Output, + trace: &mut Option<&mut Trace>, + ) { + for at in 0..self.steps.len() { + let tracing = trace.is_some(); + // The detection behind each report, only when tracing. + let mut dets: Vec = Vec::new(); + let reports: Vec = match &mut self.steps[at] { + Run::Tracker(Stage::Builtin(t)) => { + let force = force || t.spec().scans == crate::tracker::ScanGrouping::Frame; + let reports = t.run(now, force); + self.counts.late_plots += std::mem::take(&mut t.late); + if tracing { + dets = reports.iter().map(|(_, d)| d.clone()).collect(); } - continue; + reports.into_iter().map(|(obs, _)| obs).collect() } - if let Some(f) = trace.as_deref_mut() { - f.item("filter", || obs.clone()); - } - } - let obs: Observation = match serde_json::from_value(obs) { - Ok(o) => o, - Err(e) => { - self.counts.invalid += 1; - if let Some(f) = trace.as_deref_mut() { - f.dropped(self.last_json_stage(), format!("invalid: {e}")); + Run::Tracker(Stage::Plugin { tracker, version }) => match tracker.run(now, force) { + Ok(mut tracks) => { + for t in &mut tracks { + t.source_id = self.source_id.clone(); + t.provenance.tracker = Some(version.clone()); + } + tracks } - out.last_error = Some(e.to_string()); - continue; - } - }; - if let Some(t) = self.tracker.as_mut() { - if m.kind == RuleKind::Observation { - self.counts.plots += 1; - if self.keep_plots { - out.plots.push(obs.clone()); + Err(e) => { + self.counts.tracker_errors += 1; + out.last_error = Some(format!("tracker plugin: {e}")); + Vec::new() } - if let Some(f) = trace.as_deref_mut() { - f.plots.push(obs.clone()); + }, + _ => continue, + }; + let id = self.spec.nodes[at].id.clone(); + for (i, obs) in reports.into_iter().enumerate() { + let mut sample = None; + if let Some(t) = trace.as_deref_mut() { + sample = dets.get(i).and_then(|d| { + t.samples + .iter() + .position(|f| f.plots.iter().any(|(n, p)| n == &id && p == d)) + }); + if let Some(f) = sample.and_then(|s| t.samples.get_mut(s)) { + f.obs(&id, &obs); } - t.push(obs, received_at); - continue; - } - // A report with its own identity passes the tracker by. - if let Some(f) = trace.as_deref_mut() { - f.obs("tracker", &obs); } + let item = Item { + value: serde_json::to_value(&obs).unwrap_or(Value::Null), + kind: RuleKind::Track, + changed: false, + entity_affiliation: false, + received_at: now, + sample, + }; + self.pass_on(at, item, registry, out, trace); } - self.emit(obs, out, trace.as_deref_mut()); - } - } - - /// The last stage working on an observation's JSON form. - fn last_json_stage(&self) -> &'static str { - if self.spec.filter.is_some() { - "filter" - } else if self.spec.affiliation.is_some() { - "affiliation" - } else { - "registry" } } } @@ -950,7 +1221,11 @@ mod tests { type Reg = BTreeMap<(String, String), RegistryEntry>; fn spec() -> PipelineSpec { - serde_json::from_value(json!({ + serde_json::from_value(spec_json()).unwrap() + } + + fn spec_json() -> Value { + json!({ "codec": { "type": "json" }, "mapping": { "rules": [ { "name": "static", "kind": "static", "when": { "path": "t", "eq": "static" }, @@ -969,8 +1244,7 @@ mod tests { { "path": "ext.registry.entity_id", "exists": true }, { "path": "classification.cot_type", "matches": "^a-.-S-C" } ] } }, "throttle": { "min_interval_secs": 30, "heartbeat_secs": 600, "min_move_m": 50 } - })) - .unwrap() + }) } fn reg() -> Reg { @@ -994,6 +1268,134 @@ mod tests { json!({"t": "pos", "id": id, "lat": lat, "lon": -117.0, "ts": ts}) } + /// A flow from steps and connections, after the input. + fn flow(nodes: Value, edges: Value) -> PipelineSpec { + serde_json::from_value(json!({"codec": {"type": "json"}, "nodes": nodes, "edges": edges})) + .unwrap() + } + + fn pos_map() -> Value { + json!({"mapping": {"rules": [{"name": "pos", "key": "id", + "identifiers": [{"scheme": "mmsi", "value": "id"}], + "fields": {"position.latitude": "lat", "position.longitude": "lon", "name": "name", + "observed_at": {"path": "ts", "transforms": ["time"]}}}]}}) + } + + #[test] + fn a_flow_branches_to_its_output_and_nats_topics() { + // input → map → { filter → filter → output, nats "all" }, map → filter → nats "north" + let spec = flow( + json!([ + {"id": "m", "type": "map", "config": pos_map()}, + {"id": "named", "type": "filter", "config": {"keep_if": {"path": "name", "exists": true}}}, + {"id": "south", "type": "filter", "config": {"keep_if": {"path": "position.latitude", "lt": 33}}}, + {"id": "out", "type": "output"}, + {"id": "all", "type": "nats", "config": {"subject": "feed.all"}}, + {"id": "north", "type": "filter", "config": {"keep_if": {"path": "position.latitude", "gte": 33}}}, + {"id": "pub", "type": "nats", "config": {"subject": "feed.north"}} + ]), + json!([ + {"source": "input", "target": "m"}, + {"source": "m", "target": "named"}, {"source": "named", "target": "south"}, + {"source": "south", "target": "out"}, + {"source": "m", "target": "all"}, + {"source": "m", "target": "north"}, {"source": "north", "target": "pub"} + ]), + ); + let mut p = Pipeline::new("ais", spec).unwrap(); + let reg = reg(); + let rec = |id: u32, lat: f64, name: Option<&str>| { + let mut r = json!({"id": id, "lat": lat, "lon": -117.0, "ts": "2026-09-24T12:00:00Z"}); + if let Some(n) = name { + r["name"] = json!(n); + } + r + }; + let a = feed(&mut p, ®, rec(1, 32.0, Some("ALPHA"))); + assert_eq!(a.observations.len(), 1, "named and south: to correlation"); + assert_eq!( + a.published + .iter() + .map(|(s, _)| s.as_str()) + .collect::>(), + ["feed.all"] + ); + // The Output looks the entity up, with no Entity links step before it. + assert_eq!(a.observations[0].ext["registry"]["entity_id"], "e1"); + assert!( + a.published[0].1["ext"].get("registry").is_none(), + "a topic gets the mapped data" + ); + let b = feed(&mut p, ®, rec(2, 34.0, None)); + assert!(b.observations.is_empty(), "unnamed, and north"); + assert_eq!( + b.published + .iter() + .map(|(s, _)| s.as_str()) + .collect::>(), + ["feed.all", "feed.north"] + ); + assert_eq!(b.published[1].1["position"]["latitude"], 34.0); + assert_eq!( + (p.counts.emitted, p.counts.published, p.counts.filtered), + (1, 3, 2) + ); + } + + #[test] + fn a_trace_follows_each_branch() { + let spec = flow( + json!([ + {"id": "m", "type": "map", "config": pos_map()}, + {"id": "f", "type": "filter", "config": {"drop_if": {"path": "position.latitude", "gt": 33}}}, + {"id": "out", "type": "output"}, + {"id": "raw", "type": "nats", "config": {"subject": "raw"}} + ]), + json!([ + {"source": "input", "target": "m"}, {"source": "m", "target": "f"}, + {"source": "f", "target": "out"}, {"source": "m", "target": "raw"} + ]), + ); + let mut p = Pipeline::new("ais", spec).unwrap(); + let mut trace = Trace::new(5); + let body = json!([{"id": 1, "lat": 32.0, "lon": 1.0, "ts": "2026-09-24T12:00:00Z"}, + {"id": 2, "lat": 40.0, "lon": 1.0, "ts": "2026-09-24T12:00:00Z"}]); + p.process_traced( + &Frame::new(body.to_string().into_bytes()), + &Reg::new(), + &mut trace, + ); + let ids: Vec<&str> = trace.samples[0] + .stages + .iter() + .map(|s| s.id.as_str()) + .collect(); + assert_eq!(ids, ["input", "m", "f", "raw", "out"]); + let (a, b) = (&trace.samples[0], &trace.samples[1]); + assert_eq!( + (stage(a, "out").items.len(), stage(a, "raw").items.len()), + (1, 1) + ); + assert_eq!(stage(b, "f").dropped, ["drop_if matched"]); + assert!(stage(b, "out").items.is_empty()); + assert_eq!( + stage(b, "raw").items.len(), + 1, + "the other branch still has it" + ); + } + + #[test] + fn a_loop_or_a_step_fed_the_wrong_data_cannot_run() { + let spec = flow( + json!([{"id": "f", "type": "filter", "config": {}}, {"id": "out", "type": "output"}]), + json!([{"source": "input", "target": "f"}, {"source": "f", "target": "out"}]), + ); + assert!( + matches!(Pipeline::new("x", spec), Err(PipelineError::Flow(m)) if m.contains("put a Map before it")) + ); + } + #[test] fn an_affiliation_the_entity_set_outranks_the_country_lists() { let spec: PipelineSpec = serde_json::from_value(json!({ @@ -1231,7 +1633,11 @@ mod tests { assert_eq!(p.counts.plots, 8); assert_eq!(p.counts.emitted, 6); // Nothing waits: frames are scans. - assert!(p.flush(Utc::now(), true).observations.is_empty()); + assert!( + p.flush(Utc::now(), true, &Reg::new()) + .observations + .is_empty() + ); } fn stage<'a>(f: &'a crate::trace::SampleTrace, id: &str) -> &'a crate::trace::StageTrace { @@ -1257,14 +1663,10 @@ mod tests { #[test] fn a_trace_follows_each_record_through_its_stages() { let reg = reg(); - let mut spec = spec(); - spec.throttle = None; - spec.filter = Some(FilterSpec { - keep_if: None, - drop_if: Some( - serde_json::from_value(json!({"path": "position.latitude", "gt": 33})).unwrap(), - ), - }); + let mut v = spec_json(); + v.as_object_mut().unwrap().remove("throttle"); + v["filter"] = json!({"drop_if": {"path": "position.latitude", "gt": 33}}); + let spec: PipelineSpec = serde_json::from_value(v).unwrap(); let mut p = Pipeline::new("ais", spec).unwrap(); let mut trace = Trace::new(5); let frame = |rec: Value| Frame::new(rec.to_string().into_bytes()); @@ -1280,27 +1682,30 @@ mod tests { let f = &trace.samples; assert_eq!((f.len(), trace.frames.len()), (4, 4)); assert_eq!(f.iter().map(|s| s.frame).collect::>(), [0, 1, 2, 3]); - let ids: Vec<&str> = f[0].stages.iter().map(|s| s.id).collect(); + let ids: Vec<&str> = f[0].stages.iter().map(|s| s.id.as_str()).collect(); assert_eq!( ids, - ["decode", "map", "join", "registry", "affiliation", "filter"], - "only the stages the pipeline has" + [ + "input", + "map", + "registry", + "affiliation", + "filter", + "output" + ], + "the flow's steps, in the order they are met" ); assert_eq!(trace.frames[0].format, "json"); - // A static report is kept by the join. - assert_eq!(stage(&f[0], "map").items[0]["kind"], "static"); - let join = stage(&f[0], "join"); + // A static report is kept by the map, to join later reports. + let map = stage(&f[0], "map"); + assert_eq!(map.items[0]["kind"], "static"); assert!( - join.dropped.is_empty(), + map.dropped.is_empty(), "keeping a static identity is not a drop" ); - assert!(join.notes[0].starts_with("kept as the static identity of 1")); + assert!(map.notes[0].starts_with("kept the static identity of 1")); // A position report: joined, graded, published. - assert!( - stage(&f[1], "map").items[0].get("name").is_none(), - "before the join" - ); - assert_eq!(stage(&f[1], "join").items[0]["name"], "EXAMPLE"); + assert_eq!(stage(&f[1], "map").items[0]["name"], "EXAMPLE"); assert_eq!( stage(&f[1], "registry").items[0]["ext"]["registry"]["grade"], "exact" @@ -1314,7 +1719,7 @@ mod tests { assert!(f[2].emitted.is_empty()); // A frame that does not decode is a sample of its own. assert_eq!(trace.frames[3].format, "text"); - assert!(stage(&f[3], "decode").dropped[0].starts_with("decode error")); + assert!(stage(&f[3], "input").dropped[0].starts_with("decode error")); } #[test] @@ -1334,7 +1739,7 @@ mod tests { assert!(s.iter().all(|s| s.frame == 0)); assert_eq!(trace.frames[0].content["plots"][2]["n"], 3); for (i, sample) in s.iter().enumerate() { - let decoded = &stage(sample, "decode").items; + let decoded = &stage(sample, "input").items; assert_eq!(decoded.len(), 1); assert_eq!(decoded[0]["n"], i + 1, "decode order"); } @@ -1345,7 +1750,7 @@ mod tests { assert_eq!(stage(&s[2], "map").items[0]["source_track_key"], "3"); // First scan: nothing confirmed yet, and nothing waits (frames are scans). assert!(!stage(&s[0], "tracker").held); - p.flush_traced(Utc::now(), true, &mut trace); + p.flush_traced(Utc::now(), true, &Reg::new(), &mut trace); let s = &trace.samples; assert!(stage(&s[0], "tracker").dropped[0].starts_with("no track report")); assert!( @@ -1379,7 +1784,7 @@ mod tests { // Each frame's scan waits for more plots of its time. assert_eq!(trace.samples.len(), 2); assert!(trace.samples.iter().all(|f| stage(f, "tracker").held)); - let out = p.flush_traced(Utc::now(), true, &mut trace); + let out = p.flush_traced(Utc::now(), true, &Reg::new(), &mut trace); assert_eq!(out.observations.len(), 1, "confirmed on the second scan"); let (a, b) = (&trace.samples[0], &trace.samples[1]); assert!(stage(a, "tracker").items.is_empty()); diff --git a/crates/ot-source/src/source.rs b/crates/ot-source/src/source.rs index 751770c..73ac94a 100644 --- a/crates/ot-source/src/source.rs +++ b/crates/ot-source/src/source.rs @@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize}; +use crate::flow::Step; use crate::pipeline::PipelineSpec; use crate::transport::TransportConfig; @@ -185,6 +186,9 @@ pub enum SourceError { Tracker(String), #[error("mapping targets schema version {wanted}, but it is {found}")] SchemaVersion { wanted: u32, found: String }, + /// The flow's shape, or a step's own setting. + #[error("{0}")] + Flow(String), #[error("rule {rule:?} maps ext.{key}, which schema version {version} does not define")] UndeclaredExtension { rule: String, @@ -228,56 +232,64 @@ impl SourceSpec { /// schema checks): see [`Self::problems`] for all of them. pub fn validate(&self) -> Result<(), SourceError> { match self.checks(None).into_iter().next() { - Some((_, _, e)) => Err(e), + Some(c) => Err(c.error), None => Ok(()), } } - /// Everything that stops this source running, each with the pipeline - /// stage (and, where there is one, the setting) it belongs to. Empty: - /// the source is complete and may be enabled. A problem of kind + /// Everything that stops this source running, each on the step of its + /// flow it belongs to (and, where there is one, the setting). Empty: the + /// source is complete and may be enabled. A problem of kind /// [`ProblemKind::Invalid`] also stops it being saved; the rest leave /// it a draft. pub fn problems(&self, schema: Option<&crate::schema::ExtensionSchema>) -> Vec { self.checks(Some(schema)) .into_iter() - .map(|(stage, field, e)| Problem { - kind: match e { - SourceError::BadId(_) | SourceError::NoName => ProblemKind::Invalid, - _ => ProblemKind::Incomplete, + .map(|c| Problem { + kind: if c.invalid { + ProblemKind::Invalid + } else { + ProblemKind::Incomplete }, - stage, - field: field.map(str::to_owned), - message: e.to_string(), + node: c.node, + field: c.field.map(str::to_owned), + message: c.error.to_string(), }) .collect() } - /// Every check, in order, each failure with its stage and setting. + /// Every check, in order, each failure with its step and setting. /// `schema`: `None` skips the extension schema checks; `Some(None)` - /// means the mapping's schema version is not published. - fn checks( - &self, - schema: Option>, - ) -> Vec<(Option, Option<&'static str>, SourceError)> { + /// means the mappings' schema version is not published. + fn checks(&self, schema: Option>) -> Vec { let mut out = Vec::new(); + let input = || Some(crate::flow::INPUT.to_owned()); + let push = |out: &mut Vec, + node: Option, + field: Option<&'static str>, + error: SourceError| { + let invalid = matches!(error, SourceError::BadId(_) | SourceError::NoName); + out.push(Check { + node, + field, + error, + invalid, + }); + }; if let Some(label) = &self.security && let Err(e) = label.validate() { - out.push(( - Some(Stage::Output), - Some("security"), - SourceError::Tracker(e), - )); + push(&mut out, None, Some("security"), SourceError::Tracker(e)); } if let Some(m) = &self.emitter_motion && let Err(e) = m.check() { - out.push(( - Some(Stage::Output), + push( + &mut out, + None, Some("emitter_motion"), SourceError::Tracker(e), - )); + ); } let id_ok = !self.id.is_empty() && self.id.len() <= 64 @@ -286,18 +298,45 @@ impl SourceSpec { .bytes() .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-' || b == b'_'); if !id_ok { - out.push((None, Some("id"), SourceError::BadId(self.id.clone()))); + push( + &mut out, + None, + Some("id"), + SourceError::BadId(self.id.clone()), + ); } if self.name.trim().is_empty() { - out.push((None, Some("name"), SourceError::NoName)); + push(&mut out, None, Some("name"), SourceError::NoName); } - for e in self.pipeline.mapping.problems() { - out.push((Some(Stage::Map), None, e.into())); + // The flow's shape, then each step's own settings. + for p in self.pipeline.shape_problems() { + out.push(Check { + node: p.node, + field: None, + error: SourceError::Flow(p.message), + invalid: p.invalid, + }); } - if let Some(t) = &self.pipeline.tracker - && let Err(e) = t.validate() - { - out.push((Some(Stage::Tracker), None, SourceError::Tracker(e))); + for n in &self.pipeline.nodes { + let node = || Some(n.id.clone()); + match &n.step { + Step::Map(m) => { + for e in m.mapping.problems() { + push(&mut out, node(), None, e.into()); + } + } + Step::Tracker(t) => { + if let Err(e) = t.validate() { + push(&mut out, node(), None, SourceError::Tracker(e)); + } + } + Step::Nats(nats) => { + if let Err(e) = check_subject(&nats.subject) { + push(&mut out, node(), Some("subject"), SourceError::Flow(e)); + } + } + _ => {} + } } let grpc = matches!( self.transport, @@ -315,32 +354,35 @@ impl SourceSpec { if (plugin || (protobuf && !grpc)) && let Err(e) = crate::codec::Codec::new(self.pipeline.codec.clone()) { - out.push(( - Some(Stage::Decode), - None, + push( + &mut out, + input(), + Some("codec"), SourceError::Tracker(e.to_string()), - )); + ); } let missing = self.transport.missing(); for (field, what) in &missing { - out.push(( - Some(Stage::Source), + push( + &mut out, + input(), Some(*field), SourceError::Transport(format!("{what} is not set")), - )); + ); } if let Err(e) = self.transport.check() { - out.push((Some(Stage::Source), None, SourceError::Transport(e))); + push(&mut out, input(), None, SourceError::Transport(e)); } // gRPC carries protobuf: the method must fit the codec's message. if grpc { match crate::transport::ProtoContext::of(&self.pipeline.codec) { - Err(e) => out.push((Some(Stage::Decode), None, SourceError::Transport(e))), - Ok(None) => out.push(( - Some(Stage::Decode), - None, + Err(e) => push(&mut out, input(), Some("codec"), SourceError::Transport(e)), + Ok(None) => push( + &mut out, + input(), + Some("codec"), SourceError::Transport("a gRPC source decodes with the protobuf codec".into()), - )), + ), Ok(Some(_)) if !missing.is_empty() => {} Ok(Some(proto)) => { let fit = match &self.transport { @@ -353,7 +395,7 @@ impl SourceSpec { _ => Ok(()), }; if let Err(e) = fit { - out.push((Some(Stage::Source), None, SourceError::Transport(e))); + push(&mut out, input(), None, SourceError::Transport(e)); } } } @@ -362,39 +404,52 @@ impl SourceSpec { | TransportConfig::TcpServer { framing, .. } = &self.transport && let Err(e) = crate::frame::Framer::new(framing.clone()) { - out.push((Some(Stage::Source), Some("framing"), e.into())); + push(&mut out, input(), Some("framing"), e.into()); } // Last, so that a broken setting is reported before its security. if let Some(e) = self.authentication_problem() { - out.push(( - Some(Stage::Source), + push( + &mut out, + input(), Some("unauthenticated"), SourceError::Unauthenticated(e), - )); + ); } if let Some(schema) = schema { - let wanted = self.pipeline.mapping.schema_version; + let wanted = self.pipeline.schema_version(); + let maps: Vec<&crate::flow::FlowNode> = self + .pipeline + .nodes + .iter() + .filter(|n| matches!(n.step, Step::Map(_))) + .collect(); match schema.filter(|s| s.version == wanted) { - None => out.push(( - Some(Stage::Map), + None if !maps.is_empty() => push( + &mut out, + Some(maps[0].id.clone()), Some("schema_version"), SourceError::SchemaVersion { wanted, found: "not published".into(), }, - )), + ), + None => {} Some(schema) => { - for (key, rules) in crate::schema::mapped_ext_keys(&self.pipeline.mapping) { - if schema.field(&key).is_none() { - out.push(( - Some(Stage::Map), - None, - SourceError::UndeclaredExtension { - rule: rules[0].clone(), - key, - version: wanted, - }, - )); + for n in maps { + let Step::Map(m) = &n.step else { continue }; + for (key, rules) in crate::schema::mapped_ext_keys(&m.mapping) { + if schema.field(&key).is_none() { + push( + &mut out, + Some(n.id.clone()), + None, + SourceError::UndeclaredExtension { + rule: rules[0].clone(), + key, + version: wanted, + }, + ); + } } } } @@ -404,13 +459,41 @@ impl SourceSpec { } } +/// One failed check: the step and setting it concerns, and whether it also +/// stops a save. +struct Check { + node: Option, + field: Option<&'static str>, + error: SourceError, + invalid: bool, +} + +/// A NATS subject a NATS topic step may publish on: concrete (no wildcards), +/// dot-separated tokens without spaces. (The server also keeps it outside +/// the system tracks' subjects.) +pub fn check_subject(subject: &str) -> Result<(), String> { + if subject.trim().is_empty() { + return Err("the subject is not set".into()); + } + let ok = subject + .split('.') + .all(|t| !t.is_empty() && t != "*" && t != ">" && !t.chars().any(char::is_whitespace)); + if ok { + Ok(()) + } else { + Err(format!( + "{subject:?} is not a NATS subject to publish on (dot-separated tokens, no wildcards or spaces)" + )) + } +} + /// One reason a source may not run (see [`SourceSpec::problems`]). #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Problem { pub kind: ProblemKind, - /// The pipeline stage it belongs to; `None` for the source as a whole - /// (its id and name). - pub stage: Option, + /// The step of the flow it belongs to (`input` for the transport and + /// codec); `None` for the source as a whole. + pub node: Option, /// The setting, where there is one (for example `host`). #[serde(default, skip_serializing_if = "Option::is_none")] pub field: Option, @@ -426,26 +509,6 @@ pub enum ProblemKind { Incomplete, } -/// A source's pipeline stages, in the order a frame meets them. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum Stage { - /// The transport and its framing. - Source, - /// The codec. - Decode, - Map, - StaticJoin, - Registry, - Affiliation, - Filter, - Tracker, - Throttle, - /// What the source reports and how: tracks or detections, security - /// label, priority. - Output, -} - #[cfg(test)] mod tests { use super::*; @@ -505,13 +568,13 @@ mod tests { } #[test] - fn problems_name_their_stage_and_a_draft_is_only_incomplete() { + fn problems_name_their_step_and_a_draft_is_only_incomplete() { let ok: SourceSpec = serde_json::from_value(spec("cot")).unwrap(); assert_eq!( ok.problems(None), vec![Problem { kind: ProblemKind::Incomplete, - stage: Some(Stage::Map), + node: Some("map".into()), field: Some("schema_version".into()), message: "mapping targets schema version 1, but it is not published".into(), }] @@ -527,16 +590,20 @@ mod tests { let found: Vec<_> = draft .problems(None) .into_iter() - .map(|p| (p.kind, p.stage, p.field)) + .map(|p| (p.kind, p.node, p.field)) .collect(); let incomplete = ProblemKind::Incomplete; assert_eq!( found, vec![ - (incomplete, Some(Stage::Map), None), - (incomplete, Some(Stage::Source), Some("host".into())), - (incomplete, Some(Stage::Source), Some("port".into())), - (incomplete, Some(Stage::Map), Some("schema_version".into())), + (incomplete, Some("map".into()), None), + (incomplete, Some("input".into()), Some("host".into())), + (incomplete, Some("input".into()), Some("port".into())), + ( + incomplete, + Some("map".into()), + Some("schema_version".into()) + ), ] ); assert!(matches!(draft.validate(), Err(SourceError::Mapping(_)))); @@ -552,7 +619,7 @@ mod tests { let found: Vec<_> = s .problems(None) .into_iter() - .map(|p| (p.kind, p.stage, p.message)) + .map(|p| (p.kind, p.node, p.message)) .collect(); assert_eq!( found[0], @@ -563,7 +630,10 @@ mod tests { ) ); assert_eq!( - found.iter().filter(|p| p.1 == Some(Stage::Map)).count(), + found + .iter() + .filter(|p| p.1.as_deref() == Some("map")) + .count(), 5, "unknown field, no position (twice), duplicate rule, schema: {found:?}" ); diff --git a/crates/ot-source/src/trace.rs b/crates/ot-source/src/trace.rs index 3ca7976..bb1fcad 100644 --- a/crates/ot-source/src/trace.rs +++ b/crates/ot-source/src/trace.rs @@ -1,5 +1,5 @@ -//! A stage-by-stage record of what a pipeline did with its first few -//! records, for the pipeline designer's live preview. +//! A step-by-step record of what a source's flow did with its first few +//! records, for the editor's samples. //! //! Tracing is opt-in: [`crate::pipeline::Pipeline::process_traced`] fills a //! [`Trace`]; [`crate::pipeline::Pipeline::process`] takes no trace and @@ -40,14 +40,14 @@ pub struct Trace { pub struct SampleTrace { /// Index in [`Trace::frames`] of the frame it came from. pub frame: usize, - /// The pipeline's stages after the transport, in order, only those it has. + /// The flow's steps, input first, in the order they are met. pub stages: Vec, /// What the pipeline emitted from this record, to show as published. #[serde(skip)] pub emitted: Vec, - /// Detections this record gave the tracker stage. + /// Detections this record gave each Tracker step, by step id. #[serde(skip)] - pub(crate) plots: Vec, + pub(crate) plots: Vec<(String, Observation)>, } /// A raw frame, readable: JSON when it parses, else text, else escaped bytes. @@ -69,8 +69,8 @@ pub struct FrameView { /// A record after one stage. #[derive(Debug, Serialize)] pub struct StageTrace { - /// The stage id, as the UI names it (`decode`, `map`, `filter`…). - pub id: &'static str, + /// The step's id (`input` holds the decoded record). + pub id: String, /// What this stage passed on: the record up to the reject stage, the /// mapping's outputs, then observations. pub items: Vec, @@ -103,7 +103,7 @@ impl Trace { /// Start tracing a record of `frame` (or the frame itself, when it does /// not decode), if the trace still has room. - pub(crate) fn begin(&mut self, frame: &Frame, stages: &[&'static str]) -> Option { + pub(crate) fn begin(&mut self, frame: &Frame, stages: &[String]) -> Option { if self.samples.len() >= self.limit { return None; } @@ -121,7 +121,7 @@ impl Trace { stages: stages .iter() .map(|id| StageTrace { - id, + id: id.clone(), items: Vec::new(), dropped: Vec::new(), notes: Vec::new(), diff --git a/crates/ot-source/tests/sapient_acoustic_example.rs b/crates/ot-source/tests/sapient_acoustic_example.rs index 6788afa..e15909a 100644 --- a/crates/ot-source/tests/sapient_acoustic_example.rs +++ b/crates/ot-source/tests/sapient_acoustic_example.rs @@ -28,7 +28,8 @@ fn acoustic_example_maps_a_sapient_bearing() { } } }); - let MapOutcome::Mapped(mapped) = source.pipeline.mapping.apply(&record) else { + let MapOutcome::Mapped(mapped) = source.pipeline.mappings().next().unwrap().apply(&record) + else { panic!("acoustic record did not match the source mapping"); }; let observation = finalize(&source.id, 1, &mapped[0], Utc::now()).expect("valid bearing"); diff --git a/crates/ot-source/tests/stanag4607.rs b/crates/ot-source/tests/stanag4607.rs index 52fd535..fdb4c9d 100644 --- a/crates/ot-source/tests/stanag4607.rs +++ b/crates/ot-source/tests/stanag4607.rs @@ -92,7 +92,7 @@ fn replay(path: &std::path::Path) -> (Vec, Pipeline, usize let last = out.last().map(|o| o.observed_at).unwrap(); out.extend( pipeline - .flush(last + chrono::TimeDelta::hours(1), true) + .flush(last + chrono::TimeDelta::hours(1), true, ®istry) .observations, ); (out, pipeline, packets, dwells) @@ -223,7 +223,7 @@ fn a_slow_revisit_recording_tunes_the_tracker_itself() { fn platform_types_have_valid_symbols() { let spec: SourceSpec = serde_json::from_str(include_str!("../../../docs/examples/stanag4607.json")).unwrap(); - let json = serde_json::to_value(&spec.pipeline.mapping).unwrap(); + let json = serde_json::to_value(spec.pipeline.mappings().next().unwrap()).unwrap(); let rule = json["rules"] .as_array() .unwrap() @@ -276,7 +276,9 @@ fn gmti_plots_carry_ground_error_ellipses() { continue; } located += usize::from(r.get("lat").is_some()); - let ot_source::mapping::MapOutcome::Mapped(mapped) = spec.pipeline.mapping.apply(r) else { + let ot_source::mapping::MapOutcome::Mapped(mapped) = + spec.pipeline.mappings().next().unwrap().apply(r) + else { continue; }; for m in mapped.iter().filter(|m| m.rule == "target") { diff --git a/docs/guides/admin.md b/docs/guides/admin.md index b5b6a41..f718693 100644 --- a/docs/guides/admin.md +++ b/docs/guides/admin.md @@ -347,7 +347,7 @@ pipelines"). | Variable | Default | | |---|---|---| | `OT_ENGINE_CONSUMER` | `engine-1` | The engine's consumer name on the observation streams. | -| `OT_CONFIRM_AFTER` | `3` | Reports a new system track needs before it is confirmed (at least 1). A source's own **Confirm after** (in its publish stage) overrides it for that source's tracks. | +| `OT_CONFIRM_AFTER` | `3` | Reports a new system track needs before it is confirmed (at least 1). A source's own **Confirm after** (in its Feed settings) overrides it for that source's tracks. | | `OT_DROP_AFTER_HOURS` | `6` | Hours without a report before a system track is dropped and deleted downstream. | | `OT_CORRELATION` | `kinematics-metadata` | How tracks with no shared identifier pair, until someone saves correlation settings: `identifiers` (never), `kinematics`, or `kinematics-metadata` (kinematics, vetoed by conflicting identifiers or domains). Once correlation settings are saved in the Correlation tab, the saved ones win and this is ignored; the engine logs a warning at start when it is set and the saved approach differs. | | `OT_WRITER_CONSUMER` | `writer-1` | The writer's consumer name; unique per writer. | @@ -475,7 +475,7 @@ FIPS module ([FIPS cryptography](#fips-cryptography)). | OCSP responder | TCP, HTTP (or HTTPS) | Outbound | The client certificate's authority information access URL, or `OT_TLS_CLIENT_OCSP_URL` | The revocation status of a client certificate presented to the control plane (with `OT_TLS_CLIENT_CA`) | OCSP requests and responses (RFC 6960); answers are signed by the CA or its delegated responder, so plain HTTP is usual. 5 s limit, no redirects | | Admin listener | TCP; HTTP/1.1 and HTTP/2, HTTPS with a certificate | Inbound | `OT_ADMIN_BIND`, none | The admin API routes, and the web UI and the rest of the API ([Admin listener](#admin-listener)); only here while it is set | As the control plane, or `OT_ADMIN_TLS_CERT`/`OT_ADMIN_TLS_KEY` and `OT_ADMIN_TLS_CLIENT_CA`; the same sign-in | | Redis | TCP, RESP | Outbound | `redis://127.0.0.1:6379` | Observation streams, the live picture, the outbox, commands between roles | `rediss://`, `OT_REDIS_CA`, mutual TLS with `OT_REDIS_CERT`/`OT_REDIS_KEY`; a password in the URL | -| NATS | TCP, NATS | Outbound | `nats://127.0.0.1:4222` | Tracks (`tracks.>`, JetStream), contacts (`contacts.>`) and raw output (`opentrack.raw.`); the multi-node sync boundary (`.out.*` and `.in.*`, prefix `ot.sync`) | `tls://`, `OT_NATS_CA`, mutual TLS with `OT_NATS_CERT`/`OT_NATS_KEY`; `.creds`, a token, or a user and password | +| NATS | TCP, NATS | Outbound | `nats://127.0.0.1:4222` | Tracks (`tracks.>`, JetStream), contacts (`contacts.>`) and sources' NATS topics (their subjects); the multi-node sync boundary (`.out.*` and `.in.*`, prefix `ot.sync`) | `tls://`, `OT_NATS_CA`, mutual TLS with `OT_NATS_CERT`/`OT_NATS_KEY`; `.creds`, a token, or a user and password | | OpenTelemetry | TCP; OTLP over gRPC (4317) or HTTP (4318) | Outbound | Off | Logs (the audit record included), traces and metrics to the collector ([OpenTelemetry](#opentelemetry)) | `https://` endpoint, `OTEL_EXPORTER_OTLP_CERTIFICATE`, mutual TLS, headers | | TAK Server output | TCP, CoT XML | Outbound | None configured | Tracks to a TAK Server's streaming input ([TAK output](#tak-output)) | TLS (8089) with a client certificate; 8087 is plaintext | | TAK multicast output | UDP, CoT XML, IPv4 or IPv6 | Outbound | Group `239.2.3.1`, port `6969`, TTL 1 (IPv6: hop limit) | Tracks to ATAK and WinTAK on the network | None: always plaintext | @@ -941,15 +941,38 @@ result on the Sources and Correlation tabs. ### Sources -**Sources → Add source** runs a wizard: transport, framing, codec, then the pipeline designer, -with a live preview of each stage on sample data. Enable a source to start it. Each save is a -revision: the source's **History** tab lists them, with who saved each and the spec it saved. Write secrets as -`${env:NAME}`, resolved when the source starts. Only admins see a source's secrets: for viewers and -track managers, passwords, tokens, header and metadata values, credentials in URLs and API keys in -messages show as `••••••` (`${env:…}` references stay visible). See the README, "Sources and pipelines", and `docs/examples/`. - -**Probing a feed.** The wizard's probe connects with the transport as set and captures what -arrives. Choosing Cursor-on-Target XML on a TCP transport frames by the end tag `` (CoT +**Sources → Add source** asks for a name, an id (from the name; it never changes) and a transport, +and opens the new source in its editor as a **draft**, with its input, an empty Map and the Output +connected. A source's editor is a workspace of its steps: +- **The step list** (left): **Records** (Reject, Map), **Observations** (Filter, Entity links, + Affiliation, Tracker, Throttle) and **Outputs** (Output, NATS topic). Drag a step onto the + workspace, or select it and press Enter to add it in view. Any step but the Output may be used + any number of times. +- **The workspace:** steps go where they are dragged, and are connected by dragging from a step's + right edge to another's left. A step takes one input and can feed any number of steps, so a + flow can branch. Records come from the input through Reject steps to a **Map**, which turns them + into observations; the steps after a Map take observations; the **Output** (system tracks, to + correlation, one per source) and **NATS topics** (the mapped data, as JSON, on a subject of your + own, without correlation) end a branch. A connection that would feed a step the wrong data, give + it a second input or loop back is refused as you draw it. Select a step or a connection and press + Delete to remove it; the input stays. +- **A step's settings** open beside the workspace when it is clicked, with its samples. The + input's are the transport and the codec. Problems show on the step they belong to, as you edit. +- **Feed** (top row): the source's own settings (name, description, whether it reports tracks or + detections, publishing alone, confirming, priority, security label, emitter motion), its status + and its history. + +A draft can be saved but not enabled; enable a complete source to start it. **JSON** (top row) +shows the whole source as JSON, editable. Each save is a revision: **Feed → History** lists them, +with who saved each and the spec it saved. Write secrets as `${env:NAME}`, resolved when the source +starts. Only admins see a source's secrets: for viewers and track managers, passwords, tokens, +header and metadata values, credentials in URLs and API keys in messages show as `••••••` +(`${env:…}` references stay visible). See [reference.md](../reference.md), "Sources and +pipelines", and `docs/examples/`. + +**Capturing samples.** **Capture samples** (on the input's settings) connects with the transport +as set and captures what arrives; the codec it detects and a mapping it suggests are offered on +the input and the Map, applied only when chosen. Choosing Cursor-on-Target XML on a TCP transport frames by the end tag `` (CoT events arrive back to back, often over several lines). A probe that connects but captures nothing says why it may be: **no bytes arrived** (the feed is idle; the port needs TLS, since a TLS server waits silently for a handshake; or the port only takes data in, as a TAK Server input does), or @@ -959,13 +982,13 @@ streaming port (8089) with TLS and a client certificate it issued; its input por **A listening source must authenticate its senders.** `tcp_server` needs TLS with a client CA (mutual TLS, a certificate required: not `client_cert_optional`); `grpc_server` needs that or a bearer `token`. Otherwise the source is saved and run only with the risk accepted on it, -`"unauthenticated": "accepted"` (the red **Accept unauthenticated senders** toggle on its Transport -tab; always needed for `udp`, which cannot carry TLS). The flag is refused where it is not needed +`"unauthenticated": "accepted"` (the red **Accept unauthenticated senders** toggle in its input's +settings; always needed for `udp`, which cannot carry TLS). The flag is refused where it is not needed (a client transport, or a listener that authenticates), so it only ever marks a real exception; each one needs the authorising official's acceptance ([hardening checklist](../security/hardening.md#encrypt-every-link)). The source list shows an UNAUTHENTICATED badge on such sources, and the decision log records the flag set or cleared. **Upgrading to 0.4.5:** an enabled listener saved before without sender -authentication or the flag is not started; its state is *not started* and its Status tab says why. +authentication or the flag is not started; its state is *not started* and its status says why. Add a client CA or token, or set the flag, and save. A configuration import is checked the same way. **UDP multicast.** A `udp` source joins `multicast_group` if one is set, IPv4 (`239.2.3.1`, with @@ -975,29 +998,29 @@ this node for an IPv4 group, an interface name (`eth0`) or index (`2`) for an IP the system's choice. A name is looked up when the source starts, so a missing interface shows as the source's error. -The pipeline designer's **Live preview** (right-hand pane) is a dry run of the pipeline as edited, -not yet saved, over the source's stored samples; **Capture samples** replaces them with up to 20 -frames (or 15 s) of the live feed. Nothing is published. The pane shows the first 5 samples, each -a decoded record in decode order (the first frame's, then the next frame's if it has fewer), under -its own header (#1, #2…) as pretty-printed JSON, **as they are after the stage selected on the -left**: -- **Transport:** each frame the samples came from, once, as received ("frame 1 (samples - #1–#5)"): JSON pretty-printed, other text as it is, binary escaped (`\xNN`) with its length; a - frame over 64 KiB shows only its start. -- **Decode:** the record. A frame that does not decode is a sample of its own, with the error. -- **Reject:** the record, if kept. **Map:** what the mapping made of it: the observation (before - the identity join fills it in, when there is one), or a static rule's identity fields. -- **Identity join, Entity links, Affiliation, Filter, Tracker, Throttle:** the observation after - each. A tracker's report shows under the sample whose plot made it. -- **Publish:** the message as it would be published for a new system track seeded from the +A step's **Samples** are a dry run of the source as edited, not yet saved, over its stored samples; +**Capture samples** replaces them with up to 20 frames (or 15 s) of the live feed. Nothing is +published. A draft runs as far as it can. The first 5 samples show, each a decoded record in decode +order (the first frame's, then the next frame's if it has fewer), under its own header (#1, #2…) as +pretty-printed JSON, **as they leave the selected step**: +- **Input:** each frame the samples came from, once, as received ("frame 1 (samples #1–#5)"): + JSON pretty-printed, other text as it is, binary escaped (`\xNN`) with its length; a frame over + 64 KiB shows only its start. A frame that does not decode is a sample of its own, with the error. +- **Reject:** the record, if kept. **Map:** the observation, its identity fields from static rules + joined in, or a static rule's identity fields. +- **Entity links, Affiliation, Filter, Tracker, Throttle:** the observation after each. A tracker's + report shows under the sample whose plot made it. +- **Output:** the message as it would be published for a new system track seeded from the observation (the GOLD fields and the output schema's attributes; its UID is correlation's to - assign, and correlation decides whether and when it is published). + assign, and correlation decides whether and when it is published). **NATS topic:** the JSON it + publishes. -Sample #1 is the same record at every stage. A sample dropped on the way says where and why -("rejected: no_position", "dropped by Filter: drop_if matched"), and a tracker waiting for the rest -of a scan says it held the sample. The frames' other records are processed too (they count, and -move the tracker and throttle) but not shown. Above the samples, one line counts what the pipeline -did with all the stored samples (frames, records, dropped…), with any errors. +Sample #1 is the same record at every step. A sample dropped on the way to the step says where and +why ("rejected: no_position", "dropped by Filter 2: drop_if matched"); a drop on another branch is +that branch's. A tracker waiting for the rest of a scan says it held the sample. The frames' other +records are processed too (they count, and move trackers and throttles) but not shown. Above the +samples, one line counts what the flow did with all the stored samples (frames, records, +dropped…), with any errors. **Filter** keeps or drops observations by condition, in two parts: **Keep only if** (an observation that does not match is dropped) and **Drop if** (one that matches is dropped, even @@ -1013,7 +1036,7 @@ matched **all** or **any**: Under the rows a line says the condition in words, and under both, what the filter as edited does to the stored samples ("drops 55 (e.g. tms-OTK000058833, …)"). **Edit as JSON** shows the -condition as the pipeline stores it; a condition the rows cannot show (groups inside groups, a +condition as the source stores it; a condition the rows cannot show (groups inside groups, a transformed value) opens there. For example, to stop a TAK Server source reading back the tracks OpenTrack itself sends to that server: **Drop if**, `source_track_key`, starts with, `tms-`. @@ -1034,9 +1057,9 @@ domain, track type or confidence. Saved settings apply while the engine runs and ### Tracker profiles -A source's tracker stage can load a **tracker profile**, a sensor's tracker settings as a JSON file. -Profiles shipped with OpenTrack are read from `OT_PROFILES_DIR`; profiles imported or saved in the -pipeline designer go to `profiles/trackers/` beside the database. Back that directory up with the +A source's Tracker step can load a **tracker profile**, a sensor's tracker settings as a JSON file. +Profiles shipped with OpenTrack are read from `OT_PROFILES_DIR`; profiles imported or saved from a +Tracker step go to `profiles/trackers/` beside the database. Back that directory up with the database. ## Plugins @@ -1380,7 +1403,7 @@ What it holds (the file's sections): | Section | What | |---|---| | `format`, `version` | `"opentrack-config"`, `2`. Also `opentrack` (the release that wrote it), `site_code`, `exported_at`, `exported_by`, `notice`, `marking` (see [Markings](#markings)). | -| `sources` | Every source as stored: `id`, `name`, `transport`, `codec`, `enabled`, `priority`, `revision`, `spec` (with its secrets), `raw_subject` (raw output, if consented). | +| `sources` | Every source as stored: `id`, `name`, `transport`, `codec`, `enabled`, `priority`, `revision`, `spec` (with its secrets, its flow, and any NATS topics), `problems` (a draft's). | | `schema_versions` | Every output schema version (`version`, `status`, `published_at_ms`, `notes`, `fields`), the draft too. | | `correlation_settings` | The saved correlation settings, security labels included (`null`: never saved). | | `app_settings` | The instance settings as saved (site name, banners, position history, nodes), `{}` if never saved. | @@ -1438,7 +1461,7 @@ included, removed again if the transaction fails), recorded as one `import_confi first admin. - **API tokens:** set the same `OT_SESSION_SECRET` (or copy `session.key`) for them to work; otherwise revoke them and issue new ones. -- **Raw output** of a source is consented again by the import decision. +- **Raw output** of a source in a file from before flows becomes a NATS topic step beside its Output. - **Track numbers** continue from the file's counter (or this node's, if higher), so no number is issued twice. Keep the same `OT_SITE_CODE`; the import notes it if the file came from another site. diff --git a/docs/nats-output.md b/docs/nats-output.md index acebb1f..0952645 100644 --- a/docs/nats-output.md +++ b/docs/nats-output.md @@ -239,17 +239,18 @@ track update should filter on `OT-Op: upsert` and `delete`, or subscribe to `tra ## Other subjects -Nothing else OpenTrack publishes falls under `tracks.>`. Per-source raw output, when an admin -enables it with recorded consent, goes to `opentrack.raw.` (or an admin-chosen subject -outside `tracks.>`): core NATS (no stream, no acknowledgement), one message per observation as the -source's pipeline produced it, before correlation, as JSON (`source_id`, `source_track_key`, -`observed_at`, `position`, `kinematics`, `classification`, `identifiers`, `ext`, ...). It is a -diagnostic and integration feed, not a contract: its shape follows OpenTrack's internal -observation and can change between versions. +Nothing else OpenTrack publishes falls under `tracks.>`. A source's **NATS topic** steps publish +what reaches them on their own subjects, which must lie outside `tracks.>`: core NATS (no stream, +no acknowledgement), one message per observation as the source's flow made it up to that step, +before correlation, as JSON (`source_id`, `source_track_key`, `observed_at`, `position`, +`kinematics`, `classification`, `identifiers`, `ext`, ..., and the source's `security` label when it +has one). It is an integration feed, not a contract: its shape follows OpenTrack's internal +observation and can change between versions. A source saved with a raw-output subject before +flows publishes it from a NATS topic step beside its Output. ## Permissions OpenTrack needs -Publish on `tracks.>` and on any raw-output subjects, plus the JetStream API for the stream +Publish on `tracks.>` and on your sources' NATS topic subjects, plus the JetStream API for the stream (`$JS.API.STREAM.INFO.TRACKS`, and `$JS.API.STREAM.CREATE.TRACKS` unless the stream is created beforehand). Authenticate with `OT_NATS_CREDS` (credentials file), `OT_NATS_TOKEN`, or `OT_NATS_USER` and `OT_NATS_PASSWORD`. diff --git a/docs/reference.md b/docs/reference.md index 5a00591..f125703 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -4,7 +4,7 @@ The detail behind the [README](../README.md): what each part does, how it fits t ## What it does -**Tracker.** A source that reports anonymous plots (radar, lidar, GMTI) can run a tracker stage in +**Tracker.** A source that reports anonymous plots (radar, lidar, GMTI) can run a Tracker step in its pipeline: GNN (global nearest neighbour) or MHT (multiple hypothesis tracking, fewer false tracks in clutter). Each track carries its **existence** probability, from how well its plots fit against clutter and how often it goes unseen, which confirms and drops it; its reports carry the @@ -13,7 +13,7 @@ itself by the revisit rate it measures in the stream. A tracker's tracks carry n affiliation and at most a domain. **Tracker profiles** (`profiles/trackers/`) hold a sensor's tracker settings as a JSON file: wide-area and heavy-clutter GMTI, maritime surface MTI, marine, coastal and air surveillance radars, and lidar. The shipped profiles describe sensor kinds, not -named platforms. A source's tracker stage loads +named platforms. A source's Tracker step loads one; a tuned tracker is saved, imported or downloaded as one. **Track correlation.** The engine keeps every source's own tracks (source tracks) and pairs them @@ -140,8 +140,8 @@ ui/ React + TypeScript (Vite) on stareSDK The UI's tabs: * **Overview**: system status and metrics. -* **Sources**: topology of every pipeline, the source list, the add-source wizard, and the pipeline - designer with a live preview of each stage. +* **Sources**: every source's flow as a topology, the source list, and each source's editor: a + workspace of its steps, each with its settings and a live preview of its samples. * **Correlation**: the engine's suggestions to accept or reject, correlation settings (applied while running) and every decision with its evidence. * **Track Management**: map, track card (GOLD fields, attributes, provenance, **Edit**) and the @@ -155,10 +155,43 @@ The UI's tabs: ## Sources and pipelines -A source is a transport (`tcp_client`, `tcp_server`, `udp` with IPv4 or IPv6 multicast, `http_poll`, `websocket`, -`mqtt`, `grpc_client`, `grpc_server`, `file`, which reads only under the data directory), framing, a codec and a pipeline. Transport metadata reaches the mapping under -`_frame` (an MQTT topic is `_frame.topic`, `_frame.topic_levels[1]` its second level). Secrets are -written as `${env:NAME}` and resolved when the source starts. +A source is an input (a transport: `tcp_client`, `tcp_server`, `udp` with IPv4 or IPv6 multicast, +`http_poll`, `websocket`, `mqtt`, `grpc_client`, `grpc_server`, `file`, which reads only under the +data directory; its framing; and a codec) and a **flow** of steps after it, under `pipeline`: + +```json +"pipeline": { + "codec": {"type": "json"}, + "nodes": [ + {"id": "input", "type": "input"}, + {"id": "map", "type": "map", "config": {"mapping": {"rules": [ … ]}, "static_join": {"ttl_secs": 604800}}}, + {"id": "filter", "type": "filter", "config": {"drop_if": { … }}}, + {"id": "output", "type": "output"}, + {"id": "raw", "type": "nats", "config": {"subject": "feeds.ais.raw"}} + ], + "edges": [{"source": "input", "target": "map"}, {"source": "map", "target": "filter"}, + {"source": "filter", "target": "output"}, {"source": "map", "target": "raw"}] +} +``` + +Steps are `reject` (records), `map` (records to observations), `filter`, `registry` (Entity links), +`affiliation`, `tracker`, `throttle` (observations), `output` (system tracks, to correlation; at most +one) and `nats` (published as JSON on its subject, without correlation). Each takes one input, any +step may feed several, the flow does not loop, and each step takes what the step before passes on. +A path to the Output with no `registry` step runs the entity lookup with its defaults before the +Output. A `position` on a step is where the editor draws it. A spec in the older fixed form +(`mapping`, `filter`, `tracker`… beside `codec`) still reads, as the same stages in a line. +Transport metadata reaches the mapping under `_frame` (an MQTT topic is `_frame.topic`, +`_frame.topic_levels[1]` its second level). Secrets are written as `${env:NAME}` and resolved when +the source starts. + +A source that is not complete yet (a setting left empty, no mapping rules, a step leading nowhere, +an unpublished schema version) saves as a **draft**: its row has `"draft": true` and `problems`, +each `{kind, node, field?, message}`, where `node` is the step it belongs to (`input` for the +transport and codec) and `null` for the source as a whole. A draft cannot be enabled. A source +without a valid id or name, or whose flow cannot run at all (a loop, a step fed the wrong data, two +inputs into one step, two Outputs), is not saved (`kind: "invalid"`), and an enabled source is saved +only complete. A refusal is a 422 whose body lists the `problems` beside the `error`. A source that is not complete yet (a setting left empty, no mapping rules, a mapping problem, an unpublished schema version) saves as a **draft**: its row has `"draft": true` and `problems`, each @@ -181,9 +214,9 @@ bearer `token` for `grpc_server`); one that does not, and every `udp` source, ru A source reports either **tracks** (a key per object: AIS, ADS-B, TAK, a radar's own tracks) or **detections** (`"reports": "detections"`: anonymous plots). Detections update the nearest system -track another source keeps or, with a tracker stage, become tracks first. A mapping rule of kind +track another source keeps or, through a Tracker step, become tracks first. A mapping rule of kind `static` caches identity fields (AIS static data) that the static join fills into later reports; a -rule of kind `track` bypasses the tracker. A mapping can send a value straight to the matched entity +rule of kind `track` passes Tracker steps by. A mapping can send a value straight to the matched entity with an `entity.` destination. **Protobuf over gRPC.** Upload the producer's `.proto` files with the source (the `protobuf` @@ -201,7 +234,7 @@ from several sensors are cross-fixed into positions that start or update tracks, out. See [docs/non-point-contacts.md](non-point-contacts.md). **Codec plugins** decode formats beyond JSON and XML (`"codec": {"type": "plugin", "plugin": -"stanag4607", "options": {...}}`), and **tracker plugins** run as a tracker stage (see Plugins +"stanag4607", "options": {...}}`), and **tracker plugins** run in a Tracker step (see Plugins below). **STANAG 4607** GMTI streams over TCP; [docs/examples/stanag4607.json](examples/stanag4607.json) turns every dwell target into a detection with a ground error ellipse from the radar geometry, forms ground tracks with the GNN tracker timed by the measured revisit rate, and publishes the sensor @@ -210,7 +243,7 @@ plugin: each `SapientMessage` (over TCP with a 4-byte little-endian length prefi message) becomes a JSON record for the mapping; [docs/examples/sapient.json](examples/sapient.json) maps detection reports. -Per source, the publish stage sets: +Per source (its Feed settings): * **Publish alone**: whether a track only this source reports for is published (default: yes for track feeds, no for detections, whose tracks wait for a track feed to report for them too). diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index b773ef6..18e1f64 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -832,14 +832,23 @@ const enc = encodeURIComponent // --- Sources ------------------------------------------------------------------------------- /** A source spec as stored: transport + pipeline, all JSON. */ +/** A Map step's mapping. */ +export interface MappingSpec { + schema_version?: number + rules: unknown[] + reject?: unknown[] +} + export interface SourceSpec { id: string name: string description?: string transport: { type: string; [k: string]: unknown } + /** The codec, and the flow of steps after the input (crates/ot-source/src/flow.rs). */ pipeline: { codec: { type: string; [k: string]: unknown } - mapping: { schema_version?: number; rules: unknown[]; reject?: unknown[] } + nodes?: unknown[] + edges?: unknown[] [k: string]: unknown } priority?: number @@ -913,25 +922,12 @@ export interface SourceRow { status: SourceStatus | null } -/** A pipeline stage, in the order a frame meets them. */ -export type SourceStage = - | 'source' - | 'decode' - | 'map' - | 'static_join' - | 'registry' - | 'affiliation' - | 'filter' - | 'tracker' - | 'throttle' - | 'output' - /** One reason a source may not run. */ export interface SourceProblem { /** `invalid`: the source cannot be saved. `incomplete`: it saves as a draft. */ kind: 'invalid' | 'incomplete' - /** The stage it belongs to; null for the source as a whole (its id and name). */ - stage: SourceStage | null + /** The step of the flow it belongs to (`input` for the transport and codec); null for the source as a whole. */ + node: string | null /** The setting, where there is one (for example `host`). */ field?: string message: string @@ -980,7 +976,7 @@ export interface ProbeResult { codec: SourceSpec['pipeline']['codec'] suggested_records_path: string | null fields: FieldStat[] - suggestion: { mapping: SourceSpec['pipeline']['mapping']; proposals: Proposal[]; identifiers: unknown[]; missing: string[] } + suggestion: { mapping: MappingSpec; proposals: Proposal[]; identifiers: unknown[]; missing: string[] } samples_saved: number | null sample_frames: { received_at: string @@ -1001,7 +997,10 @@ export interface PreviewObservation extends Observation { } export interface PreviewResult { + /** Whether the source is complete; a draft is dry-run as far as it can be. */ valid: boolean + /** What stops the source running, each on its stage. */ + problems?: SourceProblem[] counts?: Record errors?: string[] observations?: PreviewObservation[] diff --git a/ui/src/index.css b/ui/src/index.css index 4eb131c..e0e4ddd 100644 --- a/ui/src/index.css +++ b/ui/src/index.css @@ -561,24 +561,6 @@ body { font-size: 11px; } -/* Pipeline designer (a modal): stage flowchart, the stage's form, the live preview. */ -.designer { - display: grid; - grid-template-columns: 220px minmax(0, 1fr) minmax(320px, 420px); - gap: var(--space-lg); - align-items: start; - min-height: 60vh; -} - -/* The live preview fills the modal's height (its max, less the title bar and padding) and its - samples scroll inside it; it stays in view if a long stage form scrolls the modal. */ -.designer-preview { - position: sticky; - top: 0; - height: calc(100vh - 160px - 2 * var(--space-lg)); - min-height: 0; -} - .stage-samples { flex: 1; min-height: 0; @@ -618,17 +600,6 @@ body { padding-top: 4px; } -@media (max-width: 1300px) { - .designer { - grid-template-columns: 220px minmax(0, 1fr); - } - .designer-preview { - grid-column: 1 / -1; - position: static; - height: 70vh; - } -} - .value-editor { display: flex; flex-direction: column; @@ -933,3 +904,148 @@ body { .symbol-designer { gap: var(--space-md); } + +/* Source editor (#104): a workspace of the source's steps, the step list on its left; a step's + settings (or the feed's) open in a SideNav over the right of the page. It fills the page below + the tabs. */ +.source-editor { + display: flex; + flex-direction: column; + gap: var(--space-sm); + height: calc(100vh - 120px); + min-height: 480px; + padding: var(--space-sm) var(--space-md); +} + +.source-editor-head { + display: flex; + align-items: center; + gap: var(--space-sm); + flex-wrap: wrap; +} + +.source-editor-general { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + max-width: 40vw; +} + +.source-workspace { + flex: 1; + min-height: 0; +} + +.source-workspace-canvas { + height: 100%; +} + +.source-editor-json { + flex: 1; + min-height: 0; + overflow: hidden; +} + +.node-panel { + display: flex; + flex-direction: column; + flex: 1; + min-width: 0; + min-height: 0; + height: 100%; +} + +.node-panel-head { + display: flex; + align-items: center; + gap: var(--space-sm); + padding: var(--space-sm) var(--space-md); + border-bottom: 1px solid var(--color-glass-border); +} + +.node-panel-body { + flex: 1; + min-height: 0; + overflow-y: auto; + padding: var(--space-md); + display: flex; + flex-direction: column; + gap: var(--space-sm); +} + +.step-settings { + border: 0; + margin: 0; + padding: 0; + min-width: 0; +} + +/* A step on the workspace. */ +.flow-step { + width: 220px; + padding: 8px 10px; + border-radius: var(--radius-md); + background: var(--color-bg-secondary); + font-family: var(--font-sans); + display: flex; + flex-direction: column; + gap: 4px; +} + +.flow-step-head { + display: flex; + align-items: center; + gap: 6px; +} + +.flow-step-icon { + display: flex; + color: var(--color-text-secondary); +} + +.flow-step-title { + font-size: 12px; + font-weight: 600; + color: var(--color-text-primary); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.flow-step-kind { + margin-left: auto; + font-size: 9px; + color: var(--color-text-secondary); + text-transform: uppercase; + letter-spacing: 0.08em; +} + +.node-panel-head .flow-step-kind { + margin-left: 0; +} + +.flow-step-summary { + font-size: 10px; + font-family: var(--font-mono); + color: var(--color-text-secondary); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +/* The first problem, in two lines at most; the step's settings list them all. */ +.flow-step-problem { + font-size: 10px; + color: var(--color-destructive); + line-height: 1.3; + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; +} + +.flow-step-problem-more { + font-size: 10px; + color: var(--color-destructive); +} diff --git a/ui/src/lib/pipeline.test.ts b/ui/src/lib/pipeline.test.ts index 7ec2a8e..72b366b 100644 --- a/ui/src/lib/pipeline.test.ts +++ b/ui/src/lib/pipeline.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import type { SchemaOverview, SourceSpec } from '../api/client' -import { describeCondition, describeValue, destinations, fieldMap, pipelineStages, transportCodec, valueSources } from './pipeline' +import { describeCondition, describeValue, destinations, fieldMap, valueSources } from './pipeline' const schema = { published_core: [], @@ -81,42 +81,3 @@ describe('destinations', () => { }) }) -describe('pipelineStages', () => { - it('shows a tracker stage before publishing, with the classification it may give', () => { - const spec = { - id: 'radar', - name: 'Radar', - transport: { type: 'udp' }, - reports: 'detections', - pipeline: { - codec: { type: 'json' }, - mapping: { rules: [{ name: 'plot', key: 'id', fields: {} }] }, - tracker: { algorithm: 'mht', domain: 'surface' }, - }, - } as unknown as SourceSpec - const stages = pipelineStages(spec) - const ids = stages.map((s) => s.id) - expect(ids.indexOf('tracker')).toBe(ids.length - 2) - const tracker = stages.find((s) => s.id === 'tracker')! - expect(tracker.title).toBe('tracker · MHT') - expect(tracker.facts.find((f) => f.label === 'Classification')!.value).toBe('unknown affiliation, surface; no identity') - // With a tracker the feed's plots arrive as tracks. - expect(stages.at(-1)!.summary).toBe('system track, published to NATS') - }) -}) - -describe('transportCodec', () => { - it('leaves the codecs the Decode stage edits to it', () => { - expect(transportCodec(undefined)).toBeNull() - expect(transportCodec({ type: 'json', records: 'ac' })).toBeNull() - expect(transportCodec({ type: 'cot_xml' })).toBeNull() - expect(transportCodec({ type: 'xml', record_element: 'r' })).toBeNull() - }) - - it('shows protobuf and plugin codecs with their message or plugin', () => { - expect(transportCodec({ type: 'protobuf', files: {}, message: 'acme.v1.Batch' })).toEqual({ type: 'protobuf', name: 'acme.v1.Batch' }) - expect(transportCodec({ type: 'protobuf', files: {}, message: '' })).toEqual({ type: 'protobuf' }) - expect(transportCodec({ type: 'plugin', plugin: 'sapient', options: {} })).toEqual({ type: 'plugin', name: 'sapient' }) - expect(transportCodec({ type: 'future_codec' })).toEqual({ type: 'future_codec' }) - }) -}) diff --git a/ui/src/lib/pipeline.ts b/ui/src/lib/pipeline.ts index ec905ad..1eb4916 100644 --- a/ui/src/lib/pipeline.ts +++ b/ui/src/lib/pipeline.ts @@ -1,5 +1,5 @@ /** - * Plain-language views of a source pipeline, for the Pipeline flowchart and field map. + * Plain-language views of a source's settings, for its steps and the field map. * * Mirrors the server's evaluation (ot-source `expr.rs`, `mapping.rs`, `pipeline.rs`): a value spec * is a path string or an object with one source (`path`, `const`, `first`, `template`, `cases`, @@ -100,206 +100,6 @@ export function describeValue(spec: ValueSpec | undefined): string { return [bits.join(' '), ...after].filter(Boolean).join(' · ') } -// --- Stages -------------------------------------------------------------------------------- - -export interface Fact { - label: string - value: string -} - -export interface Stage { - id: string - title: string - /** One line under the title in the flowchart. */ - summary: string - facts: Fact[] -} - -const secs = (s: number) => (s >= 86400 ? `${+(s / 86400).toFixed(1)} d` : s >= 3600 ? `${+(s / 3600).toFixed(1)} h` : s >= 60 ? `${+(s / 60).toFixed(1)} min` : `${s} s`) - -function transportStage(spec: SourceSpec): Stage { - const t = spec.transport - const where = String(t.url ?? t.bind ?? (t.host ? `${t.host}:${t.port ?? ''}` : '')) - const facts: Fact[] = Object.entries(t) - .filter(([k]) => k !== 'type') - .map(([k, v]) => ({ label: k, value: typeof v === 'string' ? v : JSON.stringify(v) })) - return { id: 'transport', title: t.type.replace('_', ' '), summary: where, facts } -} - -/** The stages a record passes through, in the order the source worker runs them. */ -export function pipelineStages(spec: SourceSpec): Stage[] { - const p = spec.pipeline as SourceSpec['pipeline'] & Obj - const codec = p.codec - const rules = (p.mapping.rules as Obj[]) ?? [] - const reject = (p.mapping.reject as Obj[]) ?? [] - const fieldCount = rules.reduce((n, r) => n + Object.keys((r.fields as Obj) ?? {}).length, 0) - const out: Stage[] = [ - transportStage(spec), - { - id: 'decode', - title: `decode ${codec.type}`, - summary: typeof codec.records === 'string' ? `records at "${codec.records}"` : 'one record per frame', - facts: Object.entries(codec) - .filter(([k]) => k !== 'type') - .map(([k, v]) => ({ label: k, value: typeof v === 'string' ? v : JSON.stringify(v) })), - }, - ] - if (reject.length) { - out.push({ - id: 'reject', - title: 'reject', - summary: plural(reject.length, 'rule'), - facts: reject.map((r) => ({ label: String(r.reason), value: describeCondition(r.when) })), - }) - } - out.push({ - id: 'map', - title: 'map', - summary: `${plural(rules.length, 'rule')} · ${plural(fieldCount, 'field')}`, - facts: [], - }) - if (rules.some((r) => r.kind === 'static')) { - const ttl = Number((p.static_join as Obj | undefined)?.ttl_secs ?? 7 * 86400) - out.push({ - id: 'join', - title: 'identity join', - summary: `keeps identity fields ${secs(ttl)}`, - facts: [ - { label: 'What it does', value: 'Fields from static rules are cached per track key and fill gaps in later position reports.' }, - { label: 'Kept for', value: secs(ttl) }, - ], - }) - } - const reg = p.registry as Obj | undefined - const links = entityLinks(reg) - const toTrack = links.filter((l) => l.direction !== 'to_entity').length - out.push({ - id: 'registry', - title: 'entity', - summary: `${plural(toTrack, 'field')} from the entity${links.length > toTrack ? `, ${links.length - toTrack} back to it` : ''}${reg ? '' : ' (defaults)'}`, - facts: [ - { label: 'Schemes', value: ((reg?.schemes as string[]) ?? []).join(', ') || 'every identifier' }, - { label: 'Name to grade', value: String(reg?.broadcast_name ?? 'name') }, - { label: 'Used at grades', value: ((reg?.apply_grades as string[]) ?? DEFAULT_GRADES).join(', ') }, - ...links.map((l) => - l.direction === 'to_entity' - ? { label: `updates entity ${l.entity}`, value: `from the track's ${l.track}` } - : { label: `sets ${l.track}`, value: `from the entity's ${l.entity}` }, - ), - ], - }) - const aff = p.affiliation as Obj | undefined - if (aff) { - const n = (k: string) => ((aff[k] as string[]) ?? []).length - out.push({ - id: 'affiliation', - title: 'affiliation', - summary: `by country · ${n('friendly')} friendly, ${n('hostile')} hostile`, - facts: [ - { label: 'Country from', value: describeValue(aff.country as ValueSpec) }, - { label: 'Friend', value: ((aff.friendly as string[]) ?? []).join(' ') || '—' }, - { label: 'Hostile', value: ((aff.hostile as string[]) ?? []).join(' ') || '—' }, - { label: 'Neutral', value: ((aff.neutral as string[]) ?? []).join(' ') || '—' }, - { label: 'Other countries', value: String(aff.otherwise ?? 'left unset') }, - { label: 'No country', value: String(aff.unknown_country ?? 'left unset') }, - ], - }) - } - const filter = p.filter as Obj | undefined - if (filter) { - const facts: Fact[] = [] - if (filter.keep_if) facts.push({ label: 'Keep only if', value: describeCondition(filter.keep_if) }) - if (filter.drop_if) facts.push({ label: 'Drop if', value: describeCondition(filter.drop_if) }) - out.push({ id: 'filter', title: 'filter', summary: filter.keep_if ? 'keep only matching' : 'drop matching', facts }) - } - const tr = p.tracker as Obj | undefined - if (tr) { - const mht = (tr.mht as Obj | undefined) ?? {} - const alg = tr.algorithm === 'mht' ? 'MHT' : tr.algorithm === 'plugin' ? `plugin ${String(tr.plugin ?? '')}` : 'GNN' - const profile: Fact[] = tr.profile ? [{ label: 'Profile', value: String(tr.profile) }] : [] - const facts: Fact[] = - tr.algorithm === 'plugin' - ? [ - ...profile, - { label: 'Tracker', value: `the ${String(tr.plugin ?? '?')} plugin (Settings → General → Plugins)` }, - { label: 'Options', value: Object.keys((tr.options as Obj | undefined) ?? {}).length ? JSON.stringify(tr.options) : 'its defaults' }, - { label: 'Track keys', value: 'as the plugin names them' }, - ] - : [ - ...profile, - { label: 'Association', value: alg === 'MHT' ? `multiple hypotheses, final after ${Number(mht.n_scan ?? 3)} scans` : 'global nearest neighbour' }, - { label: 'Classification', value: `unknown affiliation${tr.domain ? `, ${String(tr.domain)}` : ', domain from the plots if they report one'}; no identity` }, - { - label: 'Confirmed at', - value: `existence ${Number(tr.confirm_probability ?? 0.95)}, after at least ${Number(tr.confirm_hits ?? 3)} plots`, - }, - { label: 'Clutter', value: `${Number(tr.clutter_density ?? 1e-6)} false plots per m², detection ${Number(tr.detection_probability ?? 0.9)}` }, - { - label: 'Dropped after', - value: tr.auto_timing != null ? 'a few missed revisits (auto timing)' : `${secs(Number(tr.drop_confirmed_secs ?? 8))} without a plot`, - }, - { label: 'Track keys', value: `${String(tr.key_prefix ?? 'T')}-1, -2…` }, - ] - out.push({ - id: 'tracker', - title: `tracker · ${alg}`, - summary: `plots → tracks · ${tr.domain ? `${String(tr.domain)}, ` : ''}${tr.profile ? `${String(tr.profile)} · ` : ''}unknown affiliation`, - facts, - }) - } - const th = p.throttle as Obj | undefined - if (th) { - const min = Number(th.min_interval_secs ?? 0) - const beat = Number(th.heartbeat_secs ?? 600) - const move = Number(th.min_move_m ?? 0) - out.push({ - id: 'throttle', - title: 'throttle', - summary: `≥ ${secs(min)} apart · ${move} m move`, - facts: [ - { label: 'At most every', value: secs(min) }, - { label: 'Between that and the heartbeat', value: move > 0 ? `only after moving ${move} m` : 'every report' }, - { label: 'Heartbeat', value: `always after ${secs(beat)}` }, - ], - }) - } - const plots = spec.reports === 'detections' && !tr - out.push({ - id: 'publish', - title: 'correlation → TRACKS', - summary: plots ? 'plots associated with system tracks' : 'system track, published to NATS', - facts: [ - { - label: 'Correlation', - value: plots - ? 'Each plot updates the nearest system track inside the gate; plots with none are dropped.' - : 'Each source track reports for one system track, paired with other sources’ tracks on a shared identifier or kinematic agreement.', - }, - { - label: 'Published', - value: 'Confirmed tracks that a source allowed to stand alone reports for: the GOLD fields plus the output schema’s attributes.', - }, - ], - }) - return out -} - -// --- Decode stage ------------------------------------------------------------------------ - -/** The codecs the Decode stage edits; the others (protobuf, plugins) are chosen on the Transport tab. */ -export const DECODE_CODECS = ['json', 'cot_xml', 'xml'] - -/** - * A codec the Decode stage only shows: its type and its message or plugin, or null for one it - * edits. Choosing a Decode format would replace such a codec's schema, plugin and options. - */ -export function transportCodec(codec: Obj | undefined): { type: string; name?: string } | null { - const type = String(codec?.type ?? 'json') - if (DECODE_CODECS.includes(type)) return null - const name = type === 'protobuf' ? codec?.message : type === 'plugin' ? codec?.plugin : undefined - return typeof name === 'string' && name ? { type, name } : { type } -} - // --- Entity links -------------------------------------------------------------------------- export interface EntityLink { @@ -387,7 +187,13 @@ export function destinations(target: string, schema: SchemaOverview): Destinatio export function fieldMap(spec: SourceSpec, schema: SchemaOverview): FieldRow[] { const p = spec.pipeline as SourceSpec['pipeline'] & Obj const rows: FieldRow[] = [] - for (const [i, r] of ((p.mapping.rules as Obj[]) ?? []).entries()) { + // Every Map's rules, and the first Entity links step's (a source from before flows has one of each). + const nodes = (p.nodes as { type?: string; config?: Obj }[] | undefined) ?? [] + const rules = nodes.length + ? nodes.filter((n) => n.type === 'map').flatMap((n) => ((n.config?.mapping as Obj | undefined)?.rules as Obj[]) ?? []) + : (((p.mapping as Obj | undefined)?.rules as Obj[]) ?? []) + const registry = nodes.length ? nodes.find((n) => n.type === 'registry')?.config : (p.registry as Obj | undefined) + for (const [i, r] of rules.entries()) { const stage = `${String(r.name ?? `rule ${i + 1}`)}${r.kind === 'static' ? ' (identity)' : ''}` rows.push({ id: `${i}:key`, @@ -420,7 +226,7 @@ export function fieldMap(spec: SourceSpec, schema: SchemaOverview): FieldRow[] { }) } } - for (const [n, l] of entityLinks(p.registry as Obj | undefined).entries()) { + for (const [n, l] of entityLinks(registry).entries()) { rows.push( l.direction === 'to_entity' ? { diff --git a/ui/src/lib/sourceState.ts b/ui/src/lib/sourceState.ts index 3975afd..96b468c 100644 --- a/ui/src/lib/sourceState.ts +++ b/ui/src/lib/sourceState.ts @@ -1,8 +1,9 @@ import type { BadgeColor } from '@kineticlogic/staresdk' import type { SourceRow } from '../api/client' -/** Running / starting / stopped / failing, from config plus the worker's live status. */ +/** Draft / disabled / running / starting / failing, from config plus the worker's live status. */ export function sourceState(s: SourceRow): { label: string; color: BadgeColor } { + if (s.draft) return { label: 'draft', color: 'grey' } if (!s.enabled) return { label: 'disabled', color: 'grey' } if (!s.status) return { label: 'starting', color: 'warning' } if (s.status.not_started) return { label: 'not started', color: 'danger' } @@ -12,9 +13,9 @@ export function sourceState(s: SourceRow): { label: string; color: BadgeColor } /** What each source state means, for an info tip. */ export const SOURCE_STATES_INFO = - 'disabled: switched off, not running. starting: enabled, its worker has not reported yet. running: connected (or listening) and reading. ' + + 'draft: not complete yet, so it cannot be turned on; its problems show on its steps. disabled: switched off, not running. starting: enabled, its worker has not reported yet. running: connected (or listening) and reading. ' + 'connecting: not connected yet, no errors so far. failing: not connected and its link has hit errors; it keeps retrying. ' + - 'not started: the worker refuses to run it as configured (a listener that does not authenticate its senders, say). See the error on its Status tab.' + 'not started: the worker refuses to run it as configured (a listener that does not authenticate its senders, say). See the error in its status.' /** What each pipeline counter means, for an info tip. */ export const PIPELINE_COUNTS_INFO = diff --git a/ui/src/lib/trace.test.ts b/ui/src/lib/trace.test.ts index 335e444..7442845 100644 --- a/ui/src/lib/trace.test.ts +++ b/ui/src/lib/trace.test.ts @@ -74,7 +74,7 @@ describe('samplesAt', () => { const filter = samplesAt(trace, 'filter') expect(filter[2].notes).toEqual(['dropped by Filter: drop_if matched']) // Not a stage of this sample's (older) trace. - expect(filter[3].notes[0]).toMatch(/not in the pipeline/) + expect(filter[3].notes[0]).toMatch(/not in the flow/) }) it('says when a stage holds a sample, and when nothing came out', () => { @@ -84,7 +84,7 @@ describe('samplesAt', () => { const publish = samplesAt(trace, 'publish') expect(publish[0].notes.some((n) => n.startsWith('held by Tracker'))).toBe(true) const one: PreviewTrace = { frames: trace.frames, samples: [{ frame: 0, stages: [{ id: 'map', items: [] }] }] } - expect(samplesAt(one, 'map')[0].notes).toEqual(['Nothing came out of this stage.']) + expect(samplesAt(one, 'map')[0].notes).toEqual(['Nothing came out of this step.']) }) it('shows at most five samples, and none without a trace', () => { diff --git a/ui/src/lib/trace.ts b/ui/src/lib/trace.ts index 027330a..32d8dee 100644 --- a/ui/src/lib/trace.ts +++ b/ui/src/lib/trace.ts @@ -1,14 +1,15 @@ /** - * The pipeline designer's live preview: the first stored records (samples) as they are after one - * stage, from the dry run's trace (`POST /sources/validate` with `trace`). + * A source step's samples: the first stored records as they are after that step, from the dry + * run's trace (`POST /sources/validate` with `trace`). */ import type { PreviewResult, PreviewTrace, TraceFrameView } from '../api/client' /** Records the designer follows through the pipeline. */ export const TRACED_SAMPLES = 5 -/** Stage names for "dropped by …", by the stage ids of `pipelineStages`. */ +/** Step names for "dropped by …", by step type (a step's id starts with its type). */ export const STAGE_LABEL: Record = { + input: 'Input', transport: 'Transport', decode: 'Decode', reject: 'Reject', @@ -51,9 +52,16 @@ function frameView(f: TraceFrameView | undefined, title: string): SampleView { return { title, blocks: [block], notes } } +/** A step's name for a note: its type's, numbered like its id (`Filter 2`). */ +function label(stageId: string): string { + const m = /^(.*?)(?:-(\d+))?$/.exec(stageId) + const name = STAGE_LABEL[m?.[1] ?? stageId] + return name ? `${name}${m?.[2] ? ` ${m[2]}` : ''}` : stageId +} + function note(stageId: string, reason: string): string { if (reason.startsWith('rejected:')) return reason - return `dropped by ${STAGE_LABEL[stageId] ?? stageId}: ${reason}` + return `dropped by ${label(stageId)}: ${reason}` } /** @@ -62,9 +70,9 @@ function note(stageId: string, reason: string): string { * dropped at or before the stage says where and why, and one a stage held back (a tracker * waiting for the rest of a scan) says so. */ -export function samplesAt(trace: PreviewTrace | undefined, stageId: string): SampleView[] { +export function samplesAt(trace: PreviewTrace | undefined, stageId: string, path?: string[]): SampleView[] { const samples = (trace?.samples ?? []).slice(0, TRACED_SAMPLES) - if (stageId === 'transport') { + if (stageId === 'transport' || stageId === 'input') { const byFrame = new Map() samples.forEach((s, i) => byFrame.set(s.frame, [...(byFrame.get(s.frame) ?? []), i + 1])) return [...byFrame].map(([frame, ns]) => @@ -74,20 +82,22 @@ export function samplesAt(trace: PreviewTrace | undefined, stageId: string): Sam return samples.map((s, i) => { const title = `#${i + 1}` const at = s.stages.findIndex((x) => x.id === stageId) - if (at < 0) return { title, blocks: [], notes: ['This stage was not in the pipeline the preview ran.'] } + if (at < 0) return { title, blocks: [], notes: ['This step was not in the flow the preview ran.'] } const notes: string[] = [] - for (const x of s.stages.slice(0, at + 1)) { + // Only the steps on the way here: a drop on another branch is not this step's. + const before = s.stages.slice(0, at + 1).filter((x) => !path || path.includes(x.id) || x === s.stages[at]) + for (const x of before) { for (const r of x.dropped ?? []) notes.push(note(x.id, r)) // Not a drop: said only at the stage that did it. - if (x === s.stages[at]) for (const n of x.notes ?? []) notes.push(`${STAGE_LABEL[x.id] ?? x.id}: ${n}`) + if (x === s.stages[at]) for (const n of x.notes ?? []) notes.push(`${label(x.id)}: ${n}`) if (x.held) { notes.push( - `held by ${STAGE_LABEL[x.id] ?? x.id}: waiting for the rest of its scan (the preview ran it at the end of the samples, or when a later frame completed it)`, + `held by ${label(x.id)}: waiting for the rest of its scan (the preview ran it at the end of the samples, or when a later frame completed it)`, ) } } const blocks = s.stages[at].items.map(pretty) - if (!blocks.length && !notes.length) notes.push('Nothing came out of this stage.') + if (!blocks.length && !notes.length) notes.push('Nothing came out of this step.') return { title, blocks, notes } }) } @@ -106,12 +116,21 @@ export const MAX_TRACED_SAMPLES = 10 * on), and those of them the filter dropped. With no filter in the pipeline yet, what the last * stage before where it goes passed on, and nothing dropped. */ -export function filterInput(trace: PreviewTrace | undefined): { observations: unknown[]; dropped: unknown[] } { +export function filterInput( + trace: PreviewTrace | undefined, + id = 'filter', + parent?: string, +): { observations: unknown[]; dropped: unknown[] } { const observations: unknown[] = [] const dropped: unknown[] = [] for (const s of trace?.samples ?? []) { - const at = s.stages.findIndex((x) => x.id === 'filter') - const before = at > 0 ? s.stages[at - 1] : [...s.stages].reverse().find((x) => ['affiliation', 'registry', 'join', 'map'].includes(x.id)) + const at = s.stages.findIndex((x) => x.id === id) + // What reaches it: the step that feeds it. + const before = parent + ? s.stages.find((x) => x.id === parent) + : at > 0 + ? s.stages[at - 1] + : [...s.stages].reverse().find((x) => ['affiliation', 'registry', 'join', 'map'].includes(x.id)) if (!before) continue observations.push(...before.items) if (at > 0 && s.stages[at].dropped?.length) { diff --git a/ui/src/lib/usePreview.ts b/ui/src/lib/usePreview.ts deleted file mode 100644 index 97c8a01..0000000 --- a/ui/src/lib/usePreview.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { useEffect, useRef, useState } from 'react' -import { api, type PreviewResult, type SourceSpec } from '../api/client' -import { errorMessage } from './format' -import { useCan } from '../auth/context' - -const PREVIEW_DELAY_MS = 600 - -/** - * A dry run of `spec` over `sampleSourceId`'s stored probe samples, re-run shortly after the spec - * stops changing. Nothing is saved or published. `trace` > 0 also follows that many samples stage by - * stage. - */ -export function usePreview(spec: SourceSpec, sampleSourceId: string, nonce = 0, trace = 0) { - const [preview, setPreview] = useState(null) - const [error, setError] = useState(null) - const [running, setRunning] = useState(false) - const seq = useRef(0) - const canAdmin = useCan('admin') - - useEffect(() => { - if (!canAdmin) return - const id = ++seq.current - const t = setTimeout(() => { - setRunning(true) - api - .preview(spec, sampleSourceId, [], trace) - .then((r) => { - if (id !== seq.current) return - setPreview(r) - setError(null) - }) - .catch((e) => { - if (id !== seq.current) return - setError(errorMessage(e)) - }) - .finally(() => id === seq.current && setRunning(false)) - }, PREVIEW_DELAY_MS) - return () => clearTimeout(t) - }, [spec, sampleSourceId, nonce, canAdmin, trace]) - - // The dry run is an admin's (it can reach the source's own transport). - if (!canAdmin) return { preview: null, error: 'The preview needs the admin role.', running: false } - return { preview, error, running } -} diff --git a/ui/src/pages/sources/AddSourceWizard.tsx b/ui/src/pages/sources/AddSourceWizard.tsx deleted file mode 100644 index d7195d1..0000000 --- a/ui/src/pages/sources/AddSourceWizard.tsx +++ /dev/null @@ -1,343 +0,0 @@ -import { useMemo, useState } from 'react' -import { TbArrowLeft, TbArrowRight, TbCheck, TbRadar, TbX } from 'react-icons/tb' -import { Badge, Button, CollapsiblePanel, DataTable, Input, Label, Stepper, Toggle, useToast, type DataTableColumn, type StepStatus } from '@kineticlogic/staresdk' -import { CodeEditor } from '@kineticlogic/staresdk/code-editor' -import { api, type FieldStat, type ProbeResult, type SourceSpec } from '../../api/client' -import { InfoTip } from '../../components/InfoTip' -import { errorMessage } from '../../lib/format' -import { MappingStudio } from './MappingStudio' -import { needsAcceptance } from '../../lib/senderAuth' -import { SenderAuth } from './SenderAuth' -import { TransportForm } from './TransportForm' - -const STEPS = [ - { id: 'connect', label: 'Connect' }, - { id: 'probe', label: 'Probe' }, - { id: 'map', label: 'Map' }, - { id: 'review', label: 'Review' }, -] as const - -const ID_RE = /^[a-z0-9_-]{1,64}$/ - -const FIELD_COLUMNS: DataTableColumn[] = [ - { key: 'path', header: 'Field', mono: true, width: '32%', render: (f) => f.path, sortValue: (f) => f.path }, - { key: 'types', header: 'Types', width: '16%', render: (f) => Object.keys(f.types).join(', ') }, - { - key: 'presence', - header: 'Present', - align: 'right', - width: '11%', - render: (f) => `${Math.round(f.presence * 100)}%`, - sortValue: (f) => f.presence, - }, - { - key: 'distinct', - header: 'Distinct', - align: 'right', - width: '11%', - render: (f) => `${f.distinct}${f.distinct_capped ? '+' : ''}`, - sortValue: (f) => f.distinct, - }, - { - key: 'samples', - header: 'Samples', - mono: true, - render: (f) => f.samples.map((s) => (typeof s === 'string' ? s : JSON.stringify(s))).join(' · '), - }, -] - -function connectProblems(spec: SourceSpec): string[] { - const p: string[] = [] - if (!ID_RE.test(spec.id)) p.push('an id of a-z, 0-9, - or _') - if (!spec.name.trim()) p.push('a name') - const t = spec.transport - if ((t.type === 'http_poll' || t.type === 'websocket') && !String(t.url ?? '').trim()) p.push('a URL') - if (t.type === 'mqtt') { - if (!/^mqtts?:\/\/./.test(String(t.url ?? '').trim())) p.push('a broker URL (mqtt:// or mqtts://)') - if (!Array.isArray(t.topics) || t.topics.length === 0) p.push('at least one topic') - if (t.clean_session === false && !String(t.client_id ?? '').trim()) p.push('a client id for a persistent session') - } - if (t.type === 'tcp_client' && (!String(t.host ?? '').trim() || !t.port)) p.push('a host and port') - if ((t.type === 'tcp_server' || t.type === 'udp' || t.type === 'grpc_server') && !String(t.bind ?? '').trim()) p.push('a bind address') - const c = spec.pipeline.codec - if (t.type === 'grpc_client' || t.type === 'grpc_server' || c.type === 'protobuf') { - if (c.type !== 'protobuf') p.push('the Protobuf codec (gRPC carries protobuf)') - else { - if (Object.keys((c.files as Record | undefined) ?? {}).length === 0) p.push("the producer's .proto files") - if (!String(c.message ?? '').trim()) p.push('the message each frame holds') - } - } - if (t.type === 'grpc_client') { - if (!/^https?:\/\/./.test(String(t.url ?? '').trim())) p.push('a URL (http:// or https://)') - if (!String(t.method ?? '').trim()) p.push('the method to call') - } - if (needsAcceptance(spec)) p.push(t.type === 'udp' ? 'the acceptance of unauthenticated senders (UDP)' : 'sender authentication, or its risk accepted') - return p -} - -/** Onboard a feed: connect, probe, map, review. No code. */ -export function AddSourceWizard({ onDone, onCancel }: { onDone: (id: string) => void; onCancel: () => void }) { - const { toast } = useToast() - const [step, setStep] = useState(0) - const [spec, setSpec] = useState({ - id: '', - name: '', - transport: { type: 'http_poll', url: '', interval_secs: 5 }, - pipeline: { codec: { type: 'json' }, mapping: { schema_version: 1, rules: [] } }, - }) - const [probe, setProbe] = useState(null) - const [probing, setProbing] = useState(false) - const [maxFrames, setMaxFrames] = useState(50) - const [maxSecs, setMaxSecs] = useState(20) - const [enable, setEnable] = useState(true) - const [saving, setSaving] = useState(false) - const [saveError, setSaveError] = useState(null) - - const problems = connectProblems(spec) - const probed = !!probe && probe.records > 0 - const statuses: StepStatus[] = STEPS.map((_, i) => - i === step ? 'current' : i < step ? (i === 1 && !probed ? 'error' : 'complete') : 'upcoming', - ) - - const runProbe = async () => { - setProbing(true) - try { - const r = await api.probe({ - transport: spec.transport, - codec: spec.pipeline.codec, - max_frames: maxFrames, - max_secs: maxSecs, - save_as: spec.id, - }) - setProbe(r) - // Adopt the detected codec settings and the suggested mapping. - setSpec((s) => ({ ...s, pipeline: { ...s.pipeline, codec: r.codec, mapping: r.suggestion.mapping } })) - if (r.frames === 0) { - toast({ variant: 'warning', message: r.link_error ?? r.hint ?? 'No frames arrived within the probe window.' }) - } - } catch (e) { - toast({ variant: 'error', title: 'Probe failed', message: errorMessage(e) }) - } finally { - setProbing(false) - } - } - - // An incomplete source is saved as a draft, which cannot be enabled: once - // it exists, Save updates it instead of creating it again. - const [created, setCreated] = useState(false) - - const save = async () => { - setSaving(true) - setSaveError(null) - try { - if (created) { - await api.saveSource(spec) - } else { - await api.createSource(spec) - setCreated(true) - } - if (enable) await api.setEnabled(spec.id, true) - toast({ variant: 'success', message: `Source ${spec.id} saved${enable ? ' and enabled' : ''}.` }) - onDone(spec.id) - } catch (e) { - setSaveError(errorMessage(e)) - } finally { - setSaving(false) - } - } - - const specText = useMemo(() => JSON.stringify(spec, null, 2), [spec]) - const canNext = step === 0 ? problems.length === 0 : step === 1 ? probed : true - - return ( - -
-
- ({ ...s, status: statuses[i] }))} - onStepClick={(id) => setStep(STEPS.findIndex((s) => s.id === id))} - /> - -
- - {step === 0 && ( -
-
-
-
- - - The source's permanent key: up to 64 of a-z, 0-9, - and _. Its tracks, history and samples are filed under it, so it - cannot be changed after saving. - -
- setSpec({ ...spec, id: e.target.value.trim().toLowerCase() })} - autoComplete="off" - spellCheck={false} - /> -
-
-
- - What operators see for this source in lists and track details. You can rename it at any time. -
- setSpec({ ...spec, name: e.target.value })} autoComplete="off" /> -
-
- setSpec({ ...spec, transport })} - onCodec={(codec) => setSpec({ ...spec, pipeline: { ...spec.pipeline, codec } })} - /> - - {problems.length > 0 && Still needed: {problems.join(', ')}.} -
- )} - - {step === 1 && ( -
-
-
-
- - The probe stops after this many frames (messages) arrive, or at Max seconds, whichever comes first. 1 to 2000. -
- setMaxFrames(Number(e.target.value) || 1)} /> -
-
-
- - The longest the probe listens, in seconds, if Max frames has not arrived first. 1 to 120. -
- setMaxSecs(Number(e.target.value) || 1)} /> -
- - - Connects with the settings above, captures frames and infers the schema. Captured frames are kept for - the mapping preview. - -
- {probe && ( - <> -
- 0 ? 'blue' : 'danger'} size="sm"> - frames {probe.frames} - - - records {probe.records} - - {probe.decode_errors > 0 && ( - - decode errors {probe.decode_errors} - - )} - - {probe.seconds}s - - - Frames: messages received. Records: what the codec decoded from them (one frame can hold many; at most 5000 are analysed). - Decode errors: frames the codec could not read. Then the time the probe listened. Up to 200 captured frames are kept for the - mapping preview. - - {probe.suggested_records_path && ( - - records found under {probe.suggested_records_path} - - )} -
- {probe.link_error &&
{probe.link_error}
} - {probe.hint &&
{probe.hint}
} - {probe.last_decode_error &&
{probe.last_decode_error}
} -

- Inferred fields - - Every leaf field seen in the decoded records; [*] means each element of an array. Present: the share of records with a non-null - value. Distinct: different values seen; a + means counting stopped at 1000. - -

- f.path} - maxHeight={300} - defaultSort={{ key: 'presence', direction: 'desc' }} - empty="No fields inferred." - /> - {probe.sample_frames[0] && ( - <> -

First frame

- {probe.sample_frames[0].meta?.topic !== undefined && ( - - topic {String(probe.sample_frames[0].meta.topic)}, available to the - mapping as _frame.topic and{' '} - _frame.topic_levels[n] - - )} - 
{probe.sample_frames[0].text}
- - )} - - )} -
- )} - - {step === 2 && ( - - )} - - {step === 3 && ( -
- - - {saveError &&
{saveError}
} -
- )} - -
- {step > 0 && ( - - )} - {step < STEPS.length - 1 ? ( - - ) : ( - - )} -
-
- - ) -} diff --git a/ui/src/pages/sources/MappingStudio.tsx b/ui/src/pages/sources/MappingStudio.tsx deleted file mode 100644 index 2e08fa8..0000000 --- a/ui/src/pages/sources/MappingStudio.tsx +++ /dev/null @@ -1,88 +0,0 @@ -import { useEffect, useMemo, useRef, useState } from 'react' -import { Badge } from '@kineticlogic/staresdk' -import { CodeEditor } from '@kineticlogic/staresdk/code-editor' -import type { Proposal, SourceSpec } from '../../api/client' -import { usePreview } from '../../lib/usePreview' -import { PreviewResults } from './PreviewResults' - -interface Props { - spec: SourceSpec - onChange: (spec: SourceSpec) => void - /** Source id whose stored probe samples the preview replays. */ - sampleSourceId: string - proposals?: Proposal[] - missing?: string[] - /** Show the preview's map (the onboarding wizard does; a source's Pipeline view does not). */ - showMap?: boolean -} - -/** - * Mapping studio: edit the pipeline (codec, mapping, enrich stages) as JSON and see the result - * live against the captured samples. - */ -export function MappingStudio({ spec, onChange, sampleSourceId, proposals, missing, showMap = true }: Props) { - const [text, setText] = useState(() => JSON.stringify(spec.pipeline, null, 2)) - const [parseError, setParseError] = useState(false) - - // Adopt external spec changes (e.g. a new probe) without clobbering typing. - const external = useMemo(() => JSON.stringify(spec.pipeline, null, 2), [spec.pipeline]) - const lastEmitted = useRef(external) - useEffect(() => { - if (external !== lastEmitted.current) { - setText(external) - lastEmitted.current = external - } - }, [external]) - - const edit = (next: string) => { - setText(next) - try { - const pipeline = JSON.parse(next) - setParseError(false) - const updated = { ...spec, pipeline } - lastEmitted.current = JSON.stringify(pipeline, null, 2) - onChange(updated) - } catch { - setParseError(true) - } - } - - const { preview, error: previewError, running } = usePreview(spec, sampleSourceId) - - return ( -
-
- {proposals && proposals.length > 0 && ( -
-

Suggested from the probe

-
    - {proposals.map((p) => ( -
  • - - {Math.round(p.confidence * 100)}% - - - {p.target} ← {p.reason} - -
    • - ))} -
-
- )} - {missing && missing.length > 0 && ( -
Not found in the samples, map these by hand: {missing.join(', ')}
- )} - - {parseError &&
The JSON does not parse yet; the preview shows the last valid version.
} -
-
-
-

Live preview

- {running ? 'running…' : `samples of ${sampleSourceId}`} -
- {previewError &&
{previewError}
} - {preview ? : !previewError && Running the first preview…} -
-
- ) -} diff --git a/ui/src/pages/sources/NewSource.tsx b/ui/src/pages/sources/NewSource.tsx new file mode 100644 index 0000000..e840c8a --- /dev/null +++ b/ui/src/pages/sources/NewSource.tsx @@ -0,0 +1,94 @@ +import { useState } from 'react' +import { Button, Input, Label, Modal, Select } from '@kineticlogic/staresdk' +import { api } from '../../api/client' +import { InfoTip } from '../../components/InfoTip' +import { errorMessage } from '../../lib/format' +import { idFromName, newSourceSpec, TRANSPORTS, validId } from './flow/sourceFlow' + +/** + * Add a source: its name, id and transport. It is saved as a draft with its Source, Decode, Map and + * Output steps, and opens in the editor. + */ +export function NewSource({ onClose, onCreated }: { onClose: () => void; onCreated: (id: string) => void }) { + const [name, setName] = useState('') + const [id, setId] = useState('') + const [idEdited, setIdEdited] = useState(false) + const [transport, setTransport] = useState('http_poll') + const [creating, setCreating] = useState(false) + const [error, setError] = useState(null) + const shownId = idEdited ? id : idFromName(name) + const ready = name.trim() !== '' && validId(shownId) + + const create = async () => { + setCreating(true) + setError(null) + try { + const row = await api.createSource(newSourceSpec(shownId, name.trim(), transport)) + onCreated(row.id) + } catch (e) { + setError(errorMessage(e)) + setCreating(false) + } + } + + return ( + +
{ + e.preventDefault() + if (ready && !creating) void create() + }} + > +
+ + setName(e.target.value)} /> +
+
+
+ + + Names the source's data and prefixes its track ids, so it never changes. Lowercase letters, digits, - and _, at most 64. + +
+ { + setIdEdited(true) + setId(e.target.value) + }} + spellCheck={false} + /> +
+
+ + onTransport(cotFraming(structuredClone(TRANSPORTS[id].initial), codec.type))} - /> -
-
- - onChange({ ...value, records: e.target.value || undefined })} - spellCheck={false} - /> - - - { - const c = words(e.target.value) - onChange({ ...value, context: c.length ? c : undefined }) - }} - spellCheck={false} - /> - - - )} - {type === 'xml' && ( - - onChange({ ...value, record_element: e.target.value })} spellCheck={false} /> - - )} -
- ) -} - /** Records dropped before mapping, each with a reason (counted in the source's metrics). */ export function RejectForm({ rules, onChange }: { rules: Obj[]; onChange: (r: Obj[]) => void }) { return ( diff --git a/ui/src/pages/sources/designer/StageSamples.tsx b/ui/src/pages/sources/designer/StageSamples.tsx index bc4db07..d6179b6 100644 --- a/ui/src/pages/sources/designer/StageSamples.tsx +++ b/ui/src/pages/sources/designer/StageSamples.tsx @@ -7,8 +7,8 @@ import { countsLine, samplesAt } from '../../../lib/trace' * The live preview's samples after one stage: a line of counts and any errors, then each traced * sample as pretty-printed JSON (or what became of it). The list scrolls, not the modal. */ -export function StageSamples({ result, stageId }: { result: PreviewResult; stageId: string }) { - const samples = samplesAt(result.trace, stageId) +export function StageSamples({ result, stageId, path }: { result: PreviewResult; stageId: string; path?: string[] }) { + const samples = samplesAt(result.trace, stageId, path) const errors = [...new Set(result.errors ?? [])] const counts = countsLine(result) return ( diff --git a/ui/src/pages/sources/flow/FeedPanel.tsx b/ui/src/pages/sources/flow/FeedPanel.tsx new file mode 100644 index 0000000..24af2fa --- /dev/null +++ b/ui/src/pages/sources/flow/FeedPanel.tsx @@ -0,0 +1,114 @@ +import { TbX } from 'react-icons/tb' +import { Button, Input, Label } from '@kineticlogic/staresdk' +import type { SourceRow, SourceSpec } from '../../../api/client' +import { InfoTip } from '../../../components/InfoTip' +import { EmitterMotionForm, PublishForm, SecurityForm } from '../designer/StageForms' +import { SourceHistory, SourceStatus } from '../SourceStatus' + +/** + * The feed's own settings, for everything it reports: its name and description, what it reports + * and how correlation takes it, its priority and security label; then its status and history. + */ +export function FeedPanel({ + spec, + row, + tracked, + problems, + readOnly, + onChange, + onClose, +}: { + spec: SourceSpec + row: SourceRow + /** Whether a Tracker comes before the Output. */ + tracked: boolean + problems: string[] + readOnly: boolean + onChange: (s: SourceSpec) => void + onClose: () => void +}) { + const without = (key: K): SourceSpec => { + const next = { ...spec } + delete next[key] + return next + } + return ( +
+
+ Feed +

{spec.name || spec.id}

+ +
+
+ {problems.map((m) => ( +
+ {m} +
+ ))} +
+
+
+ + onChange({ ...spec, name: e.target.value })} /> +
+
+ + +
+
+ + onChange(e.target.value ? { ...spec, description: e.target.value } : without('description'))} + /> +
+
+ onChange(reports ? { ...spec, reports } : without('reports'))} + onPublishAlone={(publish_alone) => onChange({ ...spec, publish_alone })} + onConfirmAfter={(confirm_after) => onChange({ ...spec, confirm_after })} + /> +
+
+ + + Where several sources report one track, priority breaks ties: it picks the position among equally certain reports, the + identity and classification when no report is corroborated by the registry, and each extension field. Higher wins. Default 100. + +
+ + onChange(e.target.value.trim() === '' ? without('priority') : { ...spec, priority: Math.round(Number(e.target.value)) }) + } + /> +
+ onChange({ ...spec, security })} /> + onChange({ ...spec, emitter_motion })} /> +
+

Status

+ +

History

+ +
+
+ ) +} diff --git a/ui/src/pages/sources/flow/NodePanel.tsx b/ui/src/pages/sources/flow/NodePanel.tsx new file mode 100644 index 0000000..0b9c62c --- /dev/null +++ b/ui/src/pages/sources/flow/NodePanel.tsx @@ -0,0 +1,262 @@ +import { TbDownload, TbWand, TbX } from 'react-icons/tb' +import { Button, Input, Label, Select } from '@kineticlogic/staresdk' +import type { PluginInfo, PreviewResult, ProbeResult, SchemaOverview, SourceSpec } from '../../../api/client' +import { InfoTip } from '../../../components/InfoTip' +import { sampleFields } from '../../../lib/conditions' +import { filterInput } from '../../../lib/trace' +import { MapEditor } from '../designer/MapEditor' +import { StageSamples } from '../designer/StageSamples' +import { AffiliationForm, FilterForm, JoinForm, RegistryForm, RejectForm, ThrottleForm, TrackerForm } from '../designer/StageForms' +import { SenderAuth } from '../SenderAuth' +import { TransportForm } from '../TransportForm' +import { codecChoice, codecChoices, KIND_WORD, nodeTitle, TRANSPORTS, withCodec, withTransport, type FlowNode, type StepType } from './sourceFlow' + +type Obj = Record + +/** What each step does, for the ⓘ beside its heading. */ +const STEP_HELP: Record = { + input: 'Where frames come from, and how each becomes records: the transport, how it connects or listens, how a stream is cut into frames, and the codec.', + reject: 'Drops records before mapping, each rule with a reason counted in the source’s metrics.', + map: 'Turns records into observations: the track key, identifiers and OpenTrack fields each rule takes from the record. Identity fields from static rules fill the same track’s later reports.', + filter: 'Keeps or drops observations; dropped ones are counted as filtered.', + registry: 'Finds each track’s entity by its identifiers, grades the match by broadcast name, and at a corroborated grade passes fields between entity and track. Without one before the Output, the defaults run there.', + affiliation: 'Sets affiliation from a country code, by the country lists.', + tracker: 'Forms tracks from detections (plots).', + throttle: 'Limits how often each source track is passed on, by time and distance moved.', + output: 'The source’s system tracks, to correlation. A source has one.', + nats: 'Publishes what reaches it, as JSON, on its NATS subject, without correlation.', +} + +/** + * The settings of one step, and its samples as they leave it. The input's are the source's + * transport and codec; the others' are the step's own. + */ +export function NodePanel({ + spec, + node, + path, + problems, + schema, + plugins, + preview, + previewError, + probe, + capturing, + readOnly, + onSpec, + onConfig, + onCapture, + onClose, +}: { + spec: SourceSpec + node: FlowNode + /** The steps from the input to this one. */ + path: string[] + problems: string[] + schema: SchemaOverview | null + plugins: PluginInfo[] + preview: PreviewResult | null + previewError: string | null + probe: ProbeResult | null + capturing: boolean + readOnly: boolean + onSpec: (spec: SourceSpec) => void + onConfig: (config: unknown) => void + onCapture: () => void + onClose: () => void +}) { + const config = (node.config ?? {}) as Obj + const set = (patch: Obj) => onConfig({ ...config, ...patch }) + + let settings: React.ReactNode = null + switch (node.type) { + case 'input': { + const detected = probe && JSON.stringify(probe.codec) !== JSON.stringify(spec.pipeline.codec) ? probe.codec : null + settings = ( +
+
+
+ + v && onSpec(withCodec(spec, v, plugins))} + /> +
+
+ {detected && ( +
+ + + The captured samples decode with these settings: {JSON.stringify(detected)}. Using them replaces the codec's settings. + +
+ )} + onSpec({ ...spec, transport })} + onCodec={(codec) => onSpec({ ...spec, pipeline: { ...spec.pipeline, codec } })} + /> +
+ ) + break + } + case 'reject': + settings = onConfig(rules)} /> + break + case 'map': { + const mapping = (config.mapping as Obj | undefined) ?? { rules: [] } + const suggested = probe?.suggestion.mapping + const offer = suggested && suggested.rules.length > 0 && JSON.stringify(suggested.rules) !== JSON.stringify(mapping.rules) + const statics = ((mapping.rules as Obj[] | undefined) ?? []).some((r) => r.kind === 'static') + settings = ( + <> + {offer && ( +
+ + + A mapping inferred from the captured samples: {suggested.rules.length} rule(s) + {probe.suggestion.missing.length ? `; not found in the samples: ${probe.suggestion.missing.join(', ')}` : ''}. Using it replaces + this step's rules. + +
+ )} + {schema ? set({ mapping: m })} /> : LOADING…} + {statics && set({ static_join })} />} + + ) + break + } + case 'filter': { + const input = filterInput(preview?.trace, node.id, path[path.length - 2]) + const fields = sampleFields([...input.observations, ...(preview?.observations ?? [])]) + const key = (o: unknown) => String((o as { source_track_key?: unknown }).source_track_key ?? '') + const examples = [...new Set(input.dropped.map(key).filter(Boolean))].slice(0, 3) + settings = + break + } + case 'registry': + settings = + break + case 'affiliation': + settings = + break + case 'tracker': + settings = + break + case 'throttle': + settings = + break + case 'nats': + settings = ( +
+
+
+ + + The NATS subject to publish on: dot-separated tokens, no wildcards or spaces, outside the system tracks' subjects. + +
+ set({ subject: e.target.value })} + /> +
+
+ ) + break + } + + const title = nodeTitle(spec, node) + return ( +
+
+ {KIND_WORD[node.type]} +

+ {title} + {STEP_HELP[node.type]} +

+ +
+
+ {problems.map((m) => ( +
+ {m} +
+ ))} + {settings && ( +
+ {settings} +
+ )} +
+

+ Samples + + A dry run of the source as edited (not yet saved) over its stored samples, as they leave this step. Nothing is published. + +

+ + {node.type === 'input' && ( + <> + + + Connect with these settings and take up to 20 frames or 15 s of the live feed, whichever comes first. They replace the + source's stored samples. + + + )} +
+ {previewError &&
{previewError}
} + {preview && } +
+
+ ) +} diff --git a/ui/src/pages/sources/flow/StepNode.tsx b/ui/src/pages/sources/flow/StepNode.tsx new file mode 100644 index 0000000..cc09fc2 --- /dev/null +++ b/ui/src/pages/sources/flow/StepNode.tsx @@ -0,0 +1,48 @@ +import type { ReactNode } from 'react' +import { Handle, Position, type NodeProps } from '@xyflow/react' +import { gives, KIND_WORD, takes, type StepType } from './sourceFlow' + +/** What the editor gives each step to draw (the canvas node's `data`). */ +export interface StepView extends Record { + type: StepType + icon: ReactNode + title: string + summary: string + problems: string[] +} + +/** + * One step of a source on the workspace: its icon, heading and one line of its settings, a + * connection in unless it is the input and out unless it is an output, and the first thing that + * stops it running in red (its settings list them all). + */ +export function StepNode({ data, selected, isConnectable }: NodeProps) { + const step = data as StepView + const failing = step.problems.length > 0 + return ( +
+ {takes(step.type) && } +
+ + {step.icon} + + {step.title} + {KIND_WORD[step.type]} +
+ {step.summary && {step.summary}} + {failing && ( + + {step.problems[0]} + + )} + {step.problems.length > 1 && +{step.problems.length - 1} more} + {gives(step.type) && } +
+ ) +} diff --git a/ui/src/pages/sources/flow/sourceFlow.test.ts b/ui/src/pages/sources/flow/sourceFlow.test.ts new file mode 100644 index 0000000..38db599 --- /dev/null +++ b/ui/src/pages/sources/flow/sourceFlow.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from 'vitest' +import type { PluginInfo, SourceSpec } from '../../../api/client' +import { + canConnect, + COLUMN, + defaultConfig, + flowOf, + freePlace, + idFromName, + laidOut, + newSourceSpec, + nextNodeId, + nodeTitle, + problemsByNode, + validId, + withCodec, + withFlow, + withoutPositions, + withTransport, + type Flow, +} from './sourceFlow' + +const plugin = (name: string, framing: Record | null = null): PluginInfo => + ({ name, kinds: ['codec'], enabled: true, default_options: { a: 1 }, framing }) as unknown as PluginInfo + +/** input → map → { filter → output, nats }, and a throttle on its own. */ +const branched = (): Flow => ({ + nodes: [ + { id: 'input', type: 'input' }, + { id: 'map', type: 'map', config: defaultConfig('map') }, + { id: 'filter', type: 'filter', config: {} }, + { id: 'output', type: 'output' }, + { id: 'nats', type: 'nats', config: { subject: 'raw' } }, + { id: 'throttle', type: 'throttle', config: defaultConfig('throttle') }, + ], + edges: [ + { source: 'input', target: 'map' }, + { source: 'map', target: 'filter' }, + { source: 'filter', target: 'output' }, + { source: 'map', target: 'nats' }, + ], +}) + +describe('a new source', () => { + it('has its input, an empty Map and the Output, connected and placed', () => { + const s = newSourceSpec('ais', 'AIS', 'websocket') + const f = flowOf(s) + expect(f.nodes.map((n) => n.type)).toEqual(['input', 'map', 'output']) + expect(f.edges).toEqual([ + { source: 'input', target: 'map' }, + { source: 'map', target: 'output' }, + ]) + expect(f.nodes.every((n) => n.position)).toBe(true) + expect(s.transport).toEqual({ type: 'websocket', url: '' }) + expect(nodeTitle(s, f.nodes[0])).toBe('WebSocket') + }) + + it('takes its id from its name', () => { + expect(idFromName('adsb.lol feed (EU)')).toBe('adsb-lol-feed-eu') + expect(validId(idFromName('adsb.lol feed (EU)'))).toBe(true) + expect(validId('Bad Id')).toBe(false) + }) +}) + +describe('canConnect', () => { + const f = branched() + + it('lets a step that passes observations on feed one that takes them', () => { + expect(canConnect(f, 'filter', 'throttle')).toBe(true) + expect(canConnect(f, 'map', 'throttle')).toBe(true) + }) + + it('refuses the wrong data, a second input, a loop, and anything out of an output', () => { + expect(canConnect(f, 'input', 'throttle')).toBe(false) // records into an observation step + expect(canConnect(f, 'filter', 'map')).toBe(false) // observations into a Map + expect(canConnect(f, 'map', 'output')).toBe(false) // the Output has its input + expect(canConnect(f, 'output', 'throttle')).toBe(false) // an output passes nothing on + expect(canConnect(f, 'throttle', 'throttle')).toBe(false) + const chain: Flow = { ...f, edges: f.edges.filter((e) => e.target !== 'filter').concat({ source: 'throttle', target: 'filter' }) } + // filter feeds output; throttle feeds filter: output → … → throttle would loop, filter → throttle too. + expect(canConnect(chain, 'filter', 'throttle')).toBe(false) + }) +}) + +describe('steps', () => { + it('are numbered by type when taken', () => { + expect(nextNodeId('filter', ['input', 'map'])).toBe('filter') + expect(nextNodeId('filter', ['filter', 'filter-2'])).toBe('filter-3') + }) + + it('without places are laid out in columns from the input', () => { + const f = laidOut(branched()) + const at = (id: string) => f.nodes.find((n) => n.id === id)!.position! + expect(at('input')).toEqual({ x: 0, y: 0 }) + expect(at('map').x).toBe(COLUMN) + expect(at('filter').x).toBe(2 * COLUMN) + expect(at('nats').x).toBe(2 * COLUMN) + expect(at('filter').y).not.toBe(at('nats').y) + expect(at('throttle').x).toBe(4 * COLUMN) // not connected: after the rest + }) + + it('are placed clear of the others', () => { + const f = laidOut(branched()) + expect(freePlace(f, { x: 0, y: 0 })).toEqual({ x: 0, y: 130 }) + expect(freePlace(f, { x: 0.4, y: 500 })).toEqual({ x: 0, y: 500 }) + }) + + it('are dry-run without their places', () => { + const s = withFlow(newSourceSpec('x', 'X', 'udp'), laidOut(branched())) + expect(flowOf(withoutPositions(s)).nodes.some((n) => n.position)).toBe(false) + }) +}) + +describe('the input', () => { + const base = newSourceSpec('x', 'X', 'tcp_client') + + it('swaps its transport, and the acceptance that belonged to the old one goes', () => { + const accepted: SourceSpec = { ...withTransport(base, 'udp'), unauthenticated: 'accepted' } + const mqtt = withTransport(accepted, 'mqtt') + expect(mqtt.transport).toEqual({ type: 'mqtt', url: '', topics: [] }) + expect(mqtt.unauthenticated).toBeUndefined() + expect(withTransport(accepted, 'udp')).toBe(accepted) + }) + + it('frames CoT on a stream by its end tag, and takes it away again', () => { + const cot = withCodec(base, 'cot_xml', []) + expect(cot.pipeline.codec).toEqual({ type: 'cot_xml' }) + expect(cot.transport.framing).toEqual({ type: 'end_tag', tag: '' }) + expect(withCodec(cot, 'json', []).transport.framing).toEqual({ type: 'lines' }) + const custom = { ...cot, transport: { ...cot.transport, framing: { type: 'delimiter', delimiter: '\\x03' } } } + expect(withCodec(custom, 'json', []).transport.framing).toEqual(custom.transport.framing) + }) + + it('takes a codec plugin with its options and framing', () => { + const p = plugin('stanag4607', { type: 'length_field', offset: 2, width: 'u32' }) + const s = withCodec(base, 'plugin:stanag4607', [p]) + expect(s.pipeline.codec).toEqual({ type: 'plugin', plugin: 'stanag4607', options: { a: 1 } }) + expect(s.transport.framing).toEqual({ type: 'length_field', offset: 2, width: 'u32' }) + expect(withCodec(s, 'json', [p]).transport.framing).toEqual({ type: 'lines' }) + }) +}) + +describe('problemsByNode', () => { + it('puts each problem on its step, the rest with the feed', () => { + const p = problemsByNode( + [ + { kind: 'incomplete', node: 'input', field: 'host', message: 'the host is not set' }, + { kind: 'incomplete', node: 'map', message: 'mapping has no rules' }, + { kind: 'incomplete', node: 'gone', message: 'not shown' }, + { kind: 'invalid', node: null, field: 'name', message: 'source name must not be empty' }, + ], + branched(), + ) + expect(p.byNode).toEqual({ input: ['the host is not set'], map: ['mapping has no rules'] }) + expect(p.general).toEqual(['not shown', 'source name must not be empty']) + }) +}) diff --git a/ui/src/pages/sources/flow/sourceFlow.ts b/ui/src/pages/sources/flow/sourceFlow.ts new file mode 100644 index 0000000..2c33055 --- /dev/null +++ b/ui/src/pages/sources/flow/sourceFlow.ts @@ -0,0 +1,369 @@ +/** + * A source as a flow of steps (#104), as the server keeps it (crates/ot-source/src/flow.rs): one + * input (the transport and its codec), then steps the user places and connects, each taking one + * input and feeding any number of others, ending at the source's one Output (system tracks, to + * correlation) and any NATS topics (its mapped data, without correlation). + * + * Records flow from the input through Reject steps to a Map, which makes observations; Filter, + * Entity links, Affiliation, Tracker and Throttle steps take observations; Output and NATS topics + * take them and pass nothing on. + */ +import type { PluginInfo, SourceProblem, SourceSpec } from '../../../api/client' +import { DEFAULT_LINKS } from '../../../lib/pipeline' + +type Obj = Record +type Transport = SourceSpec['transport'] +type Codec = SourceSpec['pipeline']['codec'] + +export const INPUT = 'input' + +export type StepType = 'input' | 'reject' | 'map' | 'filter' | 'registry' | 'affiliation' | 'tracker' | 'throttle' | 'output' | 'nats' + +export interface FlowNode { + id: string + type: StepType + config?: unknown + position?: { x: number; y: number } +} + +export interface FlowEdge { + source: string + target: string +} + +/** A flow's steps and connections, as the spec's `pipeline` holds them. */ +export interface Flow { + nodes: FlowNode[] + edges: FlowEdge[] +} + +type Data = 'records' | 'observations' + +/** What each step takes and passes on (`null`: nothing). */ +const DATA: Record = { + input: { takes: null, gives: 'records' }, + reject: { takes: 'records', gives: 'records' }, + map: { takes: 'records', gives: 'observations' }, + filter: { takes: 'observations', gives: 'observations' }, + registry: { takes: 'observations', gives: 'observations' }, + affiliation: { takes: 'observations', gives: 'observations' }, + tracker: { takes: 'observations', gives: 'observations' }, + throttle: { takes: 'observations', gives: 'observations' }, + output: { takes: 'observations', gives: null }, + nats: { takes: 'observations', gives: null }, +} + +export const takes = (t: StepType) => DATA[t].takes +export const gives = (t: StepType) => DATA[t].gives + +/** Each step's name, and the group the palette lists it under. */ +export const STEPS: Record, { label: string; group: 'Records' | 'Observations' | 'Outputs' }> = { + reject: { label: 'Reject', group: 'Records' }, + map: { label: 'Map', group: 'Records' }, + filter: { label: 'Filter', group: 'Observations' }, + registry: { label: 'Entity links', group: 'Observations' }, + affiliation: { label: 'Affiliation', group: 'Observations' }, + tracker: { label: 'Tracker', group: 'Observations' }, + throttle: { label: 'Throttle', group: 'Observations' }, + output: { label: 'Output', group: 'Outputs' }, + nats: { label: 'NATS topic', group: 'Outputs' }, +} + +/** The word under a step's heading. */ +export const KIND_WORD: Record = { + input: 'Input', + reject: 'Records', + map: 'Map', + filter: 'Observations', + registry: 'Observations', + affiliation: 'Observations', + tracker: 'Observations', + throttle: 'Observations', + output: 'Correlation', + nats: 'NATS', +} + +/** What a new step starts as. */ +export function defaultConfig(type: StepType): unknown { + switch (type) { + case 'reject': + return [{ reason: 'no_position', when: { path: 'lat', exists: false } }] + case 'map': + return { mapping: { rules: [] } } + case 'filter': + return {} + case 'registry': + return { links: DEFAULT_LINKS } + case 'affiliation': + return { country: 'platform.flag', friendly: [], hostile: [], neutral: [] } + case 'tracker': + return { algorithm: 'gnn' } + case 'throttle': + return { min_interval_secs: 5, min_move_m: 0, heartbeat_secs: 600 } + case 'nats': + return { subject: '' } + default: + return undefined + } +} + +/** Transports, as the input's settings name them, with what a new one starts as. */ +export const TRANSPORTS: Record = { + tcp_client: { label: 'TCP client', initial: { type: 'tcp_client', host: '', port: 0, framing: { type: 'lines' } } }, + tcp_server: { label: 'TCP server', initial: { type: 'tcp_server', bind: '0.0.0.0:8087', framing: { type: 'lines' } } }, + udp: { label: 'UDP', initial: { type: 'udp', bind: '0.0.0.0:6969' } }, + http_poll: { label: 'HTTP poll', initial: { type: 'http_poll', url: '', interval_secs: 5 } }, + websocket: { label: 'WebSocket', initial: { type: 'websocket', url: '' } }, + mqtt: { label: 'MQTT', initial: { type: 'mqtt', url: '', topics: [] } }, + grpc_client: { label: 'gRPC client', initial: { type: 'grpc_client', url: '', method: '' } }, + grpc_server: { label: 'gRPC server', initial: { type: 'grpc_server', bind: '0.0.0.0:50051' } }, + file: { label: 'File', initial: { type: 'file', path: '' } }, +} + +/** Codecs, likewise. A plugin codec is offered by its plugin's name. */ +export const CODECS: Record = { + json: { label: 'JSON', initial: { type: 'json' } }, + cot_xml: { label: 'CoT XML', initial: { type: 'cot_xml' } }, + xml: { label: 'XML', initial: { type: 'xml', record_element: '' } }, + protobuf: { label: 'Protobuf', initial: { type: 'protobuf', files: {}, message: '' } }, +} + +/** The codecs the input can use: the built-in ones and each enabled codec plugin, as `plugin:`. */ +export function codecChoices(plugins: PluginInfo[]): { value: string; label: string }[] { + return [ + ...Object.entries(CODECS).map(([value, c]) => ({ value, label: c.label })), + ...plugins.filter((p) => p.kinds.includes('codec') && p.enabled).map((p) => ({ value: `plugin:${p.name}`, label: p.name })), + ] +} + +export const codecChoice = (c: Codec) => (c.type === 'plugin' ? `plugin:${String(c.plugin ?? '')}` : c.type) + +const streamed = (t: Transport) => t.type === 'tcp_client' || t.type === 'tcp_server' +const LINES = { type: 'lines' } +const COT_FRAMING = { type: 'end_tag', tag: '' } +const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b) + +/** CoT arrives as back-to-back XML events, framed by `` rather than by lines on a stream. */ +function cotFraming(t: Transport, codec: Codec): Transport { + const framing = t.framing as { type?: string } | undefined + return codec.type === 'cot_xml' && streamed(t) && framing?.type === 'lines' ? { ...t, framing: COT_FRAMING } : t +} + +/** The framing a codec sets on a stream when chosen (CoT's end tag, a plugin's own), if any. */ +function framingOf(codec: Codec, plugins: PluginInfo[]): unknown { + if (codec.type === 'cot_xml') return COT_FRAMING + if (codec.type === 'plugin') return plugins.find((p) => p.name === codec.plugin)?.framing ?? null + return null +} + +/** The source with another transport: its defaults, framed for the codec; the risk acceptance goes with the old one. */ +export function withTransport(spec: SourceSpec, type: string): SourceSpec { + const d = TRANSPORTS[type] + if (!d || type === spec.transport.type) return spec + const next: SourceSpec = { ...spec, transport: cotFraming(structuredClone(d.initial), spec.pipeline.codec) } + delete next.unauthenticated + return next +} + +/** The source with another codec (`plugin:` for a plugin's): the framing the old one set goes, the new one's comes. */ +export function withCodec(spec: SourceSpec, choice: string, plugins: PluginInfo[]): SourceSpec { + if (choice === codecChoice(spec.pipeline.codec)) return spec + let codec: Codec | null = null + let transport = spec.transport + const before = framingOf(spec.pipeline.codec, plugins) + if (before && streamed(transport) && same(transport.framing, before)) transport = { ...transport, framing: LINES } + if (choice.startsWith('plugin:')) { + const p = plugins.find((x) => x.name === choice.slice('plugin:'.length)) + if (p) { + codec = { type: 'plugin', plugin: p.name, options: structuredClone(p.default_options) } + if (p.framing && streamed(transport)) transport = { ...transport, framing: structuredClone(p.framing) } + } + } else if (CODECS[choice]) { + codec = structuredClone(CODECS[choice].initial) + } + if (!codec) return spec + return { ...spec, transport: cotFraming(transport, codec), pipeline: { ...spec.pipeline, codec } } +} + +/** The source's flow. */ +export const flowOf = (spec: SourceSpec): Flow => ({ + nodes: (spec.pipeline.nodes as FlowNode[] | undefined) ?? [], + edges: (spec.pipeline.edges as FlowEdge[] | undefined) ?? [], +}) + +/** The source with this flow. */ +export const withFlow = (spec: SourceSpec, flow: Flow): SourceSpec => ({ ...spec, pipeline: { ...spec.pipeline, nodes: flow.nodes, edges: flow.edges } }) + +/** A new step's id: its type, numbered when taken (`filter`, `filter-2`…). */ +export function nextNodeId(type: StepType, ids: string[]): string { + if (!ids.includes(type)) return type + let n = 2 + while (ids.includes(`${type}-${n}`)) n++ + return `${type}-${n}` +} + +/** Whether `source` may feed `target`: one input each, no loops, and the right data. */ +export function canConnect(flow: Flow, source: string, target: string): boolean { + if (source === target) return false + const from = flow.nodes.find((n) => n.id === source) + const to = flow.nodes.find((n) => n.id === target) + if (!from || !to) return false + const out = gives(from.type) + if (!out || takes(to.type) !== out) return false + if (flow.edges.some((e) => e.target === target)) return false + // No loop: `target` must not already lead to `source`. + const children = new Map() + for (const e of flow.edges) children.set(e.source, [...(children.get(e.source) ?? []), e.target]) + const stack = [target] + const seen = new Set() + while (stack.length) { + const at = stack.pop()! + if (at === source) return false + if (seen.has(at)) continue + seen.add(at) + stack.push(...(children.get(at) ?? [])) + } + return true +} + +/** Where steps go that have no place yet (a source from before flows): columns by distance from the input. */ +export const COLUMN = 300 +export const ROW = 130 + +export function laidOut(flow: Flow): Flow { + if (flow.nodes.every((n) => n.position)) return flow + const children = new Map() + for (const e of flow.edges) children.set(e.source, [...(children.get(e.source) ?? []), e.target]) + const depth = new Map([[INPUT, 0]]) + const queue = [INPUT] + while (queue.length) { + const at = queue.shift()! + for (const c of children.get(at) ?? []) { + if (!depth.has(c)) { + depth.set(c, depth.get(at)! + 1) + queue.push(c) + } + } + } + const rows = new Map() + const loose = Math.max(0, ...depth.values()) + 1 + return { + ...flow, + nodes: flow.nodes.map((n) => { + if (n.position) return n + const col = depth.get(n.id) ?? loose + const row = rows.get(col) ?? 0 + rows.set(col, row + 1) + return { ...n, position: { x: col * COLUMN, y: row * ROW } } + }), + } +} + +/** Where a new step goes: `at`, or below it as far as needed not to cover another step. */ +export function freePlace(flow: Flow, at: { x: number; y: number }): { x: number; y: number } { + const taken = (p: { x: number; y: number }) => + flow.nodes.some((n) => n.position && Math.abs(n.position.x - p.x) < 240 && Math.abs(n.position.y - p.y) < 110) + let p = { x: Math.round(at.x), y: Math.round(at.y) } + for (let i = 0; i < 50 && taken(p); i++) p = { ...p, y: p.y + ROW } + return p +} + +export interface NodeProblems { + byNode: Record + /** Problems with the source as a whole (its feed settings), or with a step it no longer has. */ + general: string[] +} + +/** The server's problems, each on its step. */ +export function problemsByNode(problems: SourceProblem[], flow: Flow): NodeProblems { + const ids = new Set(flow.nodes.map((n) => n.id)) + const out: NodeProblems = { byNode: {}, general: [] } + for (const p of problems) { + if (p.node && ids.has(p.node)) (out.byNode[p.node] ??= []).push(p.message) + else out.general.push(p.message) + } + return out +} + +/** A step's heading. */ +export function nodeTitle(spec: SourceSpec, n: FlowNode): string { + if (n.type === 'input') return TRANSPORTS[spec.transport.type]?.label ?? spec.transport.type + if (n.type === 'nats') return String((n.config as Obj | undefined)?.subject || 'NATS topic') + return STEPS[n.type].label +} + +/** One line of a step's settings. */ +export function nodeSummary(spec: SourceSpec, n: FlowNode): string { + const c = (n.config ?? {}) as Obj + switch (n.type) { + case 'input': { + const codec = spec.pipeline.codec + const t = spec.transport + const where = String(t.url ?? t.bind ?? t.path ?? (t.host ? `${t.host}:${t.port}` : '')) + const decode = codec.type === 'plugin' ? String(codec.plugin ?? 'plugin') : (CODECS[codec.type]?.label ?? codec.type) + return [where, decode].filter(Boolean).join(' · ') + } + case 'reject': + return plural(((n.config as unknown[]) ?? []).length, 'rule') + case 'map': + return plural((((c.mapping as Obj | undefined)?.rules as unknown[]) ?? []).length, 'rule') + case 'filter': + return c.keep_if ? 'keep only matching' : c.drop_if ? 'drop matching' : 'keeps everything' + case 'tracker': + return String(c.algorithm ?? 'gnn').toUpperCase() + case 'throttle': + return `at most every ${String(c.min_interval_secs ?? 0)} s` + case 'output': + return spec.reports === 'detections' ? 'detections' : 'tracks' + default: + return '' + } +} + +const plural = (n: number, one: string) => `${n} ${n === 1 ? one : `${one}s`}` + +/** A new source: its transport, a JSON input, an empty Map and the Output, connected. */ +export function newSourceSpec(id: string, name: string, transport: string): SourceSpec { + const flow: Flow = { + nodes: [ + { id: INPUT, type: 'input', position: { x: 0, y: 0 } }, + { id: 'map', type: 'map', config: { mapping: { rules: [] } }, position: { x: COLUMN, y: 0 } }, + { id: 'output', type: 'output', position: { x: 2 * COLUMN, y: 0 } }, + ], + edges: [ + { source: INPUT, target: 'map' }, + { source: 'map', target: 'output' }, + ], + } + return { + id, + name, + transport: structuredClone(TRANSPORTS[transport]?.initial ?? TRANSPORTS.http_poll.initial), + pipeline: { codec: { type: 'json' }, nodes: flow.nodes, edges: flow.edges }, + } +} + +/** A source id from its name: lowercase letters, digits, `-` and `_`, at most 64 (crates/ot-source/src/source.rs). */ +export function idFromName(name: string): string { + return name + .toLowerCase() + .normalize('NFKD') + .replace(/[^a-z0-9_-]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 64) +} + +export const validId = (id: string) => /^[a-z0-9_-]{1,64}$/.test(id) + +/** The spec without where its steps are drawn: what a dry run depends on. */ +export function withoutPositions(spec: SourceSpec): SourceSpec { + const flow = flowOf(spec) + return withFlow(spec, { + ...flow, + nodes: flow.nodes.map((n) => { + const copy = { ...n } + delete copy.position + return copy + }), + }) +} diff --git a/ui/src/pages/sources/flow/stepIcons.tsx b/ui/src/pages/sources/flow/stepIcons.tsx new file mode 100644 index 0000000..2ccf174 --- /dev/null +++ b/ui/src/pages/sources/flow/stepIcons.tsx @@ -0,0 +1,52 @@ +import type { ReactNode } from 'react' +import { + TbAddressBook, + TbApi, + TbArrowsExchange, + TbBan, + TbBroadcast, + TbClockPause, + TbFile, + TbFilter, + TbFlag, + TbMessages, + TbNetwork, + TbPlugConnected, + TbRadar, + TbRoute, + TbSend, + TbServer, + TbTopologyStar, + TbWorldDownload, +} from 'react-icons/tb' +import type { SourceSpec } from '../../../api/client' +import type { StepType } from './sourceFlow' + +const TRANSPORT_ICONS: Record = { + tcp_client: , + tcp_server: , + udp: , + http_poll: , + websocket: , + mqtt: , + grpc_client: , + grpc_server: , + file: , +} + +const STEP_ICONS: Record, ReactNode> = { + reject: , + map: , + filter: , + registry: , + affiliation: , + tracker: , + throttle: , + output: , + nats: , +} + +/** A step's icon: the input's is its transport's. */ +export function stepIcon(spec: SourceSpec, type: StepType): ReactNode { + return type === 'input' ? TRANSPORT_ICONS[spec.transport.type] : STEP_ICONS[type] +}