From b2b13658e06daae7613463e0081b14b61f3d2c5b Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Wed, 21 Oct 2026 14:11:57 -0500 Subject: [PATCH 01/16] UI: stareSDK 0.2.26 (#104) FlowCanvas with palette groups for the Sources flow editor. Every release since 0.2.1 is additive. Co-Authored-By: Claude Opus 5.5 --- ui/package-lock.json | 16 ++++++++++++---- ui/package.json | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index c11f479..777e394 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -21,7 +21,7 @@ "@dnd-kit/utilities": "^3.2.2", "@fontsource/montserrat": "^5.3.0", "@fontsource/oswald": "^5.3.0", - "@kineticlogic/staresdk": "^0.2.1", + "@kineticlogic/staresdk": "^0.2.26", "@lezer/highlight": "^1.2.4", "@xyflow/react": "^12.12.0", "framer-motion": "^12.43.0", @@ -210,15 +210,17 @@ "license": "MIT" }, "node_modules/@kineticlogic/staresdk": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@kineticlogic/staresdk/-/staresdk-0.2.1.tgz", - "integrity": "sha512-NrZ5wPlmq6xa0WMNHnmlbzmpeSq5jRb/D5TCDL7A+Jlv6QPr4K4B+wMNkgtUlWn5j690Eubavre/+W+DD6pL5g==", + "version": "0.2.26", + "resolved": "https://registry.npmjs.org/@kineticlogic/staresdk/-/staresdk-0.2.26.tgz", + "integrity": "sha512-iPw0k72j+D7tAb0ZV2nsIxjYWWMZBSS+a4k28wNgSO/+6X8oKkSM/Bm9GdrnAaPFMs0InunBLA0rmcZSlOfh6A==", "license": "MIT", "engines": { "node": ">=20" }, "peerDependencies": { + "@codemirror/autocomplete": "^6.20.0", "@codemirror/commands": "^6.11.1", + "@codemirror/lang-html": "^6.4.11", "@codemirror/lang-json": "^6.0.2", "@codemirror/language": "^6.12.4", "@codemirror/lint": "^6.9.7", @@ -242,9 +244,15 @@ "uplot": "^1.6.32" }, "peerDependenciesMeta": { + "@codemirror/autocomplete": { + "optional": true + }, "@codemirror/commands": { "optional": true }, + "@codemirror/lang-html": { + "optional": true + }, "@codemirror/lang-json": { "optional": true }, diff --git a/ui/package.json b/ui/package.json index 4e41e2a..877081c 100644 --- a/ui/package.json +++ b/ui/package.json @@ -24,7 +24,7 @@ "@dnd-kit/utilities": "^3.2.2", "@fontsource/montserrat": "^5.3.0", "@fontsource/oswald": "^5.3.0", - "@kineticlogic/staresdk": "^0.2.1", + "@kineticlogic/staresdk": "^0.2.26", "@lezer/highlight": "^1.2.4", "@xyflow/react": "^12.12.0", "framer-motion": "^12.43.0", From 511e6ac3c84f163d2064299137137b4a602c4f2d Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Wed, 21 Oct 2026 14:14:41 -0500 Subject: [PATCH 02/16] UI: add CodeEditor's codemirror peers for stareSDK 0.2.26 (#104) Since stareSDK 0.2.10 CodeEditor imports @codemirror/autocomplete and @codemirror/lang-html, so the production build fails without them. Co-Authored-By: Claude Opus 5.5 --- ui/package-lock.json | 92 ++++++++++++++++++++++++++++++++++++++++++++ ui/package.json | 2 + 2 files changed, 94 insertions(+) diff --git a/ui/package-lock.json b/ui/package-lock.json index 777e394..08d4ba2 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -9,7 +9,9 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "@codemirror/autocomplete": "^6.20.4", "@codemirror/commands": "^6.11.1", + "@codemirror/lang-html": "^6.4.12", "@codemirror/lang-json": "^6.0.2", "@codemirror/language": "^6.12.4", "@codemirror/lint": "^6.9.7", @@ -48,6 +50,18 @@ "vitest": "^4.1.11" } }, + "node_modules/@codemirror/autocomplete": { + "version": "6.20.4", + "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.20.4.tgz", + "integrity": "sha512-yF8RhOf2SXlg00+qtupzLhrglcas61boqaONiNcbPIebHCpJocpqTBA6dyrtUrQO47hqyTdeob/3XnYilT7TaQ==", + "license": "MIT", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.17.0", + "@lezer/common": "^1.0.0" + } + }, "node_modules/@codemirror/commands": { "version": "6.11.1", "resolved": "https://registry.npmjs.org/@codemirror/commands/-/commands-6.11.1.tgz", @@ -60,6 +74,51 @@ "@lezer/common": "^1.1.0" } }, + "node_modules/@codemirror/lang-css": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@codemirror/lang-css/-/lang-css-6.3.1.tgz", + "integrity": "sha512-kr5fwBGiGtmz6l0LSJIbno9QrifNMUusivHbnA1H6Dmqy4HZFte3UAICix1VuKo0lMPKQr2rqB+0BkKi/S3Ejg==", + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@lezer/common": "^1.0.2", + "@lezer/css": "^1.1.7" + } + }, + "node_modules/@codemirror/lang-html": { + "version": "6.4.12", + "resolved": "https://registry.npmjs.org/@codemirror/lang-html/-/lang-html-6.4.12.tgz", + "integrity": "sha512-pw2ReWKUqSkbvh76RAT4NYxiogRu+PWkR2ukAwO9uOgrm8uipkzjtKKtNpyeAQwHOqxEeSvAXZ6vr3AfyB9y/w==", + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/lang-css": "^6.0.0", + "@codemirror/lang-javascript": "^6.0.0", + "@codemirror/language": "^6.4.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.17.0", + "@lezer/common": "^1.0.0", + "@lezer/css": "^1.1.0", + "@lezer/html": "^1.3.12" + } + }, + "node_modules/@codemirror/lang-javascript": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/@codemirror/lang-javascript/-/lang-javascript-6.2.5.tgz", + "integrity": "sha512-zD4e5mS+50htS7F+TYjBPsiIFGanfVqg4HyUz6WNFikgOPf2BgKlx+TQedI1w6n/IqRBVBbBWmGFdLB/7uxO4A==", + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/language": "^6.6.0", + "@codemirror/lint": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.17.0", + "@lezer/common": "^1.0.0", + "@lezer/javascript": "^1.0.0" + } + }, "node_modules/@codemirror/lang-json": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@codemirror/lang-json/-/lang-json-6.0.2.tgz", @@ -294,6 +353,17 @@ "integrity": "sha512-H0iErY4e43LpXbYDyBci5W4v/RwTgGV3YzYOlJPiGZ8RY8w51kVE+xLn1tg3Z6UmSfyETvtsUK3r7YsgDQEI7Q==", "license": "MIT" }, + "node_modules/@lezer/css": { + "version": "1.3.9", + "resolved": "https://registry.npmjs.org/@lezer/css/-/css-1.3.9.tgz", + "integrity": "sha512-ZmzgvTOBdJWGIsY/ItAfN7UW8DV802oZG+jeG6E508SwEzz5YhuAajptpWPICvIyqYAD2+VjMdOS40GgGkyEOQ==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.3.0" + } + }, "node_modules/@lezer/highlight": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.2.4.tgz", @@ -303,6 +373,28 @@ "@lezer/common": "^1.3.0" } }, + "node_modules/@lezer/html": { + "version": "1.3.13", + "resolved": "https://registry.npmjs.org/@lezer/html/-/html-1.3.13.tgz", + "integrity": "sha512-oI7n6NJml729m7pjm9lvLvmXbdoMoi2f+1pwSDJkl9d68zGr7a9Btz8NdHTGQZtW2DA25ybeuv/SyDb9D5tseg==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0" + } + }, + "node_modules/@lezer/javascript": { + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/@lezer/javascript/-/javascript-1.5.6.tgz", + "integrity": "sha512-sEvOqpl4vJQca0NugSQK02vQPbc5KrvoojnYGoqqFgFO0MtcYhwGiHKVEB+LuXYJnfyXo4QPA8ztbrEa4Fw5NA==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.1.3", + "@lezer/lr": "^1.3.0" + } + }, "node_modules/@lezer/json": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@lezer/json/-/json-1.0.3.tgz", diff --git a/ui/package.json b/ui/package.json index 877081c..2f41e33 100644 --- a/ui/package.json +++ b/ui/package.json @@ -12,7 +12,9 @@ "test": "vitest run" }, "dependencies": { + "@codemirror/autocomplete": "^6.20.4", "@codemirror/commands": "^6.11.1", + "@codemirror/lang-html": "^6.4.12", "@codemirror/lang-json": "^6.0.2", "@codemirror/language": "^6.12.4", "@codemirror/lint": "^6.9.7", From 008b1a09d6af5c71d3922973b0a4a45467e4839f Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Wed, 21 Oct 2026 15:20:17 -0500 Subject: [PATCH 03/16] Dependencies: clear the open RustSec advisories cargo deny reported 28 advisories against main: - rustls 0.23.46: RUSTSEC-2026-0346. - rustls-webpki 0.103.16, webpki-roots and webpki-root-certs 1.0.10: distrusted emSign intermediates (RUSTSEC-2026-0381, -0382, -0383). - wasmtime and wasmtime-wasi 49.0.4: RUSTSEC-2026-0321 to -0327, -0352 to -0364, -0378, -0379. - zip 9 (RUSTSEC-2026-0367 to -0369), through calamine 0.37 and rust_xlsxwriter 0.101. - yoke-derive 0.8.4 replaces a yanked release. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 212 +++++++++++++++++++++++++++-------------------------- Cargo.toml | 4 +- 2 files changed, 110 insertions(+), 106 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ece49da..e53b143 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -480,9 +480,9 @@ dependencies = [ [[package]] name = "calamine" -version = "0.36.1" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" +checksum = "c9ca341b94bc4b29e08970b1a03914fad2dfe177e2ecb67fcd6de9046004aafb" dependencies = [ "atoi_simd", "byteorder", @@ -490,7 +490,7 @@ dependencies = [ "encoding_rs", "fast-float2", "log", - "quick-xml 0.41.0", + "quick-xml", "serde", "zip", ] @@ -749,27 +749,27 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.136.1" +version = "0.136.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8a495ae3e4d638712a0c0378719e526e448d04d7ebbbe1745712e12c5feb4c3" +checksum = "c9c587da2696fbaa58e7a2fa2e5bfdd365d0928219f36992304ac1f15e5bc254" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.136.1" +version = "0.136.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb2e299e844a27d799f43988e4415a37696fdb738cb463312ef5d06cae15b2a" +checksum = "f28a60027b7556b680b051651291053ec381edf4ffbc01784997ae51a64269bb" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.136.1" +version = "0.136.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4710c34489bc844e45c1c0005d3e44d3d2b29cee77187507ba8d0bb400c621f" +checksum = "62bfdb846acebfd591d601994ea283c98d525ac0bc9a84ca3f2c36bcf5b69ff1" dependencies = [ "cranelift-entity", "wasmtime-internal-core", @@ -777,9 +777,9 @@ dependencies = [ [[package]] name = "cranelift-bitset" -version = "0.136.1" +version = "0.136.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76b63a4b0cfea80610150fffda11b1a9c3dc5a835403d9f381bf7d1a2874b4d7" +checksum = "aa9f8cc1080739d5f72b4ee7e00aff7d589f1284beaf0a19472c07878be7f299" dependencies = [ "serde", "serde_derive", @@ -788,9 +788,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.136.1" +version = "0.136.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1af881f3b3392433e6f4f9b172b8d99270c538db4effea62b28bc48f02f0833" +checksum = "d85417d493ecec833b5134546b4a252c78cc0bd5be11729a35d25dcff9e4e70b" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -819,9 +819,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.136.1" +version = "0.136.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24ce99c30dc7a692294613241119eea0de3c3cb4cf10ac2570ed72adf9a8c2eb" +checksum = "d01c8ed0098360bae69ad6b82952d4247516799ce8050d45ad168d6689f4eae6" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -832,24 +832,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.136.1" +version = "0.136.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34c2a529b638d95c347c2337403dacec86197ad6503cedc63caf53466ae5c4de" +checksum = "04f9b86d809cec9c4e88b201f1b3ed3b0347674c9ccfbb8c355228669e34be81" [[package]] name = "cranelift-control" -version = "0.136.1" +version = "0.136.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "570eb4e02cca190896beb3c8a0a67583dca8b6965517a9f3e04cc89e17c9a986" +checksum = "ce182038fb10b719841e65fdf5deeee258c684f63439f5be680c2be059274b58" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.136.1" +version = "0.136.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e73c65538d460b1062cb9a97428ee6bb4b616f5cb17be3b3ac0a3e4f1f789166" +checksum = "7bca812b03a7b942796391f25bc02aa70a1a007fee3ce2756490c788429e9c39" dependencies = [ "cranelift-bitset", "serde", @@ -859,9 +859,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.136.1" +version = "0.136.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ed70ab5849184973c42b57366f57611dc81c3e355d5b612c68a2900bea92242" +checksum = "a0c30563d2aef95e8d1a00a920199c99117b350339599f0e9487a7a1cf5d59de" dependencies = [ "cranelift-codegen", "hashbrown 0.17.1", @@ -872,15 +872,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.136.1" +version = "0.136.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379f10a451c9b1bf4f59e4ce38061c53ca2454783041a3c1fafa6deef3c37108" +checksum = "9c54184333a275698edb1d3bdff226131a25d47133d792aa716e47e32b8b7f05" [[package]] name = "cranelift-native" -version = "0.136.1" +version = "0.136.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f496865b1a14bc85bcc9fb51a9539ca93b1d1f44fb92bd6a091fba02d6ab4a7" +checksum = "eb67a4c8a3180b5f1d77dffcd6317c05386ced42e5a31c7004523a9d99dc85c1" dependencies = [ "cranelift-codegen", "libc", @@ -889,9 +889,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.136.1" +version = "0.136.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89643801d9192df80c294f174d5fe0416395e7edb0bb3024784ba10fc933e76f" +checksum = "b533db1679b4b728568bd490da6533cf50974c48a4d7534e78f1e0e02bc39769" [[package]] name = "crc32fast" @@ -2311,6 +2311,18 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nkeys" version = "0.4.5" @@ -2664,7 +2676,7 @@ dependencies = [ "ot-source", "ot-store", "ot-sync", - "quick-xml 0.42.0", + "quick-xml", "rcgen", "reqwest", "rumqttc", @@ -2702,14 +2714,14 @@ dependencies = [ "hyper", "hyper-util", "miette", - "nix", + "nix 0.30.1", "ot-codec-stanag4607", "ot-core", "ot-sapient", "prost", "prost-reflect", "protox", - "quick-xml 0.42.0", + "quick-xml", "rcgen", "regex", "reqwest", @@ -3015,9 +3027,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0091857bc410bd7117eb6ac042ced7714c69f994a8b9c2ae30add2f73e3a25ac" +checksum = "5cd3de43f4f4cff536de8902097edbdf749321b97dda548f36a37918eb85bba3" dependencies = [ "cranelift-bitset", "log", @@ -3027,31 +3039,22 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8798ebc6bf0ef227e204fa3fad05b04970599a85f456fd28365450fd679b2600" +checksum = "8319a8fa5269b4895957f8795861c42822bd26218e495f70060a963323d94847" dependencies = [ "proc-macro2", "quote", "syn 2.0.119", ] -[[package]] -name = "quick-xml" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" -dependencies = [ - "encoding_rs", - "memchr", -] - [[package]] name = "quick-xml" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" dependencies = [ + "encoding_rs", "memchr", "serde", ] @@ -3446,9 +3449,9 @@ dependencies = [ [[package]] name = "rust_xlsxwriter" -version = "0.99.1" +version = "0.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34f2d2cda7223168a14fb86bc7411dbc3379bd7d49749ecc53b91ed98585b6b0" +checksum = "49aa1d1558e9465f5053bf9fb5361fe75db827c6a4423d5e5fe72096622f8908" dependencies = [ "zip", ] @@ -3508,9 +3511,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", @@ -3581,9 +3584,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -3620,7 +3623,7 @@ dependencies = [ "openssl-probe 0.1.6", "openssl-sys", "pkg-config", - "quick-xml 0.42.0", + "quick-xml", "rand 0.9.5", "serde", "thiserror 2.0.21", @@ -4753,9 +4756,9 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.258.0" +version = "0.258.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e974fe6821a8cf64575d51ea2194e2c8f77e7b66e9afe7419ce8a97f9ee0d251" +checksum = "f64765e43c000fae008a48db8b288e175e237fc9ce20a26586836878c1bee89f" dependencies = [ "leb128fmt", "wasmparser", @@ -4763,9 +4766,9 @@ dependencies = [ [[package]] name = "wasm-metadata" -version = "0.258.0" +version = "0.258.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a11585adb92fe9b55ad1d760e8d8fb5d87e0d2e303cb8eed57f078d54293a2" +checksum = "47e10808eb27efde692f1b1706089fe19296728bf872291f5b4e0d50fba88f2f" dependencies = [ "anyhow", "indexmap", @@ -4775,9 +4778,9 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.258.0" +version = "0.258.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a61719f93a87b16d325921e251800c4833f8fab50fa21c7de73aed50086313" +checksum = "fe9269dd817e0e7021bf534ccd9761bc2d64778b6708e41228b8439d75eaa0b1" dependencies = [ "bitflags", "hashbrown 0.17.1", @@ -4788,9 +4791,9 @@ dependencies = [ [[package]] name = "wasmprinter" -version = "0.258.0" +version = "0.258.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a65e30fb2cd3cc5cb7761a6064787b2abf017e6566d8d0dedf2bb7f55ad5383" +checksum = "14a457ba8670a3db3d357c1aeac3ed4fe378a76054a221789f0df0ea507e292b" dependencies = [ "anyhow", "termcolor", @@ -4799,9 +4802,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942ddd2fc5800ec01d4b11c28ad62ab00517dcd416aff728f7d73fddf75872c1" +checksum = "36abb25de5518707e5ba96b586366b09db168d2d4976cb9dac57265a317497ea" dependencies = [ "addr2line", "async-trait", @@ -4842,9 +4845,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98f2ca21b23a51c1b1944297e9671a628c368e92aa6250e4c38f48e2dc251f0c" +checksum = "92f16a4d09fb1fbf2dcfeddb9818e1517a258e13b505b3bfdadcf47a19bf31ca" dependencies = [ "anyhow", "cpp_demangle", @@ -4873,9 +4876,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-cache" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76c83428ffb93757881522619aef1f6571a235c25ea5e9e6b67aa6a2324da72a" +checksum = "96e4469e885ac50f8dc06337245f9905b22c68d55416267f9afd07f7f76687be" dependencies = [ "base64 0.22.1", "directories-next", @@ -4893,9 +4896,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92c42f95d879875994319bd6689b17570e68aebd1a17e0dc54430928ed0699b5" +checksum = "d4fca03bd4425856574c0807d1dee16fad43f947feaf4e9ae1d2129596ee4de4" dependencies = [ "anyhow", "proc-macro2", @@ -4908,15 +4911,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "294c7f12f062ceb148cd26c77170ff739552db76d1f203f6127f249684988a70" +checksum = "4006009fc3598af4e895ab18c6c33d2c6b7c657b1bd0a755ca7ee1abef6f0c16" [[package]] name = "wasmtime-internal-core" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64b67114739063bb72bb8ed2b4994b8b22809d354432db68553ad529850f9536" +checksum = "947a732ad659f42c10ec639706fc8cb1f068dd9a55eebde8a1f24dfd06dfa525" dependencies = [ "anyhow", "hashbrown 0.17.1", @@ -4926,9 +4929,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-cranelift" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7d19868a30cc20df6f6648c9a4b3f700fcd1ddd0a63af9a98cb75f2b58dc874" +checksum = "fbfb8b455d524939e03df42e3e749f8e26e26ba2acd8b08fa00b3bda55686750" dependencies = [ "cranelift-codegen", "cranelift-control", @@ -4952,9 +4955,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1999aee268c3f625666b12ba0ded55a6ac2c800b60b426af7d164d63411908" +checksum = "bd8aa8711f68b2cdd94c2b85ba5c8317edaa1ac4ee0a0534b957123680239307" dependencies = [ "cc", "libc", @@ -4966,9 +4969,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca2499301c6f11cc9eb42c2beafc8efae396ae4cb91077e3ac080dc5d765e4" +checksum = "e873e4504faf50aa8ce255d6ea7210d87b39d5f379c2486782e2c5f179c70ca8" dependencies = [ "cc", "wasmtime-internal-versioned-export-macros", @@ -4976,9 +4979,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7162d829c56265e183d23b17d67afc5c6876433daaed54c137388a8078a1bb6b" +checksum = "197ab1fda5da958a5c0b8a97eb241beafc48ac568b82ef4f12f38d6866c74173" dependencies = [ "libc", "wasmtime-internal-core", @@ -4987,9 +4990,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-unwinder" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e167cf144f3cee40838f7663e3a589f93bba65151efa084a2d58b2bd47469f1" +checksum = "5577f3a5a979a7c3b1b23c1e9f13e70a8583acc8828471ec56f282117fc4952b" dependencies = [ "cranelift-codegen", "log", @@ -4999,9 +5002,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d09e038ad69c741dae29da4606ccfc0df66f156e62cb7f0664c8d7d1272a36d" +checksum = "f6399c9c4050ed431cf66e21c05fadd6f9731db44476f452f8c7142b4597b883" dependencies = [ "proc-macro2", "quote", @@ -5010,9 +5013,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c63d97a742fc9b415dad7ed2ac33ddb8fe6c5413fcd0c7ee58907f2ab6f9a430" +checksum = "b4eaf45d4da7681d234bb6efd846b4a528b08a75384a7fc23f125d0145364b06" dependencies = [ "anyhow", "bitflags", @@ -5024,9 +5027,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce11727cb331abeaace3681a4b072e4fe04818d7b99786d183b318671f9e4e0e" +checksum = "821e45765c35555d3b0bdf9277cb99478f7d1f29a17b3315ba76e01220f02a38" dependencies = [ "async-trait", "bitflags", @@ -5048,9 +5051,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi-io" -version = "49.0.1" +version = "49.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35f5ed8d710d8daf71cb4742c6b90c1fd121acb62b6848556eb4e0661cbd28bd" +checksum = "8cdff38d1c730bdaf4fcfbe0eedf3265fb9c7a4deb119deecbfc1c1778950716" dependencies = [ "async-trait", "bytes", @@ -5081,9 +5084,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] @@ -5094,14 +5097,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.9", + "webpki-roots 1.0.10", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] @@ -5302,9 +5305,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "wit-component" -version = "0.258.0" +version = "0.258.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "481b5c47b2ecce0389b5e08a05557d6a190c9cd761773b8880a8017ee04dc7ef" +checksum = "f0fca88950af0c1f42bc77e17315e48798b8d68340171b6d204f62bc19a878a6" dependencies = [ "anyhow", "bitflags", @@ -5321,9 +5324,9 @@ dependencies = [ [[package]] name = "wit-parser" -version = "0.258.0" +version = "0.258.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4daaa3cd97ae49ecd0a99dc009d453f93e0f083dd3be38c0f24a83a93e37ac" +checksum = "72a2127663dcf3ee16614fb0e3c33d5c6c66fec9dad317a8b0fdd3869487a533" dependencies = [ "anyhow", "hashbrown 0.17.1", @@ -5391,9 +5394,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", @@ -5497,14 +5500,15 @@ dependencies = [ [[package]] name = "zip" -version = "8.6.0" +version = "9.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e" dependencies = [ "crc32fast", "flate2", "indexmap", "memchr", + "nix 0.31.3", "typed-path", "zopfli", ] diff --git a/Cargo.toml b/Cargo.toml index 754f35a..03d8f94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,9 +29,9 @@ bytes = "1" futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } quick-xml = "0.42" # Registry spreadsheets: xlsx out, xlsx/ods in, CSV both ways. -calamine = "0.36" +calamine = "0.37" csv = "1" -rust_xlsxwriter = "0.99" +rust_xlsxwriter = "0.101" regex = "1" rumqttc = "0.25" # Named so the process can pick its TLS crypto provider (see ot-server main). From e380e05a68ffcef7f71bfd749f7ce10b107521d5 Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Wed, 21 Oct 2026 15:32:29 -0500 Subject: [PATCH 04/16] Sources: drafts, and every problem on its pipeline stage (#104) A source that is not complete yet saves as a draft that cannot be enabled, instead of being refused: - SourceSpec::problems lists every problem, each with its stage (source, decode, map, ..., output) and setting. Only an invalid id or name stops a save. validate() still returns the first, as before. - Empty transport settings are reported by name. - MappingSpec::problems reports every rule's problems. - Sources store their problems (migration 0020); a draft cannot be enabled, in the store as well as the API. - PUT on an enabled source is refused unless the spec is complete; enable re-checks. - A refusal is a 422 whose body lists the problems beside the error. - Configuration backups carry drafts; an enabled source in a file must be complete. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 12 + crates/ot-server/src/api.rs | 240 +++++++++---- crates/ot-server/src/auth/api.rs | 1 + crates/ot-server/src/config_backup.rs | 15 +- crates/ot-server/src/control.rs | 36 +- crates/ot-server/src/engine/bench.rs | 1 + crates/ot-server/src/sources.rs | 2 + crates/ot-source/src/mapping.rs | 19 +- crates/ot-source/src/source.rs | 317 ++++++++++++++---- crates/ot-source/src/transport.rs | 51 +++ .../migrations/0020_source_problems.sql | 5 + crates/ot-store/src/backup.rs | 44 ++- crates/ot-store/src/sources.rs | 90 ++++- crates/ot-store/src/sqlite.rs | 5 +- docs/reference.md | 8 + ui/src/api/client.ts | 28 ++ 16 files changed, 727 insertions(+), 147 deletions(-) create mode 100644 crates/ot-store/migrations/0020_source_problems.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 37552ab..5d89723 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## Unreleased + +### Added + +- **Draft sources.** A source that is not complete yet saves as a draft, which cannot be enabled, + instead of being refused. Its row carries `draft` and `problems`, each with the pipeline stage it + belongs to. An enabled source is still saved only complete, and enabling re-checks it. +- **Every problem with a source at once.** A refused save or enable is a 422 that lists all of the + source's `problems` (`{kind, stage, field?, message}`), not just the first, and an empty + transport setting (host, port, URL, bind address, topic, method, path) is now reported by name. + The SQLite schema moves to version 20. + ## 1.0.0, 2026-10-06 The first open-source release, under the MIT License. 1.0 carries everything up to 0.4.7, plus the diff --git a/crates/ot-server/src/api.rs b/crates/ot-server/src/api.rs index b337be4..ce40d04 100644 --- a/crates/ot-server/src/api.rs +++ b/crates/ot-server/src/api.rs @@ -13,7 +13,7 @@ use axum::{Extension, Json, Router}; use ot_source::frame::Frame; use ot_source::pipeline::Pipeline; use ot_source::schema::{ExtensionField, ExtensionSchema}; -use ot_source::source::SourceSpec; +use ot_source::source::{Problem, ProblemKind, SourceSpec, Stage}; use ot_source::trace::Trace; use ot_store::{Entity, SourceRow, SourceWrite}; use serde::Deserialize; @@ -73,25 +73,72 @@ pub(crate) fn actor(headers: &HeaderMap) -> String { .map_or_else(|| "op:api".to_owned(), str::to_owned) } -/// Parse and validate a source spec, including against the published -/// extension schema version its mapping targets. Returns the spec, its -/// normalised JSON (defaults filled in) and that schema. -async fn parse_spec( - s: &AppState, - body: Value, -) -> Result<(SourceSpec, Value, ExtensionSchema), ApiError> { - let spec: SourceSpec = serde_json::from_value(body) - .map_err(|e| ApiError::unprocessable(format!("invalid source spec: {e}")))?; - crate::probe::file_allowed(&s.common, &spec.transport).map_err(ApiError::unprocessable)?; +/// A source spec as sent, checked: the spec, its normalised JSON (defaults +/// filled in), the published extension schema its mapping targets (if it +/// is published) and everything that stops it running. A spec that does +/// not parse, or is not allowed at all, is refused here. +struct Checked { + spec: SourceSpec, + normalised: Value, + schema: Option, + problems: Vec, +} + +impl Checked { + /// Refuse it if it cannot be saved (an invalid id or name). + fn savable(self) -> Result { + let invalid: Vec = self + .problems + .iter() + .filter(|p| p.kind == ProblemKind::Invalid) + .cloned() + .collect(); + if invalid.is_empty() { + Ok(self) + } else { + Err(ApiError::source_problems(&invalid)) + } + } + + /// Refuse it unless it is complete (it may run). + fn complete(self) -> Result<(SourceSpec, Value, ExtensionSchema), ApiError> { + match self.schema { + Some(schema) if self.problems.is_empty() => Ok((self.spec, self.normalised, schema)), + _ => Err(ApiError::source_problems(&self.problems)), + } + } +} + +/// One problem with a spec as a whole, for a refusal. +fn invalid(message: String) -> ApiError { + ApiError::source_problems(&[Problem { + kind: ProblemKind::Invalid, + stage: None, + field: None, + message, + }]) +} + +async fn check_spec(s: &AppState, body: Value) -> Result { + let spec: SourceSpec = + serde_json::from_value(body).map_err(|e| invalid(format!("invalid source spec: {e}")))?; + crate::probe::file_allowed(&s.common, &spec.transport).map_err(|e| { + ApiError::source_problems(&[Problem { + kind: ProblemKind::Invalid, + stage: Some(Stage::Source), + field: Some("path".into()), + message: e, + }]) + })?; let schema = published_schema(s, spec.pipeline.mapping.schema_version).await?; - spec.validate_against(schema.as_ref()) - .map_err(|e| ApiError::unprocessable(e.to_string()))?; + let problems = spec.problems(schema.as_ref()); let normalised = serde_json::to_value(&spec).map_err(|e| ApiError::internal(e.to_string()))?; - Ok(( + Ok(Checked { spec, normalised, - schema.expect("validated against a published schema"), - )) + schema, + problems, + }) } /// A published extension schema version, typed. @@ -175,17 +222,20 @@ async fn get_source( ))) } -async fn save( - s: &AppState, - spec: SourceSpec, - normalised: Value, - actor: String, -) -> Result { +async fn save(s: &AppState, c: Checked, actor: String) -> Result { + let Checked { + spec, + normalised, + problems, + .. + } = c; if spec.unauthenticated.is_some() { // The decision records the acceptance; the log says it plainly. tracing::warn!(source = %spec.id, transport = spec.transport.kind(), %actor, "listening source saved with unauthenticated senders accepted"); } + let problems = + serde_json::to_value(&problems).map_err(|e| ApiError::internal(e.to_string()))?; s.with_db(move |db| { db.put_source( &SourceWrite { @@ -195,6 +245,7 @@ async fn save( codec: spec.pipeline.codec.name(), priority: spec.priority, spec: &normalised, + problems: &problems, }, &actor, ) @@ -202,37 +253,47 @@ async fn save( .await } +/// A new source. One that is not complete yet is saved as a draft. async fn create_source( State(s): State, headers: HeaderMap, Json(body): Json, ) -> Result<(StatusCode, Json), ApiError> { - let (spec, normalised, _) = parse_spec(&s, body).await?; - let id = spec.id.clone(); + let c = check_spec(&s, body).await?.savable()?; + let id = c.spec.id.clone(); if s.with_db(move |db| db.get_source(&id)).await?.is_some() { return Err(ApiError::conflict(format!( "source {} already exists; use PUT to update", - spec.id + c.spec.id ))); } - let row = save(&s, spec, normalised, actor(&headers)).await?; + let row = save(&s, c, actor(&headers)).await?; Ok((StatusCode::CREATED, Json(with_status(&s, row).await))) } +/// Save a source. A disabled one may be saved as a draft; an enabled one +/// only complete, so that an edit never stops a running source. async fn put_source( State(s): State, Path(id): Path, headers: HeaderMap, Json(body): Json, ) -> Result, ApiError> { - let (spec, normalised, _) = parse_spec(&s, body).await?; - if spec.id != id { + let c = check_spec(&s, body).await?.savable()?; + if c.spec.id != id { return Err(ApiError::unprocessable(format!( "body id {:?} does not match path id {id:?} (source ids never change)", - spec.id + c.spec.id ))); } - let row = save(&s, spec, normalised, actor(&headers)).await?; + let enabled = s + .with_db(move |db| db.get_source(&id)) + .await? + .is_some_and(|r| r.enabled); + if enabled && !c.problems.is_empty() { + return Err(ApiError::source_problems(&c.problems)); + } + let row = save(&s, c, actor(&headers)).await?; Ok(Json(with_status(&s, row).await)) } @@ -259,11 +320,22 @@ async fn toggle( Ok(Json(with_status(&s, row).await)) } +/// Enable a source: only a complete one, checked again now (the schema +/// version its mapping targets, say, may have changed since it was saved). async fn enable_source( State(s): State, Path(id): Path, headers: HeaderMap, ) -> Result, ApiError> { + let key = id.clone(); + let row = s + .with_db(move |db| db.get_source(&key)) + .await? + .ok_or_else(|| ApiError::not_found("source"))?; + let c = check_spec(&s, row.spec).await?; + if !c.problems.is_empty() { + return Err(ApiError::source_problems(&c.problems)); + } toggle(s, id, true, headers).await } @@ -401,7 +473,7 @@ async fn validate_source( State(s): State, Json(body): Json, ) -> Result, ApiError> { - let (spec, normalised, schema) = parse_spec(&s, body.spec).await?; + let (spec, normalised, schema) = check_spec(&s, body.spec).await?.complete()?; let mut frames: Vec = body .samples .iter() @@ -917,17 +989,30 @@ mod tests { spec["id"] = json!("udp-auth"); let accepted = spec.clone(); spec.as_object_mut().unwrap().remove("unauthenticated"); + // Without the acceptance it is a draft, and cannot be enabled. let (st, body) = call(&app, "POST", "/api/v1/sources", Some(spec.clone())).await; - assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(st, StatusCode::CREATED, "{body}"); + let problem = &body["problems"][0]; + assert_eq!( + (&problem["stage"], &problem["field"]), + (&json!("source"), &json!("unauthenticated")), + "{body}" + ); assert!( - body["error"] + problem["message"] .as_str() .unwrap() .contains("UDP cannot carry TLS"), "{body}" ); - let (st, body) = call(&app, "POST", "/api/v1/sources", Some(accepted)).await; - assert_eq!(st, StatusCode::CREATED, "{body}"); + let (st, body) = call(&app, "POST", "/api/v1/sources/udp-auth/enable", None).await; + assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + let (st, body) = call(&app, "PUT", "/api/v1/sources/udp-auth", Some(accepted)).await; + assert_eq!( + (st, &body["draft"]), + (StatusCode::OK, &json!(false)), + "{body}" + ); // Moved to mutual TLS, the acceptance goes: a decision whose before // and after (the two revisions) show the change. spec["transport"] = json!({"type": "tcp_server", "bind": "127.0.0.1:0", @@ -944,9 +1029,10 @@ mod tests { .cloned() .unwrap() }; - assert_eq!(rev(1)["spec"]["unauthenticated"], "accepted", "{revs}"); - assert!(rev(2)["spec"].get("unauthenticated").is_none(), "{revs}"); - assert!(rev(2)["decision_id"].is_i64(), "{revs}"); + assert!(rev(1)["spec"].get("unauthenticated").is_none(), "{revs}"); + assert_eq!(rev(2)["spec"]["unauthenticated"], "accepted", "{revs}"); + assert!(rev(3)["spec"].get("unauthenticated").is_none(), "{revs}"); + assert!(rev(3)["decision_id"].is_i64(), "{revs}"); let (_, d) = call(&app, "GET", "/api/v1/decisions?op=update_source", None).await; assert_eq!(d["decisions"][0]["evidence"]["source"], "udp-auth", "{d}"); } @@ -959,22 +1045,41 @@ mod tests { }; let spec: Value = serde_json::from_str(ADSB).unwrap(); - // Its extension schema is not published yet. + // Its extension schema is not published yet: it saves, as a draft + // that cannot be enabled. let (st, body) = call(&app, "POST", "/api/v1/sources", Some(spec.clone())).await; - assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY); + assert_eq!(st, StatusCode::CREATED, "{body}"); + assert_eq!( + (body["revision"].as_i64(), body["enabled"].as_bool()), + (Some(1), Some(false)) + ); + assert_eq!(body["transport"], "http_poll"); + assert_eq!(body["draft"], true, "{body}"); + assert_eq!(body["problems"][0]["kind"], "incomplete", "{body}"); + assert_eq!(body["problems"][0]["stage"], "map", "{body}"); + assert!( + body["problems"][0]["message"] + .as_str() + .unwrap() + .contains("schema version 2"), + "{body}" + ); + let (st, body) = call(&app, "POST", "/api/v1/sources/adsb-lol/enable", None).await; + assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); assert!( body["error"].as_str().unwrap().contains("schema version 2"), "{body}" ); - publish_example_schema(&app).await; + assert_eq!(body["problems"][0]["stage"], "map", "{body}"); - let (st, body) = call(&app, "POST", "/api/v1/sources", Some(spec.clone())).await; - assert_eq!(st, StatusCode::CREATED, "{body}"); + publish_example_schema(&app).await; + let (st, body) = call(&app, "PUT", "/api/v1/sources/adsb-lol", Some(spec.clone())).await; + assert_eq!(st, StatusCode::OK, "{body}"); assert_eq!( - (body["revision"].as_i64(), body["enabled"].as_bool()), - (Some(1), Some(false)) + (body["revision"].as_i64(), body["draft"].as_bool()), + (Some(2), Some(false)) ); - assert_eq!(body["transport"], "http_poll"); + assert_eq!(body["problems"], json!([])); let (st, _) = call(&app, "POST", "/api/v1/sources", Some(spec.clone())).await; assert_eq!(st, StatusCode::CONFLICT); @@ -982,38 +1087,53 @@ mod tests { let (st, body) = call(&app, "PUT", "/api/v1/sources/other-id", Some(spec.clone())).await; assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + // A source with no name is not saved at all. + let mut nameless = spec.clone(); + nameless["name"] = json!(" "); + let (st, body) = call(&app, "PUT", "/api/v1/sources/adsb-lol", Some(nameless)).await; + assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["problems"][0]["kind"], "invalid", "{body}"); + + // Off, a mistake leaves a draft, with every problem on its stage. let mut typo = spec.clone(); typo["pipeline"]["mapping"]["rules"][1]["fields"]["kinematic.speed_mps"] = json!("gs"); - let (st, body) = call(&app, "PUT", "/api/v1/sources/adsb-lol", Some(typo)).await; - assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY); + typo["pipeline"]["mapping"]["rules"][1]["fields"]["ext.tail_art"] = json!("r"); + let (st, body) = call(&app, "PUT", "/api/v1/sources/adsb-lol", Some(typo.clone())).await; + assert_eq!(st, StatusCode::OK, "{body}"); + assert_eq!(body["draft"], true, "{body}"); + let messages: Vec<&str> = body["problems"] + .as_array() + .unwrap() + .iter() + .inspect(|p| assert_eq!(p["stage"], "map", "{body}")) + .map(|p| p["message"].as_str().unwrap()) + .collect(); assert!( - body["error"] - .as_str() - .unwrap() - .contains("kinematic.speed_mps"), + messages.iter().any(|m| m.contains("kinematic.speed_mps")), "{body}" ); - - let mut undeclared = spec.clone(); - undeclared["pipeline"]["mapping"]["rules"][1]["fields"]["ext.tail_art"] = json!("r"); - let (st, body) = call(&app, "PUT", "/api/v1/sources/adsb-lol", Some(undeclared)).await; - assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY); assert!( - body["error"].as_str().unwrap().contains("ext.tail_art"), + messages.iter().any(|m| m.contains("ext.tail_art")), "{body}" ); let mut edited = spec.clone(); edited["name"] = json!("adsb.lol (edited)"); let (st, body) = call(&app, "PUT", "/api/v1/sources/adsb-lol", Some(edited)).await; - assert_eq!((st, body["revision"].as_i64()), (StatusCode::OK, Some(2))); + assert_eq!((st, body["revision"].as_i64()), (StatusCode::OK, Some(4))); + assert_eq!(body["draft"], false, "{body}"); let (_, body) = call(&app, "POST", "/api/v1/sources/adsb-lol/enable", None).await; assert_eq!(body["enabled"], true); + // On, the same mistake is refused, and the running source is kept. + let (st, body) = call(&app, "PUT", "/api/v1/sources/adsb-lol", Some(typo)).await; + assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["problems"].as_array().unwrap().len(), 2, "{body}"); let (_, body) = call(&app, "GET", "/api/v1/sources/adsb-lol/revisions", None).await; - assert_eq!(body["revisions"].as_array().unwrap().len(), 2); + assert_eq!(body["revisions"].as_array().unwrap().len(), 4); let (_, body) = call(&app, "GET", "/api/v1/sources", None).await; assert_eq!(body["sources"][0]["name"], "adsb.lol (edited)"); + assert_eq!(body["sources"][0]["draft"], false); // Raw output needs explicit consent and a subject outside `tracks.>`. for bad in [ diff --git a/crates/ot-server/src/auth/api.rs b/crates/ot-server/src/auth/api.rs index 6da89ea..b84bc31 100644 --- a/crates/ot-server/src/auth/api.rs +++ b/crates/ot-server/src/auth/api.rs @@ -1364,6 +1364,7 @@ mod tests { codec: "json", priority: 0, spec: &spec, + problems: &serde_json::json!([]), }, "root@x.org", ) diff --git a/crates/ot-server/src/config_backup.rs b/crates/ot-server/src/config_backup.rs index c4e1c72..ad6c64f 100644 --- a/crates/ot-server/src/config_backup.rs +++ b/crates/ot-server/src/config_backup.rs @@ -296,11 +296,21 @@ fn check(common: &Common, f: &mut ConfigFile) -> Vec { p.push(format!("{at}: {e}")); continue; } + // A draft comes back a draft; an enabled source must be complete. let schema = schemas.get(&spec.pipeline.mapping.schema_version); - if let Err(e) = spec.validate_against(schema) { - p.push(format!("{at}: {e}")); + let problems = spec.problems(schema); + let refused: Vec<_> = problems + .iter() + .filter(|q| s.enabled || q.kind == ot_source::source::ProblemKind::Invalid) + .collect(); + if let Some(q) = refused.first() { + p.push(format!("{at}: {}", q.message)); continue; } + match serde_json::to_value(&problems) { + Ok(v) => s.problems = v, + Err(e) => p.push(format!("{at}: {e}")), + } if let Some(sub) = &s.raw_subject && let Err(e) = crate::api::check_raw_subject(sub, &common.nats.tracks_subject) { @@ -817,6 +827,7 @@ pub(crate) mod tests { codec: "json", priority: 100, spec: &spec, + problems: &serde_json::json!([]), }, "op:test", ) diff --git a/crates/ot-server/src/control.rs b/crates/ot-server/src/control.rs index 2c3e0ad..d80b134 100644 --- a/crates/ot-server/src/control.rs +++ b/crates/ot-server/src/control.rs @@ -355,6 +355,9 @@ pub(crate) fn parse_uid(raw: &str) -> Result { pub struct ApiError { status: StatusCode, pub(crate) message: String, + /// What is wrong with a source, each problem with its pipeline stage + /// (`ot_source::source::Problem`), sent as the body's `problems`. + problems: Option, } impl ApiError { @@ -362,42 +365,61 @@ impl ApiError { Self { status: StatusCode::INTERNAL_SERVER_ERROR, message: m.into(), + problems: None, } } pub(crate) fn not_found(m: impl Into) -> Self { Self { status: StatusCode::NOT_FOUND, message: m.into(), + problems: None, } } pub(crate) fn unprocessable(m: impl Into) -> Self { Self { status: StatusCode::UNPROCESSABLE_ENTITY, message: m.into(), + problems: None, } } pub(crate) fn conflict(m: impl Into) -> Self { Self { status: StatusCode::CONFLICT, message: m.into(), + problems: None, } } pub(crate) fn unauthorized(m: impl Into) -> Self { Self { status: StatusCode::UNAUTHORIZED, message: m.into(), + problems: None, } } pub(crate) fn forbidden(m: impl Into) -> Self { Self { status: StatusCode::FORBIDDEN, message: m.into(), + problems: None, } } + /// A source that cannot be saved or enabled as it is: 422, with its + /// problems listed (the first is the `error`). + pub(crate) fn source_problems(problems: &[ot_source::source::Problem]) -> Self { + Self { + status: StatusCode::UNPROCESSABLE_ENTITY, + message: problems + .first() + .map_or_else(|| "invalid source".to_owned(), |p| p.message.clone()), + problems: Some(serde_json::to_value(problems).unwrap_or_default()), + } + } + pub(crate) fn bad_request(m: impl Into) -> Self { Self { status: StatusCode::BAD_REQUEST, message: m.into(), + problems: None, } } } @@ -406,10 +428,7 @@ impl From for ApiError { fn from(e: ot_store::StoreError) -> Self { match e { ot_store::StoreError::NotFound(m) => Self::not_found(m), - ot_store::StoreError::Conflict(m) => Self { - status: StatusCode::CONFLICT, - message: m, - }, + ot_store::StoreError::Conflict(m) => Self::conflict(m), other => Self::internal(other.to_string()), } } @@ -430,7 +449,14 @@ impl IntoResponse for ApiError { ) .into_response(); } - (self.status, Json(json!({ "error": self.message }))).into_response() + match self.problems { + Some(problems) => ( + self.status, + Json(json!({ "error": self.message, "problems": problems })), + ) + .into_response(), + None => (self.status, Json(json!({ "error": self.message }))).into_response(), + } } } diff --git a/crates/ot-server/src/engine/bench.rs b/crates/ot-server/src/engine/bench.rs index 217eed7..3f91600 100644 --- a/crates/ot-server/src/engine/bench.rs +++ b/crates/ot-server/src/engine/bench.rs @@ -237,6 +237,7 @@ pub async fn run(common: &Common, opts: BenchOptions) -> anyhow::Result<()> { codec: spec.pipeline.codec.name(), priority: spec.priority, spec: raw, + problems: &serde_json::json!([]), }, "bench", )?; diff --git a/crates/ot-server/src/sources.rs b/crates/ot-server/src/sources.rs index 1a6a31c..6f20055 100644 --- a/crates/ot-server/src/sources.rs +++ b/crates/ot-server/src/sources.rs @@ -624,6 +624,8 @@ mod tests { raw_subject: None, created_at_ms: 0, updated_at_ms: 0, + problems: serde_json::json!([]), + draft: false, }; let schema = ExtensionSchema { version: accepted["pipeline"]["mapping"]["schema_version"] diff --git a/crates/ot-source/src/mapping.rs b/crates/ot-source/src/mapping.rs index 7c61110..ea8c32a 100644 --- a/crates/ot-source/src/mapping.rs +++ b/crates/ot-source/src/mapping.rs @@ -209,17 +209,26 @@ pub enum MapOutcome { impl MappingSpec { pub fn validate(&self) -> Result<(), MappingError> { + match self.problems().into_iter().next() { + Some(e) => Err(e), + None => Ok(()), + } + } + + /// Every problem with the rules, in rule order. + pub fn problems(&self) -> Vec { if self.rules.is_empty() { - return Err(MappingError::NoRules); + return vec![MappingError::NoRules]; } + let mut out = Vec::new(); let mut names = std::collections::HashSet::new(); for rule in &self.rules { if !names.insert(rule.name.as_str()) { - return Err(MappingError::DuplicateRule(rule.name.clone())); + out.push(MappingError::DuplicateRule(rule.name.clone())); } for field in rule.fields.keys() { if target_type(field).is_none() { - return Err(MappingError::UnknownField { + out.push(MappingError::UnknownField { rule: rule.name.clone(), field: field.clone(), }); @@ -230,10 +239,10 @@ impl MappingSpec { && !(rule.fields.contains_key("position.latitude") && rule.fields.contains_key("position.longitude")) { - return Err(MappingError::NoPosition(rule.name.clone())); + out.push(MappingError::NoPosition(rule.name.clone())); } } - Ok(()) + out } pub fn apply(&self, record: &Value) -> MapOutcome { diff --git a/crates/ot-source/src/source.rs b/crates/ot-source/src/source.rs index 8d5e3b8..fef2885 100644 --- a/crates/ot-source/src/source.rs +++ b/crates/ot-source/src/source.rs @@ -224,12 +224,72 @@ impl SourceSpec { } } + /// The first reason this source may not run, in the order the checks + /// have always run: see [`Self::problems`] for all of them. pub fn validate(&self) -> Result<(), SourceError> { - if let Some(label) = &self.security { - label.validate().map_err(SourceError::Tracker)?; + match self.checks(None).into_iter().next() { + Some((_, _, e)) => Err(e), + None => Ok(()), } - if let Some(m) = &self.emitter_motion { - m.check().map_err(SourceError::Tracker)?; + } + + /// Validate against the extension schema version the mapping targets + /// (`None` if that version is not published). + pub fn validate_against( + &self, + schema: Option<&crate::schema::ExtensionSchema>, + ) -> Result<(), SourceError> { + match self.checks(Some(schema)).into_iter().next() { + Some((_, _, e)) => Err(e), + None => Ok(()), + } + } + + /// Everything that stops this source running, each with the pipeline + /// stage (and, where there is one, the setting) it belongs to. Empty: + /// the source is complete and may be enabled. A problem of kind + /// [`ProblemKind::Invalid`] also stops it being saved; the rest leave + /// it a draft. + pub fn problems(&self, schema: Option<&crate::schema::ExtensionSchema>) -> Vec { + self.checks(Some(schema)) + .into_iter() + .map(|(stage, field, e)| Problem { + kind: match e { + SourceError::BadId(_) | SourceError::NoName => ProblemKind::Invalid, + _ => ProblemKind::Incomplete, + }, + stage, + field: field.map(str::to_owned), + message: e.to_string(), + }) + .collect() + } + + /// Every check, in order, each failure with its stage and setting. + /// `schema`: `None` skips the extension schema checks; `Some(None)` + /// means the mapping's schema version is not published. + fn checks( + &self, + schema: Option>, + ) -> Vec<(Option, Option<&'static str>, SourceError)> { + let mut out = Vec::new(); + if let Some(label) = &self.security + && let Err(e) = label.validate() + { + out.push(( + Some(Stage::Output), + Some("security"), + SourceError::Tracker(e), + )); + } + if let Some(m) = &self.emitter_motion + && let Err(e) = m.check() + { + out.push(( + Some(Stage::Output), + Some("emitter_motion"), + SourceError::Tracker(e), + )); } let id_ok = !self.id.is_empty() && self.id.len() <= 64 @@ -238,86 +298,160 @@ impl SourceSpec { .bytes() .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-' || b == b'_'); if !id_ok { - return Err(SourceError::BadId(self.id.clone())); + out.push((None, Some("id"), SourceError::BadId(self.id.clone()))); } if self.name.trim().is_empty() { - return Err(SourceError::NoName); + out.push((None, Some("name"), SourceError::NoName)); + } + for e in self.pipeline.mapping.problems() { + out.push((Some(Stage::Map), None, e.into())); + } + if let Some(t) = &self.pipeline.tracker + && let Err(e) = t.validate() + { + out.push((Some(Stage::Tracker), None, SourceError::Tracker(e))); } - self.pipeline.mapping.validate()?; - if let Some(t) = &self.pipeline.tracker { - t.validate().map_err(SourceError::Tracker)?; + if let crate::codec::CodecConfig::Plugin { .. } | crate::codec::CodecConfig::Protobuf { .. } = + &self.pipeline.codec + && let Err(e) = crate::codec::Codec::new(self.pipeline.codec.clone()) + { + out.push(( + Some(Stage::Decode), + None, + SourceError::Tracker(e.to_string()), + )); + } + // An empty setting is reported once, not again by the checks + // that read it. + let missing = self.transport.missing(); + for (field, what) in &missing { + out.push(( + Some(Stage::Source), + Some(*field), + SourceError::Transport(format!("{what} is not set")), + )); } - if let crate::codec::CodecConfig::Plugin { .. } - | crate::codec::CodecConfig::Protobuf { .. } = &self.pipeline.codec + if missing.is_empty() + && let Err(e) = self.transport.check() { - crate::codec::Codec::new(self.pipeline.codec.clone()) - .map_err(|e| SourceError::Tracker(e.to_string()))?; + out.push((Some(Stage::Source), None, SourceError::Transport(e))); } - self.transport.check().map_err(SourceError::Transport)?; // gRPC carries protobuf: the method must fit the codec's message. if let TransportConfig::GrpcClient { .. } | TransportConfig::GrpcServer { .. } = &self.transport { - let proto = crate::transport::ProtoContext::of(&self.pipeline.codec) - .map_err(SourceError::Transport)? - .ok_or_else(|| { - SourceError::Transport("a gRPC source decodes with the protobuf codec".into()) - })?; - match &self.transport { - TransportConfig::GrpcClient { - method, request, .. - } => { - proto - .client_request(method, request) - .map_err(SourceError::Transport)?; - } - TransportConfig::GrpcServer { methods, .. } => { - proto - .server_methods(methods) - .map_err(SourceError::Transport)?; + match crate::transport::ProtoContext::of(&self.pipeline.codec) { + Err(e) => out.push((Some(Stage::Decode), None, SourceError::Transport(e))), + Ok(None) => out.push(( + Some(Stage::Decode), + None, + SourceError::Transport("a gRPC source decodes with the protobuf codec".into()), + )), + Ok(Some(_)) if !missing.is_empty() => {} + Ok(Some(proto)) => { + let fit = match &self.transport { + TransportConfig::GrpcClient { + method, request, .. + } => proto.client_request(method, request).map(drop), + TransportConfig::GrpcServer { methods, .. } => { + proto.server_methods(methods).map(drop) + } + _ => Ok(()), + }; + if let Err(e) = fit { + out.push((Some(Stage::Source), None, SourceError::Transport(e))); + } } - _ => {} } } if let TransportConfig::TcpClient { framing, .. } | TransportConfig::TcpServer { framing, .. } = &self.transport + && let Err(e) = crate::frame::Framer::new(framing.clone()) { - crate::frame::Framer::new(framing.clone())?; + out.push((Some(Stage::Source), Some("framing"), e.into())); } // Last, so that a broken setting is reported before its security. if let Some(e) = self.authentication_problem() { - return Err(SourceError::Unauthenticated(e)); + out.push(( + Some(Stage::Source), + Some("unauthenticated"), + SourceError::Unauthenticated(e), + )); } - Ok(()) - } - - /// Validate against the extension schema version the mapping targets - /// (`None` if that version is not published). - pub fn validate_against( - &self, - schema: Option<&crate::schema::ExtensionSchema>, - ) -> Result<(), SourceError> { - self.validate()?; - let wanted = self.pipeline.mapping.schema_version; - let Some(schema) = schema.filter(|s| s.version == wanted) else { - return Err(SourceError::SchemaVersion { - wanted, - found: "not published".into(), - }); - }; - for (key, rules) in crate::schema::mapped_ext_keys(&self.pipeline.mapping) { - if schema.field(&key).is_none() { - return Err(SourceError::UndeclaredExtension { - rule: rules[0].clone(), - key, - version: wanted, - }); + if let Some(schema) = schema { + let wanted = self.pipeline.mapping.schema_version; + match schema.filter(|s| s.version == wanted) { + None => out.push(( + Some(Stage::Map), + Some("schema_version"), + SourceError::SchemaVersion { + wanted, + found: "not published".into(), + }, + )), + Some(schema) => { + for (key, rules) in crate::schema::mapped_ext_keys(&self.pipeline.mapping) { + if schema.field(&key).is_none() { + out.push(( + Some(Stage::Map), + None, + SourceError::UndeclaredExtension { + rule: rules[0].clone(), + key, + version: wanted, + }, + )); + } + } + } } } - Ok(()) + out } } +/// One reason a source may not run (see [`SourceSpec::problems`]). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Problem { + pub kind: ProblemKind, + /// The pipeline stage it belongs to; `None` for the source as a whole + /// (its id and name). + pub stage: Option, + /// The setting, where there is one (for example `host`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub field: Option, + pub message: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProblemKind { + /// The source cannot be saved. + Invalid, + /// The source saves as a draft, which cannot be enabled. + Incomplete, +} + +/// A source's pipeline stages, in the order a frame meets them. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Stage { + /// The transport and its framing. + Source, + /// The codec. + Decode, + Map, + StaticJoin, + Registry, + Affiliation, + Filter, + Tracker, + Throttle, + /// What the source reports and how: tracks or detections, security + /// label, priority. + Output, +} + #[cfg(test)] mod tests { use super::*; @@ -376,6 +510,71 @@ mod tests { assert!(chosen.publishes_alone()); } + #[test] + fn problems_name_their_stage_and_a_draft_is_only_incomplete() { + let ok: SourceSpec = serde_json::from_value(spec("cot")).unwrap(); + assert_eq!( + ok.problems(None), + vec![Problem { + kind: ProblemKind::Incomplete, + stage: Some(Stage::Map), + field: Some("schema_version".into()), + message: "mapping targets schema version 1, but it is not published".into(), + }] + ); + + // A new source: the settings left empty, no rules yet. + let draft: SourceSpec = serde_json::from_value(json!({ + "id": "radar", "name": "Radar", + "transport": { "type": "tcp_client", "host": " ", "port": 0 }, + "pipeline": { "codec": { "type": "json" }, "mapping": { "rules": [] } } + })) + .unwrap(); + let found: Vec<_> = draft + .problems(None) + .into_iter() + .map(|p| (p.kind, p.stage, p.field)) + .collect(); + let incomplete = ProblemKind::Incomplete; + assert_eq!( + found, + vec![ + (incomplete, Some(Stage::Map), None), + (incomplete, Some(Stage::Source), Some("host".into())), + (incomplete, Some(Stage::Source), Some("port".into())), + (incomplete, Some(Stage::Map), Some("schema_version".into())), + ] + ); + assert!(matches!(draft.validate(), Err(SourceError::Mapping(_)))); + + // Every rule's problems, not just the first. + let mut v = spec("cot"); + v["name"] = json!(""); + v["pipeline"]["mapping"]["rules"] = json!([ + { "name": "a", "key": "k", "fields": { "nope": "x" } }, + { "name": "a", "key": "k", "fields": {} } + ]); + let s: SourceSpec = serde_json::from_value(v).unwrap(); + let found: Vec<_> = s + .problems(None) + .into_iter() + .map(|p| (p.kind, p.stage, p.message)) + .collect(); + assert_eq!( + found[0], + ( + ProblemKind::Invalid, + None, + "source name must not be empty".into() + ) + ); + assert_eq!( + found.iter().filter(|p| p.1 == Some(Stage::Map)).count(), + 5, + "unknown field, no position (twice), duplicate rule, schema: {found:?}" + ); + } + #[test] fn ids_are_restricted() { let ok: SourceSpec = serde_json::from_value(spec("cot-udp_1")).unwrap(); diff --git a/crates/ot-source/src/transport.rs b/crates/ot-source/src/transport.rs index 20a76c7..b1d46a6 100644 --- a/crates/ot-source/src/transport.rs +++ b/crates/ot-source/src/transport.rs @@ -379,6 +379,57 @@ impl TransportConfig { } } + /// The settings a source needs that are empty, as (setting, what it + /// is): a draft may leave them so, an enabled source may not. + pub fn missing(&self) -> Vec<(&'static str, &'static str)> { + let blank = |s: &str| s.trim().is_empty(); + let mut out = Vec::new(); + match self { + TransportConfig::TcpClient { host, port, .. } => { + if blank(host) { + out.push(("host", "the host")); + } + if *port == 0 { + out.push(("port", "the port")); + } + } + TransportConfig::TcpServer { bind, .. } + | TransportConfig::Udp { bind, .. } + | TransportConfig::GrpcServer { bind, .. } => { + if blank(bind) { + out.push(("bind", "the address to listen on")); + } + } + TransportConfig::HttpPoll { url, .. } | TransportConfig::Websocket { url, .. } => { + if blank(url) { + out.push(("url", "the URL")); + } + } + TransportConfig::Mqtt { url, topics, .. } => { + if blank(url) { + out.push(("url", "the broker URL")); + } + if topics.iter().all(|t| blank(t)) { + out.push(("topics", "a topic")); + } + } + TransportConfig::GrpcClient { url, method, .. } => { + if blank(url) { + out.push(("url", "the URL")); + } + if blank(method) { + out.push(("method", "the method")); + } + } + TransportConfig::File { path, .. } => { + if blank(path) { + out.push(("path", "the path")); + } + } + } + out + } + /// Settings that cannot be caught by the type alone. pub fn check(&self) -> Result<(), String> { match self { diff --git a/crates/ot-store/migrations/0020_source_problems.sql b/crates/ot-store/migrations/0020_source_problems.sql new file mode 100644 index 0000000..73eb9d6 --- /dev/null +++ b/crates/ot-store/migrations/0020_source_problems.sql @@ -0,0 +1,5 @@ +-- Why a source may not run yet, as a JSON array of problems ({kind, stage, +-- field?, message}), worked out by the server each time the source is saved. +-- Empty: complete. Otherwise the source is a draft, which cannot be enabled. +-- Every source saved before drafts was complete. +ALTER TABLE sources ADD COLUMN problems TEXT NOT NULL DEFAULT '[]'; diff --git a/crates/ot-store/src/backup.rs b/crates/ot-store/src/backup.rs index 7956e9d..d78a823 100644 --- a/crates/ot-store/src/backup.rs +++ b/crates/ot-store/src/backup.rs @@ -92,12 +92,24 @@ pub struct SourceEntry { pub created_at_ms: i64, #[serde(default)] pub updated_at_ms: i64, + /// Why it may not run yet (a draft); empty when it is complete. Files + /// from before drafts have none: every source was complete. + #[serde(default = "no_problems", skip_serializing_if = "is_empty_array")] + pub problems: Value, } fn one() -> i64 { 1 } +fn no_problems() -> Value { + Value::Array(Vec::new()) +} + +fn is_empty_array(v: &Value) -> bool { + v.as_array().is_some_and(Vec::is_empty) +} + /// An output schema version. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -372,6 +384,7 @@ impl Db { raw_subject: r.raw_subject, created_at_ms: r.created_at_ms, updated_at_ms: r.updated_at_ms, + problems: r.problems, }) .collect(); let schema_versions = self @@ -652,8 +665,8 @@ impl Db { let revision = s.revision.max(1); tx.execute( "INSERT INTO sources (id, name, transport, codec, settings, enabled, priority, - raw_subject, raw_consent, revision, created_at_ms, updated_at_ms) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", + raw_subject, raw_consent, revision, created_at_ms, updated_at_ms, problems) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", params![ s.id, s.name, @@ -667,6 +680,7 @@ impl Db { revision, if s.created_at_ms > 0 { s.created_at_ms } else { now }, now, + s.problems.to_string(), ], )?; tx.execute( @@ -757,6 +771,7 @@ mod tests { codec: "json", priority: 50, spec: &spec, + problems: &serde_json::json!([]), }, "op:test", ) @@ -888,6 +903,31 @@ mod tests { assert!(ops.contains(&"import_config".to_owned()), "{ops:?}"); } + #[test] + fn a_draft_source_comes_back_a_draft() { + let mut snap = configured().config_snapshot().unwrap(); + let problems = json!([{"kind": "incomplete", "stage": "source", "field": "url", + "message": "the URL is not set"}]); + snap.sources[0].enabled = false; + snap.sources[0].problems = problems.clone(); + // The file says so, and a file from before drafts reads as complete. + let file = serde_json::to_value(&snap).unwrap(); + assert_eq!(file["sources"][0]["problems"], problems); + let mut old = file.clone(); + old["sources"][0] + .as_object_mut() + .unwrap() + .remove("problems"); + let old: ConfigFile = serde_json::from_value(old).unwrap(); + assert_eq!(old.sources[0].problems, json!([])); + + let mut fresh = Db::open_in_memory().unwrap(); + fresh.restore_config(&snap, "cli", || Ok(())).unwrap(); + let row = fresh.get_source("ais").unwrap().unwrap(); + assert!(row.draft && !row.enabled); + assert_eq!(row.problems, problems); + } + #[test] fn a_file_without_accounts_keeps_the_first_admin() { let mut snap = configured().config_snapshot().unwrap(); diff --git a/crates/ot-store/src/sources.rs b/crates/ot-store/src/sources.rs index 97ce303..61da918 100644 --- a/crates/ot-store/src/sources.rs +++ b/crates/ot-store/src/sources.rs @@ -21,6 +21,11 @@ pub struct SourceRow { pub raw_subject: Option, pub created_at_ms: i64, pub updated_at_ms: i64, + /// Why the source may not run yet, as the server found when it was + /// saved: a JSON array, empty when it is complete. + pub problems: Value, + /// Whether it has problems: a draft cannot be enabled. + pub draft: bool, } #[derive(Debug, Clone, PartialEq, Serialize)] @@ -40,11 +45,13 @@ pub struct SourceWrite<'a> { pub codec: &'a str, pub priority: i64, pub spec: &'a Value, + /// Its problems (a JSON array; empty: complete). + pub problems: &'a Value, } -const COLUMNS: &str = "id, name, transport, codec, enabled, priority, revision, settings, raw_subject, created_at_ms, updated_at_ms"; +const COLUMNS: &str = "id, name, transport, codec, enabled, priority, revision, settings, raw_subject, created_at_ms, updated_at_ms, problems"; -fn row(r: &rusqlite::Row<'_>) -> rusqlite::Result<(SourceRow, String)> { +fn row(r: &rusqlite::Row<'_>) -> rusqlite::Result<(SourceRow, String, String)> { let spec: String = r.get(7)?; Ok(( SourceRow { @@ -59,13 +66,18 @@ fn row(r: &rusqlite::Row<'_>) -> rusqlite::Result<(SourceRow, String)> { raw_subject: r.get(8)?, created_at_ms: r.get(9)?, updated_at_ms: r.get(10)?, + problems: Value::Null, + draft: false, }, spec, + r.get(11)?, )) } -fn finish((mut row, spec): (SourceRow, String)) -> Result { +fn finish((mut row, spec, problems): (SourceRow, String, String)) -> Result { row.spec = serde_json::from_str(&spec)?; + row.problems = serde_json::from_str(&problems)?; + row.draft = row.problems.as_array().is_some_and(|a| !a.is_empty()); Ok(row) } @@ -110,19 +122,23 @@ impl Db { }; let decision_id = record_decision(tx, &decision, now)?; let spec = w.spec.to_string(); + let problems = w.problems.to_string(); if before.is_none() { tx.execute( "INSERT INTO sources (id, name, transport, codec, settings, priority, revision, - created_at_ms, updated_at_ms) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, 1, ?7, ?7)", - params![w.id, w.name, w.transport, w.codec, spec, w.priority, now], + created_at_ms, updated_at_ms, problems) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, 1, ?7, ?7, ?8)", + params![w.id, w.name, w.transport, w.codec, spec, w.priority, now, problems], )?; } else { tx.execute( "UPDATE sources SET name = ?2, transport = ?3, codec = ?4, settings = ?5, - priority = ?6, revision = ?7, updated_at_ms = ?8 + priority = ?6, revision = ?7, updated_at_ms = ?8, problems = ?9 WHERE id = ?1", - params![w.id, w.name, w.transport, w.codec, spec, w.priority, revision, now], + params![ + w.id, w.name, w.transport, w.codec, spec, w.priority, revision, now, + problems + ], )?; } tx.execute( @@ -151,13 +167,26 @@ impl Db { }; let d = Decision::new(actor, op).evidence(json!({ "source": id })); record_decision(tx, &d, now)?; - let n = tx.execute( + let draft: Option = tx + .query_row( + "SELECT problems != '[]' FROM sources WHERE id = ?1", + [id], + |r| r.get(0), + ) + .optional()?; + match draft { + None => return Err(StoreError::NotFound(format!("source {id}"))), + Some(true) if enabled => { + return Err(StoreError::Conflict(format!( + "source {id} is a draft; complete it before enabling it" + ))); + } + Some(_) => {} + } + tx.execute( "UPDATE sources SET enabled = ?2, updated_at_ms = ?3 WHERE id = ?1", params![id, enabled, now], )?; - if n == 0 { - return Err(StoreError::NotFound(format!("source {id}"))); - } Ok(()) })?; self.get_source(id)? @@ -255,6 +284,8 @@ fn masked(spec: &Value) -> Value { mod tests { use super::*; + const COMPLETE: &Value = &Value::Array(Vec::new()); + fn write<'a>(spec: &'a Value) -> SourceWrite<'a> { SourceWrite { id: "ais", @@ -263,9 +294,44 @@ mod tests { codec: "json", priority: 100, spec, + problems: COMPLETE, } } + #[test] + fn a_draft_keeps_its_problems_and_cannot_be_enabled() { + let mut db = Db::open_in_memory().unwrap(); + let spec = json!({"v": 1}); + let problems = json!([{"kind": "incomplete", "stage": "source", "field": "url", + "message": "the URL is not set"}]); + let row = db + .put_source( + &SourceWrite { + problems: &problems, + ..write(&spec) + }, + "op:test", + ) + .unwrap(); + assert!(row.draft); + assert_eq!(row.problems, problems); + assert!(matches!( + db.set_source_enabled("ais", true, "op:test"), + Err(StoreError::Conflict(_)) + )); + assert!(!db.get_source("ais").unwrap().unwrap().enabled); + // Turning a draft off is fine; completing it lets it run. + db.set_source_enabled("ais", false, "op:test").unwrap(); + let row = db.put_source(&write(&spec), "op:test").unwrap(); + assert!(!row.draft); + assert_eq!(row.problems, json!([])); + assert!( + db.set_source_enabled("ais", true, "op:test") + .unwrap() + .enabled + ); + } + #[test] fn decisions_hide_a_sources_secrets_and_the_revision_keeps_them() { let mut db = Db::open_in_memory().unwrap(); diff --git a/crates/ot-store/src/sqlite.rs b/crates/ot-store/src/sqlite.rs index d0a4ac6..0460c07 100644 --- a/crates/ot-store/src/sqlite.rs +++ b/crates/ot-store/src/sqlite.rs @@ -30,6 +30,7 @@ const MIGRATIONS: &[&str] = &[ include_str!("../migrations/0017_account_expiry.sql"), include_str!("../migrations/0018_sync_keys.sql"), include_str!("../migrations/0019_plugin_secret.sql"), + include_str!("../migrations/0020_source_problems.sql"), ]; #[derive(Debug, thiserror::Error)] @@ -567,9 +568,9 @@ mod tests { fn migrates_once_and_reopens() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("ot.db"); - assert_eq!(Db::open(&path).unwrap().schema_version().unwrap(), 19); + assert_eq!(Db::open(&path).unwrap().schema_version().unwrap(), 20); // Re-opening applies nothing and keeps the version. - assert_eq!(Db::open(&path).unwrap().schema_version().unwrap(), 19); + assert_eq!(Db::open(&path).unwrap().schema_version().unwrap(), 20); } #[test] diff --git a/docs/reference.md b/docs/reference.md index 383c25b..5a00591 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -160,6 +160,14 @@ A source is a transport (`tcp_client`, `tcp_server`, `udp` with IPv4 or IPv6 mul `_frame` (an MQTT topic is `_frame.topic`, `_frame.topic_levels[1]` its second level). Secrets are written as `${env:NAME}` and resolved when the source starts. +A source that is not complete yet (a setting left empty, no mapping rules, a mapping problem, an +unpublished schema version) saves as a **draft**: its row has `"draft": true` and `problems`, each +`{kind, stage, field?, message}`, where `stage` is the pipeline stage it belongs to (`source`, +`decode`, `map`, `static_join`, `registry`, `affiliation`, `filter`, `tracker`, `throttle`, +`output`) and `null` for the source as a whole. A draft cannot be enabled. A source without a valid +id or name is not saved (`kind: "invalid"`), and an enabled source is saved only complete. A +refusal is a 422 whose body lists the `problems` beside the `error`. + `tcp_client`, `http_poll` (`https://`), `websocket` (`wss://`) and `mqtt` (`mqtts://`) take a `tls` object: `ca_file` (trusted instead of the system roots), `cert_file` and `key_file` (a client certificate for mutual TLS), `server_name` (verify against this name instead of the host) and diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index 30ceab7..d8d8c5c 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -902,10 +902,38 @@ export interface SourceRow { raw_subject: string | null created_at_ms: number updated_at_ms: number + /** Why it may not run yet, as found when it was saved; empty when complete. */ + problems: SourceProblem[] + /** Whether it has problems: a draft cannot be enabled. */ + draft: boolean /** Live worker status; null when no worker runs it (disabled or stopped). */ status: SourceStatus | null } +/** A pipeline stage, in the order a frame meets them. */ +export type SourceStage = + | 'source' + | 'decode' + | 'map' + | 'static_join' + | 'registry' + | 'affiliation' + | 'filter' + | 'tracker' + | 'throttle' + | 'output' + +/** One reason a source may not run. */ +export interface SourceProblem { + /** `invalid`: the source cannot be saved. `incomplete`: it saves as a draft. */ + kind: 'invalid' | 'incomplete' + /** The stage it belongs to; null for the source as a whole (its id and name). */ + stage: SourceStage | null + /** The setting, where there is one (for example `host`). */ + field?: string + message: string +} + export interface MetricsResponse { source: string minutes: number From 130955934fce6faea98b25cf7560321c8fca15e4 Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Wed, 21 Oct 2026 15:38:55 -0500 Subject: [PATCH 05/16] Sources: review fixes for drafts (#104) - Enable stores what checking again finds, so a draft whose problem was resolved elsewhere (a schema published since) can be enabled without re-saving. A refused enable keeps the problems without logging a decision. - The store refuses to save problems onto an enabled source inside its write, so a concurrent enable cannot leave it running a draft. - One draft test (a non-empty JSON array) everywhere; an import's non-array problems read as none. - Transport checks run even with an empty setting; a gRPC source's protobuf compiles once. - Config import lists every problem of a source. - validate_against is gone (problems covers it); ApiError has one constructor. - The client keeps a refusal's problems on ApiError. The Add Source wizard updates a source it already saved (a draft) instead of creating it again. Co-Authored-By: Claude Opus 5.5 --- crates/ot-server/src/api.rs | 31 +++++--- crates/ot-server/src/config_backup.rs | 7 +- crates/ot-server/src/control.rs | 44 ++++-------- crates/ot-server/src/engine/bench.rs | 2 +- crates/ot-source/src/source.rs | 42 +++++------ crates/ot-store/src/backup.rs | 8 ++- crates/ot-store/src/sources.rs | 90 ++++++++++++++++++------ ui/src/api/client.ts | 7 +- ui/src/pages/sources/AddSourceWizard.tsx | 11 ++- 9 files changed, 146 insertions(+), 96 deletions(-) diff --git a/crates/ot-server/src/api.rs b/crates/ot-server/src/api.rs index ce40d04..49c4642 100644 --- a/crates/ot-server/src/api.rs +++ b/crates/ot-server/src/api.rs @@ -311,17 +311,19 @@ async fn toggle( s: AppState, id: String, enabled: bool, + problems: Option, headers: HeaderMap, ) -> Result, ApiError> { let actor = actor(&headers); let row = s - .with_db(move |db| db.set_source_enabled(&id, enabled, &actor)) + .with_db(move |db| db.set_source_enabled(&id, enabled, problems.as_ref(), &actor)) .await?; Ok(Json(with_status(&s, row).await)) } /// Enable a source: only a complete one, checked again now (the schema -/// version its mapping targets, say, may have changed since it was saved). +/// version its mapping targets, say, may have been published since it was +/// saved). What the check finds is stored, so the list shows it too. async fn enable_source( State(s): State, Path(id): Path, @@ -333,10 +335,14 @@ async fn enable_source( .await? .ok_or_else(|| ApiError::not_found("source"))?; let c = check_spec(&s, row.spec).await?; - if !c.problems.is_empty() { - return Err(ApiError::source_problems(&c.problems)); + let found = serde_json::to_value(&c.problems).map_err(|e| ApiError::internal(e.to_string()))?; + if c.problems.is_empty() { + return toggle(s, id, true, Some(found), headers).await; } - toggle(s, id, true, headers).await + // Keep what was found, for the list; the source stays as it was. + s.with_db(move |db| db.set_source_problems(&id, &found)) + .await?; + Err(ApiError::source_problems(&c.problems)) } async fn disable_source( @@ -344,7 +350,7 @@ async fn disable_source( Path(id): Path, headers: HeaderMap, ) -> Result, ApiError> { - toggle(s, id, false, headers).await + toggle(s, id, false, None, headers).await } #[derive(Deserialize)] @@ -840,8 +846,7 @@ mod tests { for (name, text) in [("aisstream", AIS), ("adsb-lol", ADSB)] { let spec: ot_source::source::SourceSpec = serde_json::from_str(text).unwrap_or_else(|e| panic!("{name}: {e}")); - spec.validate_against(Some(&schema)) - .unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!(spec.problems(Some(&schema)), [], "{name}"); } // Examples that use only the core schema. for (name, text) in [ @@ -1072,7 +1077,17 @@ mod tests { ); assert_eq!(body["problems"][0]["stage"], "map", "{body}"); + // Once the schema is published, it may run as it is. publish_example_schema(&app).await; + let (st, body) = call(&app, "POST", "/api/v1/sources/adsb-lol/enable", None).await; + assert_eq!(st, StatusCode::OK, "{body}"); + assert_eq!( + (&body["enabled"], &body["draft"]), + (&json!(true), &json!(false)) + ); + let (st, _) = call(&app, "POST", "/api/v1/sources/adsb-lol/disable", None).await; + assert_eq!(st, StatusCode::OK); + let (st, body) = call(&app, "PUT", "/api/v1/sources/adsb-lol", Some(spec.clone())).await; assert_eq!(st, StatusCode::OK, "{body}"); assert_eq!( diff --git a/crates/ot-server/src/config_backup.rs b/crates/ot-server/src/config_backup.rs index ad6c64f..0d7efd0 100644 --- a/crates/ot-server/src/config_backup.rs +++ b/crates/ot-server/src/config_backup.rs @@ -303,8 +303,8 @@ fn check(common: &Common, f: &mut ConfigFile) -> Vec { .iter() .filter(|q| s.enabled || q.kind == ot_source::source::ProblemKind::Invalid) .collect(); - if let Some(q) = refused.first() { - p.push(format!("{at}: {}", q.message)); + if !refused.is_empty() { + p.extend(refused.iter().map(|q| format!("{at}: {}", q.message))); continue; } match serde_json::to_value(&problems) { @@ -832,7 +832,8 @@ pub(crate) mod tests { "op:test", ) .unwrap(); - db.set_source_enabled("aisstream", true, "op:test").unwrap(); + db.set_source_enabled("aisstream", true, None, "op:test") + .unwrap(); db.put_app_settings(&json!({"site_name": "Garden Island"}), "op:test") .unwrap(); // Saved before the account policy was fixed: those settings are diff --git a/crates/ot-server/src/control.rs b/crates/ot-server/src/control.rs index d80b134..b65f7dc 100644 --- a/crates/ot-server/src/control.rs +++ b/crates/ot-server/src/control.rs @@ -361,47 +361,31 @@ pub struct ApiError { } impl ApiError { - pub(crate) fn internal(m: impl Into) -> Self { + fn new(status: StatusCode, m: impl Into) -> Self { Self { - status: StatusCode::INTERNAL_SERVER_ERROR, + status, message: m.into(), problems: None, } } + + pub(crate) fn internal(m: impl Into) -> Self { + Self::new(StatusCode::INTERNAL_SERVER_ERROR, m) + } pub(crate) fn not_found(m: impl Into) -> Self { - Self { - status: StatusCode::NOT_FOUND, - message: m.into(), - problems: None, - } + Self::new(StatusCode::NOT_FOUND, m) } pub(crate) fn unprocessable(m: impl Into) -> Self { - Self { - status: StatusCode::UNPROCESSABLE_ENTITY, - message: m.into(), - problems: None, - } + Self::new(StatusCode::UNPROCESSABLE_ENTITY, m) } pub(crate) fn conflict(m: impl Into) -> Self { - Self { - status: StatusCode::CONFLICT, - message: m.into(), - problems: None, - } + Self::new(StatusCode::CONFLICT, m) } pub(crate) fn unauthorized(m: impl Into) -> Self { - Self { - status: StatusCode::UNAUTHORIZED, - message: m.into(), - problems: None, - } + Self::new(StatusCode::UNAUTHORIZED, m) } pub(crate) fn forbidden(m: impl Into) -> Self { - Self { - status: StatusCode::FORBIDDEN, - message: m.into(), - problems: None, - } + Self::new(StatusCode::FORBIDDEN, m) } /// A source that cannot be saved or enabled as it is: 422, with its /// problems listed (the first is the `error`). @@ -416,11 +400,7 @@ impl ApiError { } pub(crate) fn bad_request(m: impl Into) -> Self { - Self { - status: StatusCode::BAD_REQUEST, - message: m.into(), - problems: None, - } + Self::new(StatusCode::BAD_REQUEST, m) } } diff --git a/crates/ot-server/src/engine/bench.rs b/crates/ot-server/src/engine/bench.rs index 3f91600..636e76c 100644 --- a/crates/ot-server/src/engine/bench.rs +++ b/crates/ot-server/src/engine/bench.rs @@ -241,7 +241,7 @@ pub async fn run(common: &Common, opts: BenchOptions) -> anyhow::Result<()> { }, "bench", )?; - db.set_source_enabled(&spec.id, true, "bench")?; + db.set_source_enabled(&spec.id, true, None, "bench")?; } } let mut settings = EngineSettings::default(); diff --git a/crates/ot-source/src/source.rs b/crates/ot-source/src/source.rs index fef2885..751770c 100644 --- a/crates/ot-source/src/source.rs +++ b/crates/ot-source/src/source.rs @@ -224,8 +224,8 @@ impl SourceSpec { } } - /// The first reason this source may not run, in the order the checks - /// have always run: see [`Self::problems`] for all of them. + /// The first reason this source may not run (without the extension + /// schema checks): see [`Self::problems`] for all of them. pub fn validate(&self) -> Result<(), SourceError> { match self.checks(None).into_iter().next() { Some((_, _, e)) => Err(e), @@ -233,18 +233,6 @@ impl SourceSpec { } } - /// Validate against the extension schema version the mapping targets - /// (`None` if that version is not published). - pub fn validate_against( - &self, - schema: Option<&crate::schema::ExtensionSchema>, - ) -> Result<(), SourceError> { - match self.checks(Some(schema)).into_iter().next() { - Some((_, _, e)) => Err(e), - None => Ok(()), - } - } - /// Everything that stops this source running, each with the pipeline /// stage (and, where there is one, the setting) it belongs to. Empty: /// the source is complete and may be enabled. A problem of kind @@ -311,8 +299,20 @@ impl SourceSpec { { out.push((Some(Stage::Tracker), None, SourceError::Tracker(e))); } - if let crate::codec::CodecConfig::Plugin { .. } | crate::codec::CodecConfig::Protobuf { .. } = - &self.pipeline.codec + let grpc = matches!( + self.transport, + TransportConfig::GrpcClient { .. } | TransportConfig::GrpcServer { .. } + ); + let protobuf = matches!( + self.pipeline.codec, + crate::codec::CodecConfig::Protobuf { .. } + ); + let plugin = matches!( + self.pipeline.codec, + crate::codec::CodecConfig::Plugin { .. } + ); + // A gRPC source's protobuf is compiled (once) by the check below. + if (plugin || (protobuf && !grpc)) && let Err(e) = crate::codec::Codec::new(self.pipeline.codec.clone()) { out.push(( @@ -321,8 +321,6 @@ impl SourceSpec { SourceError::Tracker(e.to_string()), )); } - // An empty setting is reported once, not again by the checks - // that read it. let missing = self.transport.missing(); for (field, what) in &missing { out.push(( @@ -331,15 +329,11 @@ impl SourceSpec { SourceError::Transport(format!("{what} is not set")), )); } - if missing.is_empty() - && let Err(e) = self.transport.check() - { + if let Err(e) = self.transport.check() { out.push((Some(Stage::Source), None, SourceError::Transport(e))); } // gRPC carries protobuf: the method must fit the codec's message. - if let TransportConfig::GrpcClient { .. } | TransportConfig::GrpcServer { .. } = - &self.transport - { + if grpc { match crate::transport::ProtoContext::of(&self.pipeline.codec) { Err(e) => out.push((Some(Stage::Decode), None, SourceError::Transport(e))), Ok(None) => out.push(( diff --git a/crates/ot-store/src/backup.rs b/crates/ot-store/src/backup.rs index d78a823..4cb6380 100644 --- a/crates/ot-store/src/backup.rs +++ b/crates/ot-store/src/backup.rs @@ -680,7 +680,11 @@ impl Db { revision, if s.created_at_ms > 0 { s.created_at_ms } else { now }, now, - s.problems.to_string(), + if s.problems.is_array() { + s.problems.to_string() + } else { + "[]".to_owned() + }, ], )?; tx.execute( @@ -776,7 +780,7 @@ mod tests { "op:test", ) .unwrap(); - db.set_source_enabled("ais", true, "op:test").unwrap(); + db.set_source_enabled("ais", true, None, "op:test").unwrap(); db.set_raw_output("ais", Some("opentrack.raw.ais"), "op:test") .unwrap(); db.put_schema_draft(&[json!({"key": "hull", "type": "string"})], None, "op:test") diff --git a/crates/ot-store/src/sources.rs b/crates/ot-store/src/sources.rs index 61da918..ba263a9 100644 --- a/crates/ot-store/src/sources.rs +++ b/crates/ot-store/src/sources.rs @@ -77,10 +77,27 @@ fn row(r: &rusqlite::Row<'_>) -> rusqlite::Result<(SourceRow, String, String)> { fn finish((mut row, spec, problems): (SourceRow, String, String)) -> Result { row.spec = serde_json::from_str(&spec)?; row.problems = serde_json::from_str(&problems)?; - row.draft = row.problems.as_array().is_some_and(|a| !a.is_empty()); + row.draft = is_draft(&row.problems); Ok(row) } +/// Whether a source with these problems is a draft: any problem listed. +fn is_draft(problems: &Value) -> bool { + problems.as_array().is_some_and(|a| !a.is_empty()) +} + +/// A source's stored problems, if it exists. +fn stored_problems(tx: &rusqlite::Transaction<'_>, id: &str) -> Result> { + tx.query_row( + "SELECT enabled, problems FROM sources WHERE id = ?1", + [id], + |r| Ok((r.get::<_, bool>(0)?, r.get::<_, String>(1)?)), + ) + .optional()? + .map(|(enabled, p)| Ok((enabled, serde_json::from_str(&p)?))) + .transpose() +} + impl Db { pub fn list_sources(&self) -> Result> { let mut stmt = self @@ -102,10 +119,19 @@ impl Db { .transpose() } - /// Create or update a source. Returns the saved row. + /// Create or update a source. Returns the saved row. An enabled source + /// is saved only complete (an edit never stops a running source). pub fn put_source(&mut self, w: &SourceWrite<'_>, actor: &str) -> Result { let before = self.get_source(w.id)?; self.write(|tx| { + if is_draft(w.problems) + && stored_problems(tx, w.id)?.is_some_and(|(enabled, _)| enabled) + { + return Err(StoreError::Conflict(format!( + "source {} is enabled; it can be saved only complete", + w.id + ))); + } let now = now_ms(); let (op, revision) = match &before { None => ("create_source", 1), @@ -152,14 +178,32 @@ impl Db { .ok_or_else(|| StoreError::NotFound(format!("source {}", w.id))) } + /// Enable or disable a source. `problems`, when given, is what checking + /// it again found now (a schema published since it was saved, say), + /// stored first: a draft cannot be enabled. pub fn set_source_enabled( &mut self, id: &str, enabled: bool, + problems: Option<&Value>, actor: &str, ) -> Result { self.write(|tx| { let now = now_ms(); + let Some((_, stored)) = stored_problems(tx, id)? else { + return Err(StoreError::NotFound(format!("source {id}"))); + }; + if let Some(p) = problems { + tx.execute( + "UPDATE sources SET problems = ?2 WHERE id = ?1", + params![id, p.to_string()], + )?; + } + if enabled && is_draft(problems.unwrap_or(&stored)) { + return Err(StoreError::Conflict(format!( + "source {id} is a draft; complete it before enabling it" + ))); + } let op = if enabled { "enable_source" } else { @@ -167,22 +211,6 @@ impl Db { }; let d = Decision::new(actor, op).evidence(json!({ "source": id })); record_decision(tx, &d, now)?; - let draft: Option = tx - .query_row( - "SELECT problems != '[]' FROM sources WHERE id = ?1", - [id], - |r| r.get(0), - ) - .optional()?; - match draft { - None => return Err(StoreError::NotFound(format!("source {id}"))), - Some(true) if enabled => { - return Err(StoreError::Conflict(format!( - "source {id} is a draft; complete it before enabling it" - ))); - } - Some(_) => {} - } tx.execute( "UPDATE sources SET enabled = ?2, updated_at_ms = ?3 WHERE id = ?1", params![id, enabled, now], @@ -193,6 +221,21 @@ impl Db { .ok_or_else(|| StoreError::NotFound(format!("source {id}"))) } + /// Store what checking a source again found, without a new revision + /// (nothing about the source itself changed). + pub fn set_source_problems(&mut self, id: &str, problems: &Value) -> Result<()> { + self.write(|tx| { + let n = tx.execute( + "UPDATE sources SET problems = ?2 WHERE id = ?1", + params![id, problems.to_string()], + )?; + if n == 0 { + return Err(StoreError::NotFound(format!("source {id}"))); + } + Ok(()) + }) + } + /// Set or clear the raw-output subject. Setting one records the admin's /// consent to publish this source's raw tracks. pub fn set_raw_output( @@ -316,17 +359,18 @@ mod tests { assert!(row.draft); assert_eq!(row.problems, problems); assert!(matches!( - db.set_source_enabled("ais", true, "op:test"), + db.set_source_enabled("ais", true, None, "op:test"), Err(StoreError::Conflict(_)) )); assert!(!db.get_source("ais").unwrap().unwrap().enabled); // Turning a draft off is fine; completing it lets it run. - db.set_source_enabled("ais", false, "op:test").unwrap(); + db.set_source_enabled("ais", false, None, "op:test") + .unwrap(); let row = db.put_source(&write(&spec), "op:test").unwrap(); assert!(!row.draft); assert_eq!(row.problems, json!([])); assert!( - db.set_source_enabled("ais", true, "op:test") + db.set_source_enabled("ais", true, None, "op:test") .unwrap() .enabled ); @@ -384,7 +428,7 @@ mod tests { let v1 = db.sources_version().unwrap(); assert_ne!(v0, v1); - let row = db.set_source_enabled("ais", true, "op:test").unwrap(); + let row = db.set_source_enabled("ais", true, None, "op:test").unwrap(); assert!(row.enabled); let revs = db.source_revisions("ais").unwrap(); assert_eq!(revs.len(), 2); @@ -417,7 +461,7 @@ mod tests { ] ); assert!(matches!( - db.set_source_enabled("nope", true, "x"), + db.set_source_enabled("nope", true, None, "x"), Err(StoreError::NotFound(_)) )); } diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index d8d8c5c..b773ef6 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -388,9 +388,12 @@ export interface ConfigImported { export class ApiError extends Error { readonly status: number - constructor(status: number, message: string) { + /** For a source that cannot be saved or enabled: everything wrong with it, by stage. */ + readonly problems: SourceProblem[] + constructor(status: number, message: string, problems: SourceProblem[] = []) { super(message) this.status = status + this.problems = problems } } @@ -423,7 +426,7 @@ async function request(method: string, path: string, body?: unknown): Promise }) const text = await res.text() const parsed = text ? JSON.parse(text) : {} - if (!res.ok) throw new ApiError(res.status, parsed.error ?? res.statusText) + if (!res.ok) throw new ApiError(res.status, parsed.error ?? res.statusText, parsed.problems ?? []) return parsed as T } diff --git a/ui/src/pages/sources/AddSourceWizard.tsx b/ui/src/pages/sources/AddSourceWizard.tsx index 30c7edf..d7195d1 100644 --- a/ui/src/pages/sources/AddSourceWizard.tsx +++ b/ui/src/pages/sources/AddSourceWizard.tsx @@ -122,11 +122,20 @@ export function AddSourceWizard({ onDone, onCancel }: { onDone: (id: string) => } } + // An incomplete source is saved as a draft, which cannot be enabled: once + // it exists, Save updates it instead of creating it again. + const [created, setCreated] = useState(false) + const save = async () => { setSaving(true) setSaveError(null) try { - await api.createSource(spec) + if (created) { + await api.saveSource(spec) + } else { + await api.createSource(spec) + setCreated(true) + } if (enable) await api.setEnabled(spec.id, true) toast({ variant: 'success', message: `Source ${spec.id} saved${enable ? ' and enabled' : ''}.` }) onDone(spec.id) From d74647abf81b560bef9ed8a7c8193ee8102fcd7d Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Wed, 21 Oct 2026 16:15:04 -0500 Subject: [PATCH 06/16] Sources: a flow editor for creating and editing a source (#104) Phases 3 to 5 of #104: - sourceFlow.ts: a source as steps in the pipeline's fixed order, which connect themselves; the palette's transports, codecs (with codec plugins), stages and outputs; adding a stage puts it in its place, removing one closes the gap; Source, Decode, Map and Output stay. - SourceEditor at #sources/: the palette and the steps on the left; the selected step's settings and samples, or the source's own settings, status and history, on the right; problems on their steps, live from the dry run; Enabled, Save, Delete and a whole-spec JSON view. - Capture samples on the Source step; the detected decode settings and the suggested mapping are offered, never applied unasked. - Add source asks for a name, id and transport, and saves a draft with its four steps. - The transport form shows only the Source or the Decode step's part, and gains the file transport. - The dry run (/sources/validate) runs a draft as far as its decode, map and tracker stages allow, and lists the problems. - The wizard, the source drawer, the pipeline designer and their views are gone. Co-Authored-By: Claude Opus 5.5 --- crates/ot-server/src/api.rs | 84 +++- ui/src/api/client.ts | 3 + ui/src/index.css | 166 +++++-- ui/src/lib/sourceState.ts | 7 +- ui/src/lib/usePreview.ts | 44 -- ui/src/pages/sources/AddSourceWizard.tsx | 343 --------------- ui/src/pages/sources/MappingStudio.tsx | 88 ---- ui/src/pages/sources/NewSource.tsx | 94 ++++ ui/src/pages/sources/PipelineFlow.tsx | 34 -- ui/src/pages/sources/PipelineView.tsx | 120 ----- ui/src/pages/sources/PreviewResults.tsx | 121 ----- ui/src/pages/sources/SourceDetail.tsx | 244 ---------- ui/src/pages/sources/SourceEditor.tsx | 416 ++++++++++++++++++ ui/src/pages/sources/SourceStatus.tsx | 107 +++++ ui/src/pages/sources/SourcesPage.tsx | 59 +-- ui/src/pages/sources/TransportForm.tsx | 146 +++--- .../sources/designer/PipelineDesigner.tsx | 321 -------------- ui/src/pages/sources/designer/StageForms.tsx | 65 +-- ui/src/pages/sources/flow/StepNode.tsx | 47 ++ ui/src/pages/sources/flow/StepPanel.tsx | 269 +++++++++++ ui/src/pages/sources/flow/sourceFlow.test.ts | 128 ++++++ ui/src/pages/sources/flow/sourceFlow.ts | 292 ++++++++++++ ui/src/pages/sources/flow/stepIcons.tsx | 76 ++++ 23 files changed, 1744 insertions(+), 1530 deletions(-) delete mode 100644 ui/src/lib/usePreview.ts delete mode 100644 ui/src/pages/sources/AddSourceWizard.tsx delete mode 100644 ui/src/pages/sources/MappingStudio.tsx create mode 100644 ui/src/pages/sources/NewSource.tsx delete mode 100644 ui/src/pages/sources/PipelineFlow.tsx delete mode 100644 ui/src/pages/sources/PipelineView.tsx delete mode 100644 ui/src/pages/sources/PreviewResults.tsx delete mode 100644 ui/src/pages/sources/SourceDetail.tsx create mode 100644 ui/src/pages/sources/SourceEditor.tsx create mode 100644 ui/src/pages/sources/SourceStatus.tsx delete mode 100644 ui/src/pages/sources/designer/PipelineDesigner.tsx create mode 100644 ui/src/pages/sources/flow/StepNode.tsx create mode 100644 ui/src/pages/sources/flow/StepPanel.tsx create mode 100644 ui/src/pages/sources/flow/sourceFlow.test.ts create mode 100644 ui/src/pages/sources/flow/sourceFlow.ts create mode 100644 ui/src/pages/sources/flow/stepIcons.tsx diff --git a/crates/ot-server/src/api.rs b/crates/ot-server/src/api.rs index 49c4642..519a185 100644 --- a/crates/ot-server/src/api.rs +++ b/crates/ot-server/src/api.rs @@ -77,6 +77,7 @@ pub(crate) fn actor(headers: &HeaderMap) -> String { /// filled in), the published extension schema its mapping targets (if it /// is published) and everything that stops it running. A spec that does /// not parse, or is not allowed at all, is refused here. +#[derive(Clone)] struct Checked { spec: SourceSpec, normalised: Value, @@ -100,12 +101,31 @@ impl Checked { } } - /// Refuse it unless it is complete (it may run). - fn complete(self) -> Result<(SourceSpec, Value, ExtensionSchema), ApiError> { - match self.schema { - Some(schema) if self.problems.is_empty() => Ok((self.spec, self.normalised, schema)), - _ => Err(ApiError::source_problems(&self.problems)), - } + /// Its pipeline, to dry-run over samples: a draft runs too, as far as + /// its decode, map and tracker stages allow (its transport is not used, + /// and without a published schema its extension fields are not checked). + fn dry_run(&self) -> Result { + self.clone().savable()?; + let pipeline = + Pipeline::new(self.spec.id.clone(), self.spec.pipeline.clone()).map_err(|_| { + let blocking: Vec = self + .problems + .iter() + .filter(|p| { + matches!(p.stage, Some(Stage::Decode | Stage::Map | Stage::Tracker)) + }) + .cloned() + .collect(); + ApiError::source_problems(if blocking.is_empty() { + &self.problems + } else { + &blocking + }) + })?; + Ok(match &self.schema { + Some(schema) => pipeline.with_schema(schema.clone()), + None => pipeline, + }) } } @@ -473,13 +493,22 @@ struct ValidateBody { trace: Option, } -/// Validate a source spec and optionally dry-run sample frames through its -/// pipeline, using the live registry. Nothing is saved or published. +/// Check a source spec and optionally dry-run sample frames through its +/// pipeline, using the live registry. Nothing is saved or published. The +/// answer lists the spec's `problems` (`valid` when there are none); a draft +/// is dry-run as far as it can be. async fn validate_source( State(s): State, Json(body): Json, ) -> Result, ApiError> { - let (spec, normalised, schema) = check_spec(&s, body.spec).await?.complete()?; + let checked = check_spec(&s, body.spec).await?; + let mut pipeline = checked.dry_run()?; + let Checked { + normalised, + problems, + .. + } = checked; + let valid = problems.is_empty(); let mut frames: Vec = body .samples .iter() @@ -498,7 +527,9 @@ async fn validate_source( })); } if frames.is_empty() { - return Ok(Json(json!({ "valid": true, "spec": normalised }))); + return Ok(Json( + json!({ "valid": valid, "problems": problems, "spec": normalised }), + )); } let rows = s.with_db(|db| db.registry_rows()).await?; let registry: BTreeMap<(String, String), ot_source::registry::RegistryEntry> = rows @@ -515,9 +546,6 @@ async fn validate_source( ) }) .collect(); - let mut pipeline = Pipeline::new(spec.id.clone(), spec.pipeline.clone()) - .map_err(|e| ApiError::unprocessable(e.to_string()))? - .with_schema(schema); let mut trace = body.trace.filter(|n| *n > 0).map(Trace::new); let mut observations = Vec::new(); let mut errors = Vec::new(); @@ -542,7 +570,8 @@ async fn validate_source( observations.extend(flushed.observations); let counts: BTreeMap = pipeline.take_counts().pairs().into_iter().collect(); let mut body = json!({ - "valid": true, + "valid": valid, + "problems": problems, "counts": counts, "errors": errors, "observations": observations, @@ -1308,6 +1337,33 @@ mod tests { assert_eq!(o["ext"]["registry"]["scheme"], "elnot"); assert_eq!(o["ext"]["registry"]["grade"], "name"); assert_eq!(o["classification"]["cot_type"], "a-f-S-C-A"); + assert_eq!(body["valid"], true, "{body}"); + + // A draft runs as far as it can: its transport is not used. + let mut draft = spec.clone(); + draft["transport"] = json!({"type": "udp", "bind": ""}); + let (st, body) = call( + &app, + "POST", + "/api/v1/sources/validate", + Some(json!({"spec": draft, "samples": [frame]})), + ) + .await; + assert_eq!(st, StatusCode::OK, "{body}"); + assert_eq!(body["valid"], false, "{body}"); + assert_eq!(body["problems"][0]["field"], "bind", "{body}"); + assert_eq!(body["observations"].as_array().unwrap().len(), 1, "{body}"); + // One whose mapping cannot run says why, on its stage. + draft["pipeline"]["mapping"]["rules"] = json!([]); + let (st, body) = call( + &app, + "POST", + "/api/v1/sources/validate", + Some(json!({"spec": draft, "samples": [frame]})), + ) + .await; + assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); + assert_eq!(body["problems"][0]["stage"], "map", "{body}"); redis.purge_namespace().await.unwrap(); } diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index b773ef6..8b90808 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -1001,7 +1001,10 @@ export interface PreviewObservation extends Observation { } export interface PreviewResult { + /** Whether the source is complete; a draft is dry-run as far as it can be. */ valid: boolean + /** What stops the source running, each on its stage. */ + problems?: SourceProblem[] counts?: Record errors?: string[] observations?: PreviewObservation[] diff --git a/ui/src/index.css b/ui/src/index.css index 4eb131c..6c1a6ff 100644 --- a/ui/src/index.css +++ b/ui/src/index.css @@ -561,24 +561,6 @@ body { font-size: 11px; } -/* Pipeline designer (a modal): stage flowchart, the stage's form, the live preview. */ -.designer { - display: grid; - grid-template-columns: 220px minmax(0, 1fr) minmax(320px, 420px); - gap: var(--space-lg); - align-items: start; - min-height: 60vh; -} - -/* The live preview fills the modal's height (its max, less the title bar and padding) and its - samples scroll inside it; it stays in view if a long stage form scrolls the modal. */ -.designer-preview { - position: sticky; - top: 0; - height: calc(100vh - 160px - 2 * var(--space-lg)); - min-height: 0; -} - .stage-samples { flex: 1; min-height: 0; @@ -618,17 +600,6 @@ body { padding-top: 4px; } -@media (max-width: 1300px) { - .designer { - grid-template-columns: 220px minmax(0, 1fr); - } - .designer-preview { - grid-column: 1 / -1; - position: static; - height: 70vh; - } -} - .value-editor { display: flex; flex-direction: column; @@ -933,3 +904,140 @@ body { .symbol-designer { gap: var(--space-md); } + +/* Source editor (#104): the steps on the left, the selected step's settings and samples (or the + source's own settings, status and history) on the right. It fills the page below the tabs. */ +.source-editor { + display: flex; + flex-direction: column; + gap: var(--space-sm); + height: calc(100vh - 120px); + min-height: 480px; + padding: var(--space-sm) var(--space-md); +} + +.source-editor-head { + display: flex; + align-items: center; + gap: var(--space-sm); + flex-wrap: wrap; +} + +.source-editor-general { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + max-width: 50vw; +} + +.source-editor-body { + flex: 1; + min-height: 0; + display: grid; + grid-template-columns: minmax(380px, 440px) minmax(0, 1fr); + gap: var(--space-md); +} + +.source-editor-aside { + min-height: 0; + overflow-y: auto; + padding-right: var(--space-xs); +} + +.source-editor-json { + flex: 1; + min-height: 0; + overflow: auto; +} + +/* Settings beside their samples; stacked on narrower screens. */ +.step-panel { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(300px, 420px); + gap: var(--space-lg); + align-items: start; +} + +.step-samples { + position: sticky; + top: 0; + max-height: calc(100vh - 180px); + min-height: 0; +} + +.step-settings { + border: 0; + margin: 0; + padding: 0; + min-width: 0; +} + +@media (max-width: 1400px) { + .step-panel { + grid-template-columns: minmax(0, 1fr); + } + .step-samples { + position: static; + max-height: 60vh; + } +} + +/* A step on the canvas. */ +.flow-step { + width: 240px; + padding: 8px 10px; + border-radius: var(--radius-md); + background: var(--color-bg-secondary); + font-family: var(--font-sans); + display: flex; + flex-direction: column; + gap: 4px; +} + +.flow-step-head { + display: flex; + align-items: center; + gap: 6px; +} + +.flow-step-icon { + display: flex; + color: var(--color-text-secondary); +} + +.flow-step-title { + font-size: 12px; + font-weight: 600; + color: var(--color-text-primary); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.flow-step-kind { + margin-left: auto; + font-size: 9px; + color: var(--color-text-secondary); + text-transform: uppercase; + letter-spacing: 0.08em; +} + +.step-panel .flow-step-kind { + margin-left: 0; +} + +.flow-step-summary { + font-size: 10px; + font-family: var(--font-mono); + color: var(--color-text-secondary); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.flow-step-problem { + font-size: 10px; + color: var(--color-destructive); + line-height: 1.3; +} diff --git a/ui/src/lib/sourceState.ts b/ui/src/lib/sourceState.ts index 3975afd..96b468c 100644 --- a/ui/src/lib/sourceState.ts +++ b/ui/src/lib/sourceState.ts @@ -1,8 +1,9 @@ import type { BadgeColor } from '@kineticlogic/staresdk' import type { SourceRow } from '../api/client' -/** Running / starting / stopped / failing, from config plus the worker's live status. */ +/** Draft / disabled / running / starting / failing, from config plus the worker's live status. */ export function sourceState(s: SourceRow): { label: string; color: BadgeColor } { + if (s.draft) return { label: 'draft', color: 'grey' } if (!s.enabled) return { label: 'disabled', color: 'grey' } if (!s.status) return { label: 'starting', color: 'warning' } if (s.status.not_started) return { label: 'not started', color: 'danger' } @@ -12,9 +13,9 @@ export function sourceState(s: SourceRow): { label: string; color: BadgeColor } /** What each source state means, for an info tip. */ export const SOURCE_STATES_INFO = - 'disabled: switched off, not running. starting: enabled, its worker has not reported yet. running: connected (or listening) and reading. ' + + 'draft: not complete yet, so it cannot be turned on; its problems show on its steps. disabled: switched off, not running. starting: enabled, its worker has not reported yet. running: connected (or listening) and reading. ' + 'connecting: not connected yet, no errors so far. failing: not connected and its link has hit errors; it keeps retrying. ' + - 'not started: the worker refuses to run it as configured (a listener that does not authenticate its senders, say). See the error on its Status tab.' + 'not started: the worker refuses to run it as configured (a listener that does not authenticate its senders, say). See the error in its status.' /** What each pipeline counter means, for an info tip. */ export const PIPELINE_COUNTS_INFO = diff --git a/ui/src/lib/usePreview.ts b/ui/src/lib/usePreview.ts deleted file mode 100644 index 97c8a01..0000000 --- a/ui/src/lib/usePreview.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { useEffect, useRef, useState } from 'react' -import { api, type PreviewResult, type SourceSpec } from '../api/client' -import { errorMessage } from './format' -import { useCan } from '../auth/context' - -const PREVIEW_DELAY_MS = 600 - -/** - * A dry run of `spec` over `sampleSourceId`'s stored probe samples, re-run shortly after the spec - * stops changing. Nothing is saved or published. `trace` > 0 also follows that many samples stage by - * stage. - */ -export function usePreview(spec: SourceSpec, sampleSourceId: string, nonce = 0, trace = 0) { - const [preview, setPreview] = useState(null) - const [error, setError] = useState(null) - const [running, setRunning] = useState(false) - const seq = useRef(0) - const canAdmin = useCan('admin') - - useEffect(() => { - if (!canAdmin) return - const id = ++seq.current - const t = setTimeout(() => { - setRunning(true) - api - .preview(spec, sampleSourceId, [], trace) - .then((r) => { - if (id !== seq.current) return - setPreview(r) - setError(null) - }) - .catch((e) => { - if (id !== seq.current) return - setError(errorMessage(e)) - }) - .finally(() => id === seq.current && setRunning(false)) - }, PREVIEW_DELAY_MS) - return () => clearTimeout(t) - }, [spec, sampleSourceId, nonce, canAdmin, trace]) - - // The dry run is an admin's (it can reach the source's own transport). - if (!canAdmin) return { preview: null, error: 'The preview needs the admin role.', running: false } - return { preview, error, running } -} diff --git a/ui/src/pages/sources/AddSourceWizard.tsx b/ui/src/pages/sources/AddSourceWizard.tsx deleted file mode 100644 index d7195d1..0000000 --- a/ui/src/pages/sources/AddSourceWizard.tsx +++ /dev/null @@ -1,343 +0,0 @@ -import { useMemo, useState } from 'react' -import { TbArrowLeft, TbArrowRight, TbCheck, TbRadar, TbX } from 'react-icons/tb' -import { Badge, Button, CollapsiblePanel, DataTable, Input, Label, Stepper, Toggle, useToast, type DataTableColumn, type StepStatus } from '@kineticlogic/staresdk' -import { CodeEditor } from '@kineticlogic/staresdk/code-editor' -import { api, type FieldStat, type ProbeResult, type SourceSpec } from '../../api/client' -import { InfoTip } from '../../components/InfoTip' -import { errorMessage } from '../../lib/format' -import { MappingStudio } from './MappingStudio' -import { needsAcceptance } from '../../lib/senderAuth' -import { SenderAuth } from './SenderAuth' -import { TransportForm } from './TransportForm' - -const STEPS = [ - { id: 'connect', label: 'Connect' }, - { id: 'probe', label: 'Probe' }, - { id: 'map', label: 'Map' }, - { id: 'review', label: 'Review' }, -] as const - -const ID_RE = /^[a-z0-9_-]{1,64}$/ - -const FIELD_COLUMNS: DataTableColumn[] = [ - { key: 'path', header: 'Field', mono: true, width: '32%', render: (f) => f.path, sortValue: (f) => f.path }, - { key: 'types', header: 'Types', width: '16%', render: (f) => Object.keys(f.types).join(', ') }, - { - key: 'presence', - header: 'Present', - align: 'right', - width: '11%', - render: (f) => `${Math.round(f.presence * 100)}%`, - sortValue: (f) => f.presence, - }, - { - key: 'distinct', - header: 'Distinct', - align: 'right', - width: '11%', - render: (f) => `${f.distinct}${f.distinct_capped ? '+' : ''}`, - sortValue: (f) => f.distinct, - }, - { - key: 'samples', - header: 'Samples', - mono: true, - render: (f) => f.samples.map((s) => (typeof s === 'string' ? s : JSON.stringify(s))).join(' · '), - }, -] - -function connectProblems(spec: SourceSpec): string[] { - const p: string[] = [] - if (!ID_RE.test(spec.id)) p.push('an id of a-z, 0-9, - or _') - if (!spec.name.trim()) p.push('a name') - const t = spec.transport - if ((t.type === 'http_poll' || t.type === 'websocket') && !String(t.url ?? '').trim()) p.push('a URL') - if (t.type === 'mqtt') { - if (!/^mqtts?:\/\/./.test(String(t.url ?? '').trim())) p.push('a broker URL (mqtt:// or mqtts://)') - if (!Array.isArray(t.topics) || t.topics.length === 0) p.push('at least one topic') - if (t.clean_session === false && !String(t.client_id ?? '').trim()) p.push('a client id for a persistent session') - } - if (t.type === 'tcp_client' && (!String(t.host ?? '').trim() || !t.port)) p.push('a host and port') - if ((t.type === 'tcp_server' || t.type === 'udp' || t.type === 'grpc_server') && !String(t.bind ?? '').trim()) p.push('a bind address') - const c = spec.pipeline.codec - if (t.type === 'grpc_client' || t.type === 'grpc_server' || c.type === 'protobuf') { - if (c.type !== 'protobuf') p.push('the Protobuf codec (gRPC carries protobuf)') - else { - if (Object.keys((c.files as Record | undefined) ?? {}).length === 0) p.push("the producer's .proto files") - if (!String(c.message ?? '').trim()) p.push('the message each frame holds') - } - } - if (t.type === 'grpc_client') { - if (!/^https?:\/\/./.test(String(t.url ?? '').trim())) p.push('a URL (http:// or https://)') - if (!String(t.method ?? '').trim()) p.push('the method to call') - } - if (needsAcceptance(spec)) p.push(t.type === 'udp' ? 'the acceptance of unauthenticated senders (UDP)' : 'sender authentication, or its risk accepted') - return p -} - -/** Onboard a feed: connect, probe, map, review. No code. */ -export function AddSourceWizard({ onDone, onCancel }: { onDone: (id: string) => void; onCancel: () => void }) { - const { toast } = useToast() - const [step, setStep] = useState(0) - const [spec, setSpec] = useState({ - id: '', - name: '', - transport: { type: 'http_poll', url: '', interval_secs: 5 }, - pipeline: { codec: { type: 'json' }, mapping: { schema_version: 1, rules: [] } }, - }) - const [probe, setProbe] = useState(null) - const [probing, setProbing] = useState(false) - const [maxFrames, setMaxFrames] = useState(50) - const [maxSecs, setMaxSecs] = useState(20) - const [enable, setEnable] = useState(true) - const [saving, setSaving] = useState(false) - const [saveError, setSaveError] = useState(null) - - const problems = connectProblems(spec) - const probed = !!probe && probe.records > 0 - const statuses: StepStatus[] = STEPS.map((_, i) => - i === step ? 'current' : i < step ? (i === 1 && !probed ? 'error' : 'complete') : 'upcoming', - ) - - const runProbe = async () => { - setProbing(true) - try { - const r = await api.probe({ - transport: spec.transport, - codec: spec.pipeline.codec, - max_frames: maxFrames, - max_secs: maxSecs, - save_as: spec.id, - }) - setProbe(r) - // Adopt the detected codec settings and the suggested mapping. - setSpec((s) => ({ ...s, pipeline: { ...s.pipeline, codec: r.codec, mapping: r.suggestion.mapping } })) - if (r.frames === 0) { - toast({ variant: 'warning', message: r.link_error ?? r.hint ?? 'No frames arrived within the probe window.' }) - } - } catch (e) { - toast({ variant: 'error', title: 'Probe failed', message: errorMessage(e) }) - } finally { - setProbing(false) - } - } - - // An incomplete source is saved as a draft, which cannot be enabled: once - // it exists, Save updates it instead of creating it again. - const [created, setCreated] = useState(false) - - const save = async () => { - setSaving(true) - setSaveError(null) - try { - if (created) { - await api.saveSource(spec) - } else { - await api.createSource(spec) - setCreated(true) - } - if (enable) await api.setEnabled(spec.id, true) - toast({ variant: 'success', message: `Source ${spec.id} saved${enable ? ' and enabled' : ''}.` }) - onDone(spec.id) - } catch (e) { - setSaveError(errorMessage(e)) - } finally { - setSaving(false) - } - } - - const specText = useMemo(() => JSON.stringify(spec, null, 2), [spec]) - const canNext = step === 0 ? problems.length === 0 : step === 1 ? probed : true - - return ( - -
-
- ({ ...s, status: statuses[i] }))} - onStepClick={(id) => setStep(STEPS.findIndex((s) => s.id === id))} - /> - -
- - {step === 0 && ( -
-
-
-
- - - The source's permanent key: up to 64 of a-z, 0-9, - and _. Its tracks, history and samples are filed under it, so it - cannot be changed after saving. - -
- setSpec({ ...spec, id: e.target.value.trim().toLowerCase() })} - autoComplete="off" - spellCheck={false} - /> -
-
-
- - What operators see for this source in lists and track details. You can rename it at any time. -
- setSpec({ ...spec, name: e.target.value })} autoComplete="off" /> -
-
- setSpec({ ...spec, transport })} - onCodec={(codec) => setSpec({ ...spec, pipeline: { ...spec.pipeline, codec } })} - /> - - {problems.length > 0 && Still needed: {problems.join(', ')}.} -
- )} - - {step === 1 && ( -
-
-
-
- - The probe stops after this many frames (messages) arrive, or at Max seconds, whichever comes first. 1 to 2000. -
- setMaxFrames(Number(e.target.value) || 1)} /> -
-
-
- - The longest the probe listens, in seconds, if Max frames has not arrived first. 1 to 120. -
- setMaxSecs(Number(e.target.value) || 1)} /> -
- - - Connects with the settings above, captures frames and infers the schema. Captured frames are kept for - the mapping preview. - -
- {probe && ( - <> -
- 0 ? 'blue' : 'danger'} size="sm"> - frames {probe.frames} - - - records {probe.records} - - {probe.decode_errors > 0 && ( - - decode errors {probe.decode_errors} - - )} - - {probe.seconds}s - - - Frames: messages received. Records: what the codec decoded from them (one frame can hold many; at most 5000 are analysed). - Decode errors: frames the codec could not read. Then the time the probe listened. Up to 200 captured frames are kept for the - mapping preview. - - {probe.suggested_records_path && ( - - records found under {probe.suggested_records_path} - - )} -
- {probe.link_error &&
{probe.link_error}
} - {probe.hint &&
{probe.hint}
} - {probe.last_decode_error &&
{probe.last_decode_error}
} -

- Inferred fields - - Every leaf field seen in the decoded records; [*] means each element of an array. Present: the share of records with a non-null - value. Distinct: different values seen; a + means counting stopped at 1000. - -

- f.path} - maxHeight={300} - defaultSort={{ key: 'presence', direction: 'desc' }} - empty="No fields inferred." - /> - {probe.sample_frames[0] && ( - <> -

First frame

- {probe.sample_frames[0].meta?.topic !== undefined && ( - - topic {String(probe.sample_frames[0].meta.topic)}, available to the - mapping as _frame.topic and{' '} - _frame.topic_levels[n] - - )} - 
{probe.sample_frames[0].text}
- - )} - - )} -
- )} - - {step === 2 && ( - - )} - - {step === 3 && ( -
- - - {saveError &&
{saveError}
} -
- )} - -
- {step > 0 && ( - - )} - {step < STEPS.length - 1 ? ( - - ) : ( - - )} -
-
- - ) -} diff --git a/ui/src/pages/sources/MappingStudio.tsx b/ui/src/pages/sources/MappingStudio.tsx deleted file mode 100644 index 2e08fa8..0000000 --- a/ui/src/pages/sources/MappingStudio.tsx +++ /dev/null @@ -1,88 +0,0 @@ -import { useEffect, useMemo, useRef, useState } from 'react' -import { Badge } from '@kineticlogic/staresdk' -import { CodeEditor } from '@kineticlogic/staresdk/code-editor' -import type { Proposal, SourceSpec } from '../../api/client' -import { usePreview } from '../../lib/usePreview' -import { PreviewResults } from './PreviewResults' - -interface Props { - spec: SourceSpec - onChange: (spec: SourceSpec) => void - /** Source id whose stored probe samples the preview replays. */ - sampleSourceId: string - proposals?: Proposal[] - missing?: string[] - /** Show the preview's map (the onboarding wizard does; a source's Pipeline view does not). */ - showMap?: boolean -} - -/** - * Mapping studio: edit the pipeline (codec, mapping, enrich stages) as JSON and see the result - * live against the captured samples. - */ -export function MappingStudio({ spec, onChange, sampleSourceId, proposals, missing, showMap = true }: Props) { - const [text, setText] = useState(() => JSON.stringify(spec.pipeline, null, 2)) - const [parseError, setParseError] = useState(false) - - // Adopt external spec changes (e.g. a new probe) without clobbering typing. - const external = useMemo(() => JSON.stringify(spec.pipeline, null, 2), [spec.pipeline]) - const lastEmitted = useRef(external) - useEffect(() => { - if (external !== lastEmitted.current) { - setText(external) - lastEmitted.current = external - } - }, [external]) - - const edit = (next: string) => { - setText(next) - try { - const pipeline = JSON.parse(next) - setParseError(false) - const updated = { ...spec, pipeline } - lastEmitted.current = JSON.stringify(pipeline, null, 2) - onChange(updated) - } catch { - setParseError(true) - } - } - - const { preview, error: previewError, running } = usePreview(spec, sampleSourceId) - - return ( -
-
- {proposals && proposals.length > 0 && ( -
-

Suggested from the probe

-
    - {proposals.map((p) => ( -
  • - - {Math.round(p.confidence * 100)}% - - - {p.target} ← {p.reason} - -
    • - ))} -
-
- )} - {missing && missing.length > 0 && ( -
Not found in the samples, map these by hand: {missing.join(', ')}
- )} - - {parseError &&
The JSON does not parse yet; the preview shows the last valid version.
} -
-
-
-

Live preview

- {running ? 'running…' : `samples of ${sampleSourceId}`} -
- {previewError &&
{previewError}
} - {preview ? : !previewError && Running the first preview…} -
-
- ) -} diff --git a/ui/src/pages/sources/NewSource.tsx b/ui/src/pages/sources/NewSource.tsx new file mode 100644 index 0000000..e840c8a --- /dev/null +++ b/ui/src/pages/sources/NewSource.tsx @@ -0,0 +1,94 @@ +import { useState } from 'react' +import { Button, Input, Label, Modal, Select } from '@kineticlogic/staresdk' +import { api } from '../../api/client' +import { InfoTip } from '../../components/InfoTip' +import { errorMessage } from '../../lib/format' +import { idFromName, newSourceSpec, TRANSPORTS, validId } from './flow/sourceFlow' + +/** + * Add a source: its name, id and transport. It is saved as a draft with its Source, Decode, Map and + * Output steps, and opens in the editor. + */ +export function NewSource({ onClose, onCreated }: { onClose: () => void; onCreated: (id: string) => void }) { + const [name, setName] = useState('') + const [id, setId] = useState('') + const [idEdited, setIdEdited] = useState(false) + const [transport, setTransport] = useState('http_poll') + const [creating, setCreating] = useState(false) + const [error, setError] = useState(null) + const shownId = idEdited ? id : idFromName(name) + const ready = name.trim() !== '' && validId(shownId) + + const create = async () => { + setCreating(true) + setError(null) + try { + const row = await api.createSource(newSourceSpec(shownId, name.trim(), transport)) + onCreated(row.id) + } catch (e) { + setError(errorMessage(e)) + setCreating(false) + } + } + + return ( + +
{ + e.preventDefault() + if (ready && !creating) void create() + }} + > +
+ + setName(e.target.value)} /> +
+
+
+ + + Names the source's data and prefixes its track ids, so it never changes. Lowercase letters, digits, - and _, at most 64. + +
+ { + setIdEdited(true) + setId(e.target.value) + }} + spellCheck={false} + /> +
+
+ + onChange({ ...spec, name: e.target.value })} /> +
+
+ + +
+
+ + { + const { description: _, ...rest } = spec + onChange(e.target.value ? { ...rest, description: e.target.value } : rest) + }} + /> +
+ +

Status

+ +

History

+ + + ) +} + +/** + * A dry run of the source as edited over its stored samples, re-run shortly after it stops changing: + * what stops it running (each problem on its stage) and its samples stage by stage. A draft runs as + * far as it can. Nothing is saved or published. + */ +function useLiveCheck(spec: SourceSpec | null, id: string, nonce: number, trace: number, enabled: boolean) { + const [preview, setPreview] = useState(null) + const [problems, setProblems] = useState(null) + const [error, setError] = useState(null) + const seq = useRef(0) + useEffect(() => { + if (!enabled || !spec) return + const n = ++seq.current + const t = setTimeout(() => { + api.preview(spec, id, [], trace).then( + (r) => { + if (n !== seq.current) return + setPreview(r) + setProblems(r.problems ?? []) + setError(null) + }, + (e) => { + if (n !== seq.current) return + setPreview(null) + const listed = e instanceof ApiError && e.problems.length > 0 + setProblems(listed ? e.problems : null) + setError(listed ? null : errorMessage(e)) + }, + ) + }, PREVIEW_DELAY_MS) + return () => clearTimeout(t) + }, [spec, id, nonce, trace, enabled]) + return { preview, problems, error } +} + +/** + * One source as a flow of steps (#104): the palette, the source's steps in pipeline order, and the + * settings and samples of the step selected, or with none selected the source's own settings, status + * and history. Nothing changes on the running source until Save. + */ +export default function SourceEditor({ id, onClose, onDeleted }: { id: string; onClose: () => void; onDeleted: () => void }) { + const { confirm, toast } = useToast() + const canAdmin = useCan('admin') + const readOnly = !canAdmin + const [row, setRow] = useState(null) + const [loadError, setLoadError] = useState(null) + const [spec, setSpec] = useState(null) + const [baseline, setBaseline] = useState('') + const [selected, setSelected] = useState(null) + const [view, setView] = useState<'flow' | 'json'>('flow') + const [jsonText, setJsonText] = useState('') + const [saving, setSaving] = useState(false) + const [saved, setSaved] = useState(false) + const [notice, setNotice] = useState(null) + const [schema, setSchema] = useState(null) + const [plugins, setPlugins] = useState([]) + const [probe, setProbe] = useState(null) + const [capturing, setCapturing] = useState(false) + const [nonce, setNonce] = useState(0) + + const adopt = useCallback((r: SourceRow) => { + setRow(r) + setSpec(r.spec) + setBaseline(JSON.stringify(r.spec)) + }, []) + + // The row (its status, enabled, stored problems) refreshes; the spec being edited does not. + useEffect(() => { + let first = true + const load = () => + api.source(id).then( + (r) => { + setLoadError(null) + if (first) adopt(r) + else setRow(r) + first = false + }, + (e) => setLoadError(errorMessage(e)), + ) + load() + const t = setInterval(load, REFRESH_MS) + return () => clearInterval(t) + }, [id, adopt]) + + useEffect(() => { + api.schema().then(setSchema, () => setSchema(null)) + api.plugins().then(setPlugins, () => setPlugins([])) + }, []) + + const dirty = spec !== null && JSON.stringify(spec) !== baseline + const live = useLiveCheck(spec, id, nonce, selected === 'filter' ? MAX_TRACED_SAMPLES : TRACED_SAMPLES, canAdmin) + // What stops it running: the live check's, else what was found when it was saved. + const problems = useMemo(() => live.problems ?? row?.problems ?? [], [live.problems, row?.problems]) + const steps = useMemo(() => (spec ? stepsOf(spec) : []), [spec]) + const byStep = useMemo(() => problemsByStep(problems, steps), [problems, steps]) + + const edit = (next: SourceSpec) => { + setSpec(next) + setSaved(false) + setNotice(null) + } + + const nodes: FlowCanvasNode[] = useMemo(() => { + if (!spec) return [] + const summaries = new Map(pipelineStages(spec).map((s) => [s.id, s.summary])) + return steps.map((step, i) => ({ + id: step, + type: 'step', + position: positionOf(i), + selected: step === selected, + draggable: false, + deletable: !REQUIRED.has(step), + data: { + kind: kindOf(step), + icon: stepIcon(spec, step), + title: stepTitle(spec, step), + summary: summaries.get(traceStage(step)) ?? '', + problems: byStep.byStep[step] ?? [], + } satisfies StepView, + })) + }, [spec, steps, selected, byStep]) + + const edges: FlowCanvasEdge[] = useMemo( + () => steps.slice(1).map((step, i) => ({ id: `${steps[i]}->${step}`, source: steps[i], target: step, deletable: false })), + [steps], + ) + + const palette: FlowPaletteItem[] = useMemo( + () => paletteEntries(plugins).map((e) => ({ type: e.type, label: e.label, group: e.group, icon: entryIcon(e.type) })), + [plugins], + ) + + // Steps keep their places and order. Clicking one selects it; deleting an added stage removes it + // from the source. + const onNodesChange = (changes: NodeChange[]) => { + const picks = changes.flatMap((c) => (c.type === 'select' ? [c] : [])) + if (picks.length) { + const on = picks.find((c) => c.selected) + if (on) setSelected(on.id as StepId) + else if (picks.some((c) => c.id === selected)) setSelected(null) + } + const removed = changes.flatMap((c) => (c.type === 'remove' ? [c.id as StepId] : [])) + if (removed.length && spec) { + edit(removed.reduce(removeStep, spec)) + if (selected && removed.includes(selected)) setSelected(null) + } + } + + const onDropNode = async (type: string) => { + if (!spec || readOnly) return + const step = stepOfEntry(type) + if (replaces(spec, type)) { + const what = step === 'source' ? 'transport' : 'decoding' + const label = palette.find((p) => p.type === type)?.label ?? '' + const ok = await confirm(`Replace this source's ${what} settings with a new ${label} step?`, { title: 'Replace step', confirmLabel: 'Replace' }) + if (!ok) return + } + edit(addEntry(spec, type, plugins)) + if (step) setSelected(step) + } + + const save = async () => { + if (!spec) return + setSaving(true) + setNotice(null) + try { + adopt(await api.saveSource(spec)) + setSaved(true) + setNonce((n) => n + 1) + } catch (e) { + setNotice(errorMessage(e)) + } finally { + setSaving(false) + } + } + + const toggle = async (on: boolean) => { + if (on && dirty) { + setNotice('Save the source before turning it on.') + return + } + try { + setRow(await api.setEnabled(id, on)) + setNotice(null) + } catch (e) { + setNotice(errorMessage(e)) + } + } + + const remove = async () => { + const ok = await confirm(`Delete source ${id}? Its configuration history stays in the decision log.`, { title: 'Delete source', confirmLabel: 'Delete' }) + if (!ok) return + try { + await api.deleteSource(id) + onDeleted() + } catch (e) { + toast({ variant: 'error', message: errorMessage(e) }) + } + } + + const leave = async () => { + if (dirty && !(await confirm('Leave without saving? The changes to this source are lost.', { title: 'Unsaved source', confirmLabel: 'Discard' }))) return + onClose() + } + + /** Take a few frames from the feed now, keep them as the source's samples, and offer what they suggest. */ + const capture = async () => { + if (!spec) return + setCapturing(true) + try { + const r = await api.probe({ transport: spec.transport, codec: spec.pipeline.codec, max_frames: 20, max_secs: 15, save_as: id }) + setProbe(r) + if (!r.frames) toast({ variant: 'warning', message: `No frames captured: ${r.link_error ?? r.hint ?? 'none arrived in time.'}` }) + setNonce((n) => n + 1) + } catch (e) { + toast({ variant: 'error', title: 'Capture failed', message: errorMessage(e) }) + } finally { + setCapturing(false) + } + } + + const switchView = (v: 'flow' | 'json') => { + if (v === 'json' && spec) setJsonText(JSON.stringify(spec, null, 2)) + setView(v) + } + + if (!spec || !row) { + return ( +
+
{loadError ?
{loadError}
: LOADING…}
+
+ ) + } + + const state = row.draft ? { label: 'draft', color: 'grey' as const } : sourceState(row) + const general = [...(notice ? [notice] : []), ...byStep.general] + + return ( +
+
+ { + void leave() + }} + /> + + {state.label} + + {SOURCE_STATES_INFO} + {general.length > 0 && ( + + {general.join(' · ')} + + )} + + + + {canAdmin && } +
+ {view === 'flow' ? ( +
+ false} + palette={palette} + onDropNode={(type) => void onDropNode(type)} + readOnly={readOnly} + height="100%" + style={{ minHeight: 0 }} + /> + +
+ ) : ( +
+ { + setJsonText(t) + try { + const next = JSON.parse(t) as SourceSpec + if (next && typeof next === 'object' && next.id === id) edit(next) + } catch { + // Keep the last valid settings; the editor marks the parse error. + } + }} + /> +
+ )} +
+ ) +} diff --git a/ui/src/pages/sources/SourceStatus.tsx b/ui/src/pages/sources/SourceStatus.tsx new file mode 100644 index 0000000..7657ceb --- /dev/null +++ b/ui/src/pages/sources/SourceStatus.tsx @@ -0,0 +1,107 @@ +import { useEffect, useState } from 'react' +import { Badge, DataTable, type DataTableColumn } from '@kineticlogic/staresdk' +import { CodeEditor } from '@kineticlogic/staresdk/code-editor' +import { api, type MetricsResponse, type SourceRow } from '../../api/client' +import { ago, fmtCount, fmtNum, fmtTime } from '../../lib/format' +import { PIPELINE_COUNTS_INFO } from '../../lib/sourceState' +import { InfoTip } from '../../components/InfoTip' + +type Revision = Awaited>[number] + +const REVISION_COLUMNS: DataTableColumn[] = [ + { key: 'rev', header: 'Revision', width: 80, render: (r) => r.revision, sortValue: (r) => r.revision }, + { key: 'at', header: 'Saved', mono: true, render: (r) => fmtTime(r.saved_at_ms) }, + { key: 'decision', header: 'Decision', width: 90, align: 'right', render: (r) => (r.decision_id ? `#${r.decision_id}` : '—') }, +] + +/** A saved source's live status: its link, revision, raw output and the last hour's counts. */ +export function SourceStatus({ source }: { source: SourceRow }) { + const [metrics, setMetrics] = useState(null) + useEffect(() => { + api.metrics(source.id, 60).then(setMetrics, () => setMetrics(null)) + }, [source.id, source.status?.updated_at]) + const link = source.status?.link + const timing = source.status?.tracker_timing + const totals = Object.entries(metrics?.totals ?? {}).filter(([k]) => !k.includes(':')) + return ( +
+
+
Transport
+
+ {source.transport} · {String(source.spec.transport.url ?? source.spec.transport.bind ?? `${source.spec.transport.host ?? ''}:${source.spec.transport.port ?? ''}`)} +
+
Codec
+
{source.codec}
+
Link
+
+ {link ? `${link.connected ? 'connected' : 'not connected'} · ${link.connects} connect(s) · ${link.errors} error(s)` : 'no worker running'} +
+
Last frame
+
{link?.last_frame_at ? `${ago(link.last_frame_at)} ago` : '—'}
+
Revision
+
+ {source.revision} · saved {fmtTime(source.updated_at_ms)} +
+ {(source.spec.pipeline as { tracker?: { auto_timing?: unknown } }).tracker?.auto_timing != null && ( + <> +
Tracker timing
+
+ {timing + ? `revisit ${fmtNum(timing.revisit_secs, 1)} s (${timing.revisit_source}) · confirm within ${fmtNum(timing.confirm_within_secs, 0)} s · drop after ${fmtNum(timing.drop_tentative_secs, 0)} s unconfirmed, ${fmtNum(timing.drop_confirmed_secs, 0)} s confirmed` + : 'waiting for the sensor’s revisit rate'} +
+ + )} +
Raw output
+
{source.raw_subject ?? 'off'}
+
+ {(link?.last_error || source.status?.last_error) && ( +
{source.status?.last_error ?? link?.last_error}
+ )} +

+ Last 60 minutes + + Counts over the last hour. {PIPELINE_COUNTS_INFO} + +

+
+ {totals.length === 0 && No activity recorded.} + {totals.map(([k, v]) => ( + + {k} {fmtCount(v)} + + ))} +
+
+ ) +} + +/** Every saved revision of a source, each viewable. */ +export function SourceHistory({ id, revision }: { id: string; revision: number }) { + const [revisions, setRevisions] = useState([]) + const [selected, setSelected] = useState(null) + useEffect(() => { + api.revisions(id).then(setRevisions, () => setRevisions([])) + }, [id, revision]) + const shown = revisions.find((r) => r.revision === selected) + return ( +
+ + Every save of this source's settings, newest last. Click one to see it. + + Decision: the number of the decision-log entry that recorded the save, with who made it and the settings before and after. + + + String(r.revision)} + selectedKey={selected === null ? null : String(selected)} + onRowClick={(r) => setSelected(r.revision)} + maxHeight={200} + /> + {shown && } +
+ ) +} diff --git a/ui/src/pages/sources/SourcesPage.tsx b/ui/src/pages/sources/SourcesPage.tsx index c3087aa..593b6a4 100644 --- a/ui/src/pages/sources/SourcesPage.tsx +++ b/ui/src/pages/sources/SourcesPage.tsx @@ -4,16 +4,16 @@ import { Badge, Button, CollapsiblePanel, DataTable, type DataTableColumn } from import { api, type SourceRow } from '../../api/client' import { ago, errorMessage, fmtCount } from '../../lib/format' import { SOURCE_STATES_INFO, sourceState } from '../../lib/sourceState' -import { AddSourceWizard } from './AddSourceWizard' -import { SourceDetail } from './SourceDetail' +import { NewSource } from './NewSource' import { UnauthenticatedBadge } from './SenderAuth' import { useCan } from '../../auth/context' import { InfoTip } from '../../components/InfoTip' import { FILL_PANEL, usePanelOpen } from '../../lib/panelOpen' import { FILL_TABLE, useFillHeight } from '../../lib/fillHeight' -// React Flow loads only with the Sources page's topology. +// React Flow loads only with the Sources page's topology and the source editor. const Topology = lazy(() => import('./Topology')) +const SourceEditor = lazy(() => import('./SourceEditor')) const REFRESH_MS = 5000 @@ -65,8 +65,6 @@ export default function SourcesPage({ selected, onSelect }: { selected: string; const [sources, setSources] = useState(null) const [error, setError] = useState(null) const [adding, setAdding] = useState(false) - // The drawer keeps showing the last source while it slides closed. - const [shownId, setShownId] = useState(selected) const [listOpen, setListOpen] = usePanelOpen('ot.panel.sources') const [listFill, listHeight] = useFillHeight() @@ -86,28 +84,25 @@ export default function SourcesPage({ selected, onSelect }: { selected: string; return () => clearInterval(t) }, [load]) - const select = (id: string) => { - if (id) setShownId(id) - onSelect(id) - } - - if (adding) { + if (selected) { return ( -
- setAdding(false)} - onDone={(id) => { - setAdding(false) + LOADING…}> + { + onSelect('') + load() + }} + onDeleted={() => { + onSelect('') load() - select(id) }} /> -
+ ) } - const shown = sources?.find((s) => s.id === (selected || shownId)) ?? null - const open = !!selected && !!shown return (
LOADING…}> - {sources ? : LOADING…} + {sources ? : LOADING…}
@@ -143,7 +138,7 @@ export default function SourcesPage({ selected, onSelect }: { selected: string; >
- Feeds OpenTrack ingests. Select one to see its status, transport, pipeline and history. + Feeds OpenTrack ingests. Select one to open it.
- {shown && ( - select('')} - onChanged={load} - onDeleted={() => { - load() - onSelect('') - setShownId('') + {adding && ( + setAdding(false)} + onCreated={(id) => { + setAdding(false) + onSelect(id) }} /> )} diff --git a/ui/src/pages/sources/TransportForm.tsx b/ui/src/pages/sources/TransportForm.tsx index e5671b7..63090e4 100644 --- a/ui/src/pages/sources/TransportForm.tsx +++ b/ui/src/pages/sources/TransportForm.tsx @@ -9,17 +9,6 @@ import { ProtoSchema } from './ProtoSchema' type Transport = SourceSpec['transport'] type Codec = SourceSpec['pipeline']['codec'] -const TRANSPORTS: Record = { - http_poll: { label: 'HTTP poll', initial: { type: 'http_poll', url: '', interval_secs: 5 } }, - websocket: { label: 'WebSocket client', initial: { type: 'websocket', url: '' } }, - tcp_client: { label: 'TCP client', initial: { type: 'tcp_client', host: '', port: 0, framing: { type: 'lines' } } }, - tcp_server: { label: 'TCP server (listen)', initial: { type: 'tcp_server', bind: '0.0.0.0:8087', framing: { type: 'lines' } } }, - udp: { label: 'UDP (unicast or multicast)', initial: { type: 'udp', bind: '0.0.0.0:6969' } }, - mqtt: { label: 'MQTT subscribe', initial: { type: 'mqtt', url: '', topics: [] } }, - grpc_client: { label: 'gRPC client (call the producer)', initial: { type: 'grpc_client', url: '', method: '' } }, - grpc_server: { label: 'gRPC server (producers call)', initial: { type: 'grpc_server', bind: '0.0.0.0:50051' } }, -} - const QOS: Record = { '0': { label: '0 · at most once' }, '1': { label: '1 · at least once' }, @@ -41,14 +30,6 @@ const ENDIANS: Record = { /** Methods offered for an HTTP poll; any other the spec holds is offered too. */ const HTTP_METHODS = ['GET', 'POST', 'PUT', 'PATCH'] -const CODECS: Record = { - json: { label: 'JSON', initial: { type: 'json' } }, - cot_xml: { label: 'Cursor-on-Target XML', initial: { type: 'cot_xml' } }, - xml: { label: 'XML (generic)', initial: { type: 'xml', record_element: 'record' } }, - protobuf: { label: 'Protobuf (.proto)', initial: { type: 'protobuf', files: {}, message: '' } }, - plugin: { label: 'Plugin', initial: { type: 'plugin', plugin: '', options: {} } }, -} - /** A field's label with its ⓘ explanation beside it. */ function FieldLabel({ label, help, htmlFor }: { label: string; help?: React.ReactNode; htmlFor?: string }) { return ( @@ -278,13 +259,19 @@ function TlsSettings({ transport, onTransport }: { transport: Transport; onTrans ) } -/** Transport and codec settings for a source. */ +/** + * A source's settings for one step: `source`, its transport's (connection, TLS, framing); `decode`, + * its codec's (record path, .proto files, plugin options). Which transport and codec it has is the + * step itself, chosen on the canvas. + */ export function TransportForm({ + part, transport, codec, onTransport, onCodec, }: { + part: 'source' | 'decode' transport: Transport codec: Codec onTransport: (t: Transport) => void @@ -297,9 +284,9 @@ export function TransportForm({ } const [plugins, setPlugins] = useState([]) useEffect(() => { - if (codec.type !== 'plugin') return + if (part !== 'decode' || codec.type !== 'plugin') return api.plugins().then(setPlugins, () => setPlugins([])) - }, [codec.type]) + }, [part, codec.type]) const plugin = plugins.find((p) => p.name === codec.plugin) const pluginOptions = (codec.options as Record | undefined) ?? {} const choosePlugin = (name: string) => { @@ -311,45 +298,20 @@ export function TransportForm({ onTransport({ ...transport, framing: structuredClone(p.framing) }) } } - // CoT arrives as back-to-back XML events, often over several lines: framed by ``, not by - // lines. Choosing CoT XML (or a stream transport for it) switches the default lines framing. - const streamed = (t: Transport) => t.type === 'tcp_client' || t.type === 'tcp_server' - const cotFraming = (t: Transport, codecType: unknown): Transport => - codecType === 'cot_xml' && streamed(t) && (t.framing as { type?: string } | undefined)?.type === 'lines' - ? { ...t, framing: { type: 'end_tag', tag: '' } } - : t - const chooseCodec = (id: string) => { - onCodec(structuredClone(CODECS[id].initial)) - const next = cotFraming(transport, id) - if (next !== transport) onTransport(next) - } const [proto, setProto] = useState(null) const grpc = transport.type === 'grpc_client' || transport.type === 'grpc_server' + // The gRPC methods come from the Decode step's .proto files. + const protoFiles = codec.type === 'protobuf' ? (codec.files as Record | undefined) : undefined + useEffect(() => { + if (part !== 'source' || !grpc || !protoFiles || Object.keys(protoFiles).length === 0) return + api.describeProtobuf(protoFiles).then(setProto, () => setProto(null)) + }, [part, grpc, protoFiles]) const framing = (transport.framing as Record | undefined) ?? { type: 'lines' } const setFraming = (k: string, v: unknown) => set('framing', { ...framing, [k]: v }) return (
-
- - -
- {codec.type === 'json' && ( + {part === 'decode' && codec.type === 'json' && ( )} - {codec.type === 'plugin' && ( + {part === 'decode' && codec.type === 'json' && ( + { + const next: Codec = { ...codec, context: v } + if (!v.length) delete next.context + onCodec(next) + }} + placeholder="optional, comma separated, e.g. now" + help="Frame-level fields copied into every record under _frame, e.g. now." + /> + )} + {part === 'decode' && codec.type === 'plugin' && ( <>
)} - {codec.type === 'protobuf' && } - {grpc && codec.type !== 'protobuf' && ( -
gRPC carries protobuf: choose the Protobuf codec and add the producer's .proto files.
+ {part === 'decode' && codec.type === 'protobuf' && } + {part === 'decode' && grpc && codec.type !== 'protobuf' && ( +
gRPC carries protobuf: use a Protobuf decode step, with the producer's .proto files.
)} - {transport.type === 'grpc_client' && ( + {part === 'source' && transport.type === 'grpc_client' && ( <> )} - {transport.type === 'grpc_server' && ( + {part === 'source' && transport.type === 'grpc_server' && ( <> )} - {codec.type === 'xml' && ( + {part === 'decode' && codec.type === 'xml' && ( )} - {(transport.type === 'http_poll' || transport.type === 'websocket') && ( + {part === 'source' && (transport.type === 'http_poll' || transport.type === 'websocket') && ( )} - {transport.type === 'http_poll' && ( + {part === 'source' && transport.type === 'http_poll' && ( <> )} - {transport.type === 'websocket' && ( + {part === 'source' && transport.type === 'websocket' && ( <> )} - {transport.type === 'tcp_client' && ( + {part === 'source' && transport.type === 'tcp_client' && ( <> )} - {(transport.type === 'tcp_server' || transport.type === 'udp') && ( + {part === 'source' && (transport.type === 'tcp_server' || transport.type === 'udp') && ( )} - {transport.type === 'udp' && ( + {part === 'source' && transport.type === 'udp' && ( <> )} - {transport.type === 'mqtt' && ( + {part === 'source' && transport.type === 'mqtt' && ( <> )} - {TLS_TRANSPORTS.includes(transport.type) && } - {(transport.type === 'tcp_client' || transport.type === 'tcp_server') && ( + {part === 'source' && transport.type === 'file' && ( + <> + set('path', v)} + placeholder="recorded/feed.ndjson" + help="Recorded data to replay: a file, or every file in a directory, by name. A relative path is under the data directory; a file source reads only there. Mappings read the file name as _frame.file." + /> + set('extension', v)} + placeholder="optional, e.g. ndjson" + help="In a directory, only the files with this extension. Blank: every file." + /> + set('frames_per_second', num(v))} + placeholder="blank: as fast as taken" + help="Pace the replay, in frames per second. Blank: as fast as the pipeline takes them." + /> +
+ + set('repeat', on || undefined)} aria-label="Repeat" /> +
+ + )} + {part === 'source' && TLS_TRANSPORTS.includes(transport.type) && } + {part === 'source' && (transport.type === 'tcp_client' || transport.type === 'tcp_server') && ( <>
import('../PipelineFlow')) - -type Obj = Record -type Pipeline = SourceSpec['pipeline'] & Obj - -/** Optional stages the user can add, with what a new one starts as. */ -const OPTIONAL: { id: string; label: string; add: (p: Pipeline) => Pipeline; remove: (p: Pipeline) => Pipeline }[] = [ - { - id: 'reject', - label: 'Reject', - add: (p) => ({ ...p, mapping: { ...p.mapping, reject: [{ reason: 'no_position', when: { path: 'lat', exists: false } }] } }), - remove: (p) => ({ ...p, mapping: { ...p.mapping, reject: undefined } }), - }, - { - id: 'registry', - label: 'Entity links', - add: (p) => ({ ...p, registry: { links: DEFAULT_LINKS } }), - remove: (p) => ({ ...p, registry: undefined }), - }, - { - id: 'affiliation', - label: 'Affiliation', - add: (p) => ({ ...p, affiliation: { country: 'platform.flag', friendly: [], hostile: [], neutral: [] } }), - remove: (p) => ({ ...p, affiliation: undefined }), - }, - { - id: 'filter', - label: 'Filter', - add: (p) => ({ ...p, filter: {} }), - remove: (p) => ({ ...p, filter: undefined }), - }, - { - id: 'tracker', - label: 'Tracker', - add: (p) => ({ ...p, tracker: { algorithm: 'gnn' } }), - remove: (p) => ({ ...p, tracker: undefined }), - }, - { - id: 'throttle', - label: 'Throttle', - add: (p) => ({ ...p, throttle: { min_interval_secs: 5, min_move_m: 0, heartbeat_secs: 600 } }), - remove: (p) => ({ ...p, throttle: undefined }), - }, -] - -/** What each stage does, for the ⓘ beside its heading. */ -const STAGE_HELP: Record = { - transport: 'Where frames come from. Set on the source’s Transport tab.', - decode: 'Turns each frame into records.', - reject: 'Drops raw records before mapping, each rule with a reason counted in the source’s metrics.', - map: 'Turns records into observations: the track key, identifiers and OpenTrack fields each rule takes from the record.', - join: 'Keeps the identity fields that static rules report per track key, and fills them into later observations with the same key.', - registry: 'Finds each track’s entity by its identifiers, grades the match by broadcast name, and at a corroborated grade passes fields between entity and track.', - affiliation: 'Sets affiliation from a country code, by the country lists.', - filter: 'Keeps or drops observations after mapping and the entity stage; dropped ones are counted as filtered.', - tracker: 'Forms tracks from detections (plots) before they leave the source.', - throttle: 'Limits how often each source track is passed on, by time and distance moved.', - publish: 'What the source’s observations are, which decides how correlation takes them, and the label they carry.', -} - -const present = (p: Pipeline, id: string) => - id === 'reject' ? ((p.mapping.reject as unknown[]) ?? []).length > 0 : id === 'registry' ? p.registry !== undefined : p[id] !== undefined - -/** Drop keys set to undefined, so the saved spec only carries what is configured. */ -const clean = (p: Pipeline): Pipeline => JSON.parse(JSON.stringify(p)) - -/** - * Design a source's pipeline stage by stage, with a live dry run over the stored samples. Saving - * stores a new revision of the source (the worker restarts with it); nothing changes until then. - */ -export function PipelineDesigner({ - initial, - sourceId, - onClose, - onSaved, -}: { - initial: SourceSpec - sourceId: string - onClose: () => void - onSaved: (spec: SourceSpec) => void -}) { - const { confirm } = useToast() - const [spec, setSpec] = useState(initial) - const [stageId, setStageId] = useState('map') - const [schema, setSchema] = useState(null) - const [saving, setSaving] = useState(false) - const [saved, setSaved] = useState(false) - const [error, setError] = useState(null) - const [nonce, setNonce] = useState(0) - const [capturing, setCapturing] = useState(false) - // The filter builder offers the fields of every traced sample, so it asks for as many as there are. - const { preview, error: previewError, running } = usePreview(spec, sourceId, nonce, stageId === 'filter' ? MAX_TRACED_SAMPLES : TRACED_SAMPLES) - - useEffect(() => { - api.schema().then(setSchema, (e) => setError(errorMessage(e))) - }, []) - - const p = spec.pipeline as Pipeline - const setPipeline = (next: Pipeline) => setSpec({ ...spec, pipeline: clean(next) }) - const stages = useMemo(() => pipelineStages(spec), [spec]) - const stage = stages.find((s) => s.id === stageId) ?? stages.find((s) => s.id === 'map')! - const optional = OPTIONAL.find((o) => o.id === stage.id) - const dirty = JSON.stringify(spec) !== JSON.stringify(initial) - - /** Take a few frames from the feed now and keep them as this source's samples. */ - const capture = async () => { - setCapturing(true) - setError(null) - try { - const r = await api.probe({ transport: spec.transport, codec: spec.pipeline.codec, max_frames: 20, max_secs: 15, save_as: sourceId }) - if (!r.frames && (r.link_error || r.hint)) setError(`No frames captured: ${r.link_error ?? r.hint}`) - setNonce((n) => n + 1) - } catch (e) { - setError(errorMessage(e)) - } finally { - setCapturing(false) - } - } - - const save = async () => { - setSaving(true) - setError(null) - try { - await api.saveSource(spec) - setSaved(true) - setTimeout(() => setSaved(false), 1500) - onSaved(spec) - } catch (e) { - setError(errorMessage(e)) - } finally { - setSaving(false) - } - } - - const close = async () => { - if (dirty && !(await confirm('Close without saving? The pipeline changes are lost.', { title: 'Unsaved pipeline', confirmLabel: 'Discard' }))) return - onClose() - } - - let editor: React.ReactNode = null - switch (stage.id) { - case 'transport': - editor = The transport is set on the source's Transport tab. - break - case 'decode': - editor = setPipeline({ ...p, codec: codec as Pipeline['codec'] })} /> - break - case 'reject': - editor = setPipeline({ ...p, mapping: { ...p.mapping, reject: reject.length ? reject : undefined } })} /> - break - case 'map': - editor = schema ? ( - setPipeline({ ...p, mapping: mapping as Pipeline['mapping'] })} /> - ) : ( - LOADING… - ) - break - case 'join': - editor = setPipeline({ ...p, static_join })} /> - break - case 'registry': - editor = p.registry ? ( - setPipeline({ ...p, registry })} /> - ) : ( - - Runs with defaults: resolves identifiers, and at a corroborated match the entity's name, class name, domain, affiliation, track type - and symbol populate the track. Configure it to choose the links. - - ) - break - case 'affiliation': - editor = setPipeline({ ...p, affiliation })} /> - break - case 'filter': - { - const input = filterInput(preview?.trace) - const fields = sampleFields([...input.observations, ...(preview?.observations ?? [])]) - const key = (o: unknown) => String((o as { source_track_key?: unknown }).source_track_key ?? '') - const examples = [...new Set(input.dropped.map(key).filter(Boolean))].slice(0, 3) - const effect = preview ? { dropped: preview.counts?.filtered ?? 0, examples } : null - editor = setPipeline({ ...p, filter })} fields={fields} effect={effect} /> - } - break - case 'tracker': - editor = setPipeline({ ...p, tracker })} /> - break - case 'throttle': - editor = setPipeline({ ...p, throttle })} /> - break - case 'publish': - editor = ( - setSpec({ ...spec, reports })} - onPublishAlone={(publish_alone) => setSpec({ ...spec, publish_alone })} - onConfirmAfter={(confirm_after) => setSpec({ ...spec, confirm_after })} - /> - ) - editor = ( -
- {editor} - setSpec({ ...spec, security })} /> - setSpec({ ...spec, emitter_motion })} /> -
- ) - break - } - - return ( - } - > -
-
- LOADING…}> - - -
- {OPTIONAL.some((o) => !present(p, o.id)) && ( - - Add an optional stage. Reject: drop raw records by condition, with a reason. Entity links: choose how tracks find their entity and which - fields flow each way (without it, defaults apply). Affiliation: set affiliation from a country code. Filter: keep or drop observations by - condition. Tracker: form tracks from detections. Throttle: pass each track on less often. - - )} - {OPTIONAL.filter((o) => !present(p, o.id)).map((o) => ( - - ))} -
-
-
-
-

- {stage.title} - {STAGE_HELP[stage.id] && {STAGE_HELP[stage.id]}} -

- - {optional && present(p, optional.id) && ( - - )} - {optional && present(p, optional.id) && ( - - {optional.id === 'registry' - ? 'Drop these settings: the entity stage still runs, with the default links (the entity’s OTH-GOLD minimum populates the track) at grades exact, hull, name and generic.' - : 'Take this stage out of the pipeline. Nothing changes on the running source until you save.'} - - )} -
- {error &&
{error}
} - {editor} -
-
-
-

- Live preview - - A dry run of the pipeline as edited (not yet saved) over this source's stored samples, updated as you change it. It shows the - first {TRACED_SAMPLES} stored samples (decoded records, in order) as they are after the stage selected on the left: at Transport - the frames they came from, then each record, its observation, and at Publish the message as it would be published. A sample - dropped or held on the way says where and why. The counts cover everything stored. Nothing is published. - -

- {running ? 'running…' : `stored samples of ${sourceId}`} - - - - Connect with the transport as edited and take up to 20 frames or 15 s of the live feed, whichever comes first. They replace the - source's stored samples. - -
- - {previewError &&
{previewError}
} - {preview ? : !previewError && Running the first preview…} -
-
- - ) -} diff --git a/ui/src/pages/sources/designer/StageForms.tsx b/ui/src/pages/sources/designer/StageForms.tsx index 838006e..e1722c9 100644 --- a/ui/src/pages/sources/designer/StageForms.tsx +++ b/ui/src/pages/sources/designer/StageForms.tsx @@ -2,7 +2,7 @@ import { useEffect, useState } from 'react' import { TbPlus, TbTrash, TbX } from 'react-icons/tb' import { Button, FieldSelect, Input, Label, Toggle } from '@kineticlogic/staresdk' import { api, type EmitterMotion, type PluginInfo, type SecurityLabel } from '../../../api/client' -import { DECODE_CODECS, DEFAULT_GRADES, DEFAULT_LINKS, describeCondition, describeValue, entityLinks, transportCodec, type EntityLink, type ValueSpec } from '../../../lib/pipeline' +import { DEFAULT_GRADES, DEFAULT_LINKS, describeCondition, describeValue, entityLinks, type EntityLink, type ValueSpec } from '../../../lib/pipeline' import { JsonField } from './JsonField' import { ConditionBuilder } from './ConditionBuilder' import type { SampleField } from '../../../lib/conditions' @@ -36,69 +36,6 @@ function Row({ label, children, hint }: { label: string; children: React.ReactNo ) } -/** How frames become records. */ -export function DecodeForm({ value, onChange }: Props) { - const type = String(value.type ?? 'json') - const fixed = transportCodec(value) - if (fixed) - return ( -
- - {fixed.name ? `${fixed.type} · ${fixed.name}` : fixed.type} - -
- ) - return ( -
- - ({ name }))} - value={type} - onChange={(t) => onChange(t === 'xml' ? { type: t, record_element: '' } : { type: t ?? 'json' })} - style={{ width: 160 }} - /> - - {type === 'json' && ( - <> - - onChange({ ...value, records: e.target.value || undefined })} - spellCheck={false} - /> - - - { - const c = words(e.target.value) - onChange({ ...value, context: c.length ? c : undefined }) - }} - spellCheck={false} - /> - - - )} - {type === 'xml' && ( - - onChange({ ...value, record_element: e.target.value })} spellCheck={false} /> - - )} -
- ) -} - /** Records dropped before mapping, each with a reason (counted in the source's metrics). */ export function RejectForm({ rules, onChange }: { rules: Obj[]; onChange: (r: Obj[]) => void }) { return ( diff --git a/ui/src/pages/sources/flow/StepNode.tsx b/ui/src/pages/sources/flow/StepNode.tsx new file mode 100644 index 0000000..d78141b --- /dev/null +++ b/ui/src/pages/sources/flow/StepNode.tsx @@ -0,0 +1,47 @@ +import type { ReactNode } from 'react' +import { Handle, Position, type NodeProps } from '@xyflow/react' +import { KIND_WORD, type StepKind } from './sourceFlow' + +/** What the editor gives each step to draw (the canvas node's `data`). */ +export interface StepView extends Record { + kind: StepKind + icon: ReactNode + title: string + summary: string + problems: string[] +} + +/** + * One step of a source on the canvas: its icon, heading and one line of its settings, joined to the + * step before and after it, with what stops it running in red. + */ +export function StepNode({ data, selected }: NodeProps) { + const step = data as StepView + const failing = step.problems.length > 0 + return ( +
+ {step.kind !== 'source' && } +
+ + {step.icon} + + {step.title} + {KIND_WORD[step.kind]} +
+ {step.summary && {step.summary}} + {step.problems.map((message) => ( + + {message} + + ))} + {step.kind !== 'output' && } +
+ ) +} diff --git a/ui/src/pages/sources/flow/StepPanel.tsx b/ui/src/pages/sources/flow/StepPanel.tsx new file mode 100644 index 0000000..dad725a --- /dev/null +++ b/ui/src/pages/sources/flow/StepPanel.tsx @@ -0,0 +1,269 @@ +import { TbDownload, TbWand } from 'react-icons/tb' +import { Button, Input, Label } from '@kineticlogic/staresdk' +import type { PreviewResult, ProbeResult, SchemaOverview, SourceSpec } from '../../../api/client' +import { InfoTip } from '../../../components/InfoTip' +import { sampleFields } from '../../../lib/conditions' +import { filterInput } from '../../../lib/trace' +import { MapEditor } from '../designer/MapEditor' +import { StageSamples } from '../designer/StageSamples' +import { + AffiliationForm, + EmitterMotionForm, + FilterForm, + JoinForm, + PublishForm, + RegistryForm, + RejectForm, + SecurityForm, + ThrottleForm, + TrackerForm, +} from '../designer/StageForms' +import { SenderAuth } from '../SenderAuth' +import { TransportForm } from '../TransportForm' +import { KIND_WORD, kindOf, stepTitle, traceStage, type StepId } from './sourceFlow' + +type Obj = Record +type Pipeline = SourceSpec['pipeline'] & Obj + +/** What each step does, for the ⓘ beside its heading. */ +const STEP_HELP: Record = { + source: 'Where frames come from: the transport, how it connects or listens, and how a stream is cut into frames.', + decode: 'Turns each frame into records.', + reject: 'Drops raw records before mapping, each rule with a reason counted in the source’s metrics.', + map: 'Turns records into observations: the track key, identifiers and OpenTrack fields each rule takes from the record.', + join: 'Keeps the identity fields that static rules report per track key, and fills them into later observations with the same key.', + registry: 'Finds each track’s entity by its identifiers, grades the match by broadcast name, and at a corroborated grade passes fields between entity and track.', + affiliation: 'Sets affiliation from a country code, by the country lists.', + filter: 'Keeps or drops observations after mapping and the entity stage; dropped ones are counted as filtered.', + tracker: 'Forms tracks from detections (plots) before they leave the source.', + throttle: 'Limits how often each source track is passed on, by time and distance moved.', + output: 'What the source’s observations are, which decides how correlation takes them, the label they carry, and its priority.', +} + +/** Drop pipeline keys set to undefined, so the saved spec carries only what is configured. */ +const clean = (p: Pipeline): Pipeline => JSON.parse(JSON.stringify(p)) + +/** The settings of one step of a source, then its samples as they leave that step. */ +export function StepPanel({ + spec, + step, + problems, + schema, + preview, + previewError, + probe, + capturing, + readOnly, + onChange, + onCapture, +}: { + spec: SourceSpec + step: StepId + problems: string[] + schema: SchemaOverview | null + preview: PreviewResult | null + previewError: string | null + probe: ProbeResult | null + capturing: boolean + readOnly: boolean + onChange: (spec: SourceSpec) => void + onCapture: () => void +}) { + const p = spec.pipeline as Pipeline + const setPipeline = (next: Pipeline) => onChange({ ...spec, pipeline: clean(next) }) + + let settings: React.ReactNode = null + switch (step) { + case 'source': + settings = ( + <> + onChange({ ...spec, transport })} + onCodec={(codec) => setPipeline({ ...p, codec })} + /> + + + ) + break + case 'decode': { + const detected = probe && JSON.stringify(probe.codec) !== JSON.stringify(spec.pipeline.codec) ? probe.codec : null + settings = ( + <> + {detected && ( +
+ + + The captured samples decode with these settings: {JSON.stringify(detected)}. Using them replaces this step's settings. + +
+ )} + onChange({ ...spec, transport })} + onCodec={(codec) => setPipeline({ ...p, codec })} + /> + + ) + break + } + case 'reject': + settings = ( + setPipeline({ ...p, mapping: { ...p.mapping, reject: reject.length ? reject : undefined } })} + /> + ) + break + case 'map': { + const suggested = probe?.suggestion.mapping + const offer = suggested && suggested.rules.length > 0 && JSON.stringify(suggested.rules) !== JSON.stringify(p.mapping.rules) + settings = ( + <> + {offer && ( +
+ + + A mapping inferred from the captured samples: {suggested.rules.length} rule(s) + {probe.suggestion.missing.length ? `; not found in the samples: ${probe.suggestion.missing.join(', ')}` : ''}. Using it replaces + this step's rules. + +
+ )} + {schema ? ( + setPipeline({ ...p, mapping: mapping as Pipeline['mapping'] })} /> + ) : ( + LOADING… + )} + + ) + break + } + case 'join': + settings = setPipeline({ ...p, static_join })} /> + break + case 'registry': + settings = setPipeline({ ...p, registry })} /> + break + case 'affiliation': + settings = setPipeline({ ...p, affiliation })} /> + break + case 'filter': { + const input = filterInput(preview?.trace) + const fields = sampleFields([...input.observations, ...(preview?.observations ?? [])]) + const key = (o: unknown) => String((o as { source_track_key?: unknown }).source_track_key ?? '') + const examples = [...new Set(input.dropped.map(key).filter(Boolean))].slice(0, 3) + const effect = preview ? { dropped: preview.counts?.filtered ?? 0, examples } : null + settings = setPipeline({ ...p, filter })} fields={fields} effect={effect} /> + break + } + case 'tracker': + settings = setPipeline({ ...p, tracker })} /> + break + case 'throttle': + settings = setPipeline({ ...p, throttle })} /> + break + case 'output': + settings = ( +
+ { + const { reports: _, ...rest } = spec + onChange(reports ? { ...rest, reports } : rest) + }} + onPublishAlone={(publish_alone) => onChange({ ...spec, publish_alone })} + onConfirmAfter={(confirm_after) => onChange({ ...spec, confirm_after })} + /> +
+
+ + + Where several sources report one track, priority breaks ties: it picks the position among equally certain reports, the + identity and classification when no report is corroborated by the registry, and each extension field. Higher wins. Default 100. + +
+ { + const { priority: _, ...rest } = spec + onChange(e.target.value.trim() === '' ? rest : { ...rest, priority: Math.round(Number(e.target.value)) }) + }} + /> +
+ onChange({ ...spec, security })} /> + onChange({ ...spec, emitter_motion })} /> +
+ ) + break + } + + return ( +
+
+
+ {KIND_WORD[kindOf(step)]} +

+ {stepTitle(spec, step)} + {STEP_HELP[step]} +

+
+ {problems.map((m) => ( +
+ {m} +
+ ))} +
+ {settings} +
+
+
+
+

+ Samples + + A dry run of the source as edited (not yet saved) over its stored samples, as they leave this step. Nothing is published. + +

+ + {step === 'source' && ( + <> + + + Connect with this step's settings and take up to 20 frames or 15 s of the live feed, whichever comes first. They replace the + source's stored samples. + + + )} +
+ {previewError &&
{previewError}
} + {preview && } +
+
+ ) +} diff --git a/ui/src/pages/sources/flow/sourceFlow.test.ts b/ui/src/pages/sources/flow/sourceFlow.test.ts new file mode 100644 index 0000000..edde273 --- /dev/null +++ b/ui/src/pages/sources/flow/sourceFlow.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from 'vitest' +import type { PluginInfo, SourceSpec } from '../../../api/client' +import { + addEntry, + idFromName, + newSourceSpec, + paletteEntries, + problemsByStep, + REQUIRED, + removeStep, + replaces, + STEP_ORDER, + stepOfEntry, + stepsOf, + stepTitle, + validId, +} from './sourceFlow' + +/** Every source spec the docs ship (docs/examples). */ +const files = import.meta.glob('../../../../../docs/examples/**/*.json', { import: 'default', eager: true }) +const examples = Object.entries(files).filter(([, v]) => typeof v === 'object' && v !== null && 'transport' in v && 'pipeline' in v) + +const plugin = (name: string, framing: Record | null = null): PluginInfo => + ({ name, kinds: ['codec'], enabled: true, default_options: { a: 1 }, framing }) as unknown as PluginInfo + +describe('stepsOf', () => { + it('finds examples to check', () => { + expect(examples.length).toBeGreaterThan(5) + }) + + it.each(examples)('%s: every required step, in pipeline order', (_, spec) => { + const steps = stepsOf(spec) + for (const r of REQUIRED) expect(steps).toContain(r) + expect(steps).toEqual(STEP_ORDER.filter((s) => steps.includes(s))) + }) + + it.each(examples)('%s: each added stage removes back to the source as it was', (_, spec) => { + for (const entry of paletteEntries([]).filter((e) => e.group === 'Stages')) { + const step = stepOfEntry(entry.type)! + const had = stepsOf(spec).includes(step) + const added = addEntry(spec, entry.type, []) + expect(stepsOf(added)).toContain(step) + if (had) expect(added).toBe(spec) + else expect(removeStep(added, step)).toEqual(spec) + } + }) +}) + +describe('a new source', () => { + it('has its Source, Decode, Map and Output steps', () => { + const s = newSourceSpec('ais', 'AIS', 'websocket') + expect(stepsOf(s)).toEqual(['source', 'decode', 'map', 'output']) + expect(s.transport).toEqual({ type: 'websocket', url: '' }) + expect(stepTitle(s, 'source')).toBe('WebSocket') + expect(stepTitle(s, 'decode')).toBe('JSON') + expect(stepTitle(s, 'output')).toBe('Tracks') + }) + + it('takes its id from its name', () => { + expect(idFromName('adsb.lol feed (EU)')).toBe('adsb-lol-feed-eu') + expect(validId(idFromName('adsb.lol feed (EU)'))).toBe(true) + expect(validId('Bad Id')).toBe(false) + expect(validId('')).toBe(false) + }) +}) + +describe('palette entries', () => { + const base = newSourceSpec('x', 'X', 'tcp_client') + + it('swap the transport, and the acceptance that belonged to the old one goes', () => { + const udp = addEntry(base, 'source:udp', []) + expect(udp.transport.type).toBe('udp') + const accepted: SourceSpec = { ...udp, unauthenticated: 'accepted' } + expect(replaces(accepted, 'source:mqtt')).toBe(true) + expect(replaces(accepted, 'source:udp')).toBe(false) + const mqtt = addEntry(accepted, 'source:mqtt', []) + expect(mqtt.transport).toEqual({ type: 'mqtt', url: '', topics: [] }) + expect(mqtt.unauthenticated).toBeUndefined() + expect(addEntry(accepted, 'source:udp', [])).toBe(accepted) + }) + + it('frame CoT on a stream by its end tag', () => { + const cot = addEntry(base, 'decode:cot_xml', []) + expect(cot.pipeline.codec).toEqual({ type: 'cot_xml' }) + expect(cot.transport.framing).toEqual({ type: 'end_tag', tag: '' }) + const server = addEntry(cot, 'source:tcp_server', []) + expect(server.transport.framing).toEqual({ type: 'end_tag', tag: '' }) + }) + + it('take a codec plugin with its options and framing', () => { + const p = plugin('stanag4607', { type: 'length_field', offset: 2, width: 'u32' }) + expect(paletteEntries([p]).map((e) => e.type)).toContain('decode:plugin:stanag4607') + const s = addEntry(base, 'decode:plugin:stanag4607', [p]) + expect(s.pipeline.codec).toEqual({ type: 'plugin', plugin: 'stanag4607', options: { a: 1 } }) + expect(s.transport.framing).toEqual({ type: 'length_field', offset: 2, width: 'u32' }) + expect(stepTitle(s, 'decode')).toBe('stanag4607') + expect(replaces(s, 'decode:plugin:stanag4607')).toBe(false) + expect(replaces(s, 'decode:json')).toBe(true) + }) + + it('set what the output reports', () => { + const plots = addEntry(base, 'output:detections', []) + expect(plots.reports).toBe('detections') + expect(stepTitle(plots, 'output')).toBe('Detections') + expect('reports' in addEntry(plots, 'output:tracks', [])).toBe(false) + }) + + it('never remove a required step', () => { + for (const r of REQUIRED) expect(removeStep(base, r)).toBe(base) + }) +}) + +describe('problemsByStep', () => { + it('puts each problem on its step, the rest with the source', () => { + const steps = stepsOf(newSourceSpec('x', 'X', 'tcp_client')) + const p = problemsByStep( + [ + { kind: 'incomplete', stage: 'source', field: 'host', message: 'the host is not set' }, + { kind: 'incomplete', stage: 'map', message: 'mapping has no rules' }, + { kind: 'incomplete', stage: 'static_join', message: 'not shown' }, + { kind: 'invalid', stage: null, field: 'name', message: 'source name must not be empty' }, + ], + steps, + ) + expect(p.byStep).toEqual({ source: ['the host is not set'], map: ['mapping has no rules'] }) + expect(p.general).toEqual(['not shown', 'source name must not be empty']) + }) +}) diff --git a/ui/src/pages/sources/flow/sourceFlow.ts b/ui/src/pages/sources/flow/sourceFlow.ts new file mode 100644 index 0000000..2c59005 --- /dev/null +++ b/ui/src/pages/sources/flow/sourceFlow.ts @@ -0,0 +1,292 @@ +/** + * A source as a flow of steps (#104): the canvas is a view of the source's spec. Its steps follow the + * pipeline's fixed order (crates/ot-source/src/pipeline.rs), so they connect themselves: adding a + * stage puts it in its place, and removing one closes the gap. Source, Decode, Map and Output are + * always there; the other stages are added from the palette and may be removed. + */ +import type { PluginInfo, SourceProblem, SourceSpec, SourceStage } from '../../../api/client' +import { DEFAULT_LINKS } from '../../../lib/pipeline' + +type Obj = Record +type Transport = SourceSpec['transport'] +type Codec = SourceSpec['pipeline']['codec'] + +/** The steps a source can have, in the order a frame meets them. */ +export const STEP_ORDER = ['source', 'decode', 'reject', 'map', 'join', 'registry', 'affiliation', 'filter', 'tracker', 'throttle', 'output'] as const +export type StepId = (typeof STEP_ORDER)[number] + +/** Steps every source has: they cannot be removed. */ +export const REQUIRED: ReadonlySet = new Set(['source', 'decode', 'map', 'output']) + +export type StepKind = 'source' | 'decode' | 'stage' | 'output' + +export const kindOf = (step: StepId): StepKind => (step === 'source' || step === 'decode' || step === 'output' ? step : 'stage') + +/** Transports, as the palette and the step heading name them, with what a new one starts as. */ +export const TRANSPORTS: Record = { + tcp_client: { label: 'TCP client', initial: { type: 'tcp_client', host: '', port: 0, framing: { type: 'lines' } } }, + tcp_server: { label: 'TCP server', initial: { type: 'tcp_server', bind: '0.0.0.0:8087', framing: { type: 'lines' } } }, + udp: { label: 'UDP', initial: { type: 'udp', bind: '0.0.0.0:6969' } }, + http_poll: { label: 'HTTP poll', initial: { type: 'http_poll', url: '', interval_secs: 5 } }, + websocket: { label: 'WebSocket', initial: { type: 'websocket', url: '' } }, + mqtt: { label: 'MQTT', initial: { type: 'mqtt', url: '', topics: [] } }, + grpc_client: { label: 'gRPC client', initial: { type: 'grpc_client', url: '', method: '' } }, + grpc_server: { label: 'gRPC server', initial: { type: 'grpc_server', bind: '0.0.0.0:50051' } }, + file: { label: 'File', initial: { type: 'file', path: '' } }, +} + +/** Codecs, likewise. A plugin codec is named by its plugin. */ +export const CODECS: Record = { + json: { label: 'JSON', initial: { type: 'json' } }, + cot_xml: { label: 'CoT XML', initial: { type: 'cot_xml' } }, + xml: { label: 'XML', initial: { type: 'xml', record_element: '' } }, + protobuf: { label: 'Protobuf', initial: { type: 'protobuf', files: {}, message: '' } }, +} + +/** The stages added from the palette, with what a new one starts as. */ +const STAGES: Record, { label: string; add: (s: SourceSpec) => SourceSpec; remove: (s: SourceSpec) => SourceSpec }> = { + reject: { + label: 'Reject', + add: (s) => withPipeline(s, { mapping: { ...s.pipeline.mapping, reject: [{ reason: 'no_position', when: { path: 'lat', exists: false } }] } }), + remove: (s) => withPipeline(s, { mapping: { ...s.pipeline.mapping, reject: undefined } }), + }, + join: { + label: 'Static join', + add: (s) => withPipeline(s, { static_join: {} }), + remove: (s) => withPipeline(s, { static_join: undefined }), + }, + registry: { + label: 'Entity links', + add: (s) => withPipeline(s, { registry: { links: DEFAULT_LINKS } }), + remove: (s) => withPipeline(s, { registry: undefined }), + }, + affiliation: { + label: 'Affiliation', + add: (s) => withPipeline(s, { affiliation: { country: 'platform.flag', friendly: [], hostile: [], neutral: [] } }), + remove: (s) => withPipeline(s, { affiliation: undefined }), + }, + filter: { + label: 'Filter', + add: (s) => withPipeline(s, { filter: {} }), + remove: (s) => withPipeline(s, { filter: undefined }), + }, + tracker: { + label: 'Tracker', + add: (s) => withPipeline(s, { tracker: { algorithm: 'gnn' } }), + remove: (s) => withPipeline(s, { tracker: undefined }), + }, + throttle: { + label: 'Throttle', + add: (s) => withPipeline(s, { throttle: { min_interval_secs: 5, min_move_m: 0, heartbeat_secs: 600 } }), + remove: (s) => withPipeline(s, { throttle: undefined }), + }, +} + +const OUTPUTS: Record<'tracks' | 'detections', string> = { tracks: 'Tracks', detections: 'Detections' } + +/** Set pipeline keys; one set to undefined is dropped, so the spec carries only what is configured. */ +function withPipeline(spec: SourceSpec, patch: Obj): SourceSpec { + const pipeline: Obj = { ...spec.pipeline, ...patch } + for (const [k, v] of Object.entries(patch)) if (v === undefined) delete pipeline[k] + const mapping = pipeline.mapping as Obj + if (mapping && mapping.reject === undefined) delete mapping.reject + return { ...spec, pipeline: pipeline as SourceSpec['pipeline'] } +} + +const rejects = (spec: SourceSpec) => ((spec.pipeline.mapping.reject as unknown[] | undefined) ?? []).length > 0 + +/** The steps this source has, in order. */ +export function stepsOf(spec: SourceSpec): StepId[] { + const p = spec.pipeline as Obj + const has: Record = { + source: true, + decode: true, + reject: rejects(spec), + map: true, + join: p.static_join !== undefined, + registry: p.registry !== undefined, + affiliation: p.affiliation !== undefined, + filter: p.filter !== undefined, + tracker: p.tracker !== undefined, + throttle: p.throttle !== undefined, + output: true, + } + return STEP_ORDER.filter((s) => has[s]) +} + +/** A step's heading: what it is for this source (its transport, codec, or output). */ +export function stepTitle(spec: SourceSpec, step: StepId): string { + switch (step) { + case 'source': + return TRANSPORTS[spec.transport.type]?.label ?? spec.transport.type + case 'decode': { + const c = spec.pipeline.codec + if (c.type === 'plugin') return String(c.plugin || 'Plugin') + return CODECS[c.type]?.label ?? c.type + } + case 'map': + return 'Map' + case 'output': + return OUTPUTS[spec.reports ?? 'tracks'] + default: + return STAGES[step].label + } +} + +/** The word under a step's heading. */ +export const KIND_WORD: Record = { source: 'Source', decode: 'Decode', stage: 'Stage', output: 'Output' } + +/** The trace's name for a step (crates/ot-source/src/trace.rs; `samplesAt`). */ +export const traceStage = (step: StepId): string => (step === 'source' ? 'transport' : step === 'output' ? 'publish' : step) + +/** The step a problem belongs to, from the server's pipeline stage; `null`: the source as a whole. */ +const STEP_OF_STAGE: Record = { + source: 'source', + decode: 'decode', + map: 'map', + static_join: 'join', + registry: 'registry', + affiliation: 'affiliation', + filter: 'filter', + tracker: 'tracker', + throttle: 'throttle', + output: 'output', +} + +export interface StepProblems { + byStep: Partial> + /** Problems with the source as a whole, or with a stage it does not show. */ + general: string[] +} + +/** The server's problems, each on its step. */ +export function problemsByStep(problems: SourceProblem[], steps: StepId[]): StepProblems { + const out: StepProblems = { byStep: {}, general: [] } + for (const p of problems) { + const step = p.stage ? STEP_OF_STAGE[p.stage] : null + if (step && steps.includes(step)) (out.byStep[step] ??= []).push(p.message) + else out.general.push(p.message) + } + return out +} + +/** A palette item's type: `source:`, `decode:` (`decode:plugin:`), `stage:` or `output:`. */ +export interface PaletteEntry { + type: string + label: string + group: 'Source' | 'Decode' | 'Stages' | 'Output' +} + +export function paletteEntries(plugins: PluginInfo[]): PaletteEntry[] { + return [ + ...Object.entries(TRANSPORTS).map(([t, d]) => ({ type: `source:${t}`, label: d.label, group: 'Source' as const })), + ...Object.entries(CODECS).map(([c, d]) => ({ type: `decode:${c}`, label: d.label, group: 'Decode' as const })), + ...plugins.filter((p) => p.kinds.includes('codec') && p.enabled).map((p) => ({ type: `decode:plugin:${p.name}`, label: p.name, group: 'Decode' as const })), + ...(Object.keys(STAGES) as (keyof typeof STAGES)[]).map((s) => ({ type: `stage:${s}`, label: STAGES[s].label, group: 'Stages' as const })), + ...(Object.keys(OUTPUTS) as (keyof typeof OUTPUTS)[]).map((r) => ({ type: `output:${r}`, label: OUTPUTS[r], group: 'Output' as const })), + ] +} + +/** The step a palette item makes or replaces. */ +export function stepOfEntry(type: string): StepId | null { + const [kind, sub] = type.split(':') + if (kind === 'source' || kind === 'decode' || kind === 'output') return kind + if (kind === 'stage' && sub in STAGES) return sub as StepId + return null +} + +const streamed = (t: Transport) => t.type === 'tcp_client' || t.type === 'tcp_server' + +/** CoT arrives as back-to-back XML events, framed by `` rather than by lines on a stream. */ +function cotFraming(t: Transport, codec: Codec): Transport { + const framing = t.framing as { type?: string } | undefined + return codec.type === 'cot_xml' && streamed(t) && framing?.type === 'lines' ? { ...t, framing: { type: 'end_tag', tag: '' } } : t +} + +/** + * Whether adding this palette item would replace settings the source has: a different transport, + * codec or output in place of the one it has. A stage it already has is not added again. + */ +export function replaces(spec: SourceSpec, type: string): boolean { + const [kind, sub, name] = type.split(':') + if (kind === 'source') return sub !== spec.transport.type + if (kind === 'decode') { + const c = spec.pipeline.codec + return sub === 'plugin' ? c.type !== 'plugin' || c.plugin !== name : sub !== c.type + } + return false +} + +/** The source with this palette item added: a stage put in its place, or a transport, codec or output swapped in. */ +export function addEntry(spec: SourceSpec, type: string, plugins: PluginInfo[]): SourceSpec { + const [kind, sub, name] = type.split(':') + switch (kind) { + case 'source': { + const d = TRANSPORTS[sub] + if (!d || sub === spec.transport.type) return spec + const next: SourceSpec = { ...spec, transport: cotFraming(structuredClone(d.initial), spec.pipeline.codec) } + // The acceptance belongs to the listener it was given for. + delete next.unauthenticated + return next + } + case 'decode': { + let codec: Codec | null = null + if (sub === 'plugin') { + const p = plugins.find((x) => x.name === name) + if (p && !(spec.pipeline.codec.type === 'plugin' && spec.pipeline.codec.plugin === p.name)) { + codec = { type: 'plugin', plugin: p.name, options: structuredClone(p.default_options) } + // A stream of the plugin's format is framed its way. + if (p.framing && streamed(spec.transport)) spec = { ...spec, transport: { ...spec.transport, framing: structuredClone(p.framing) } } + } + } else if (CODECS[sub] && sub !== spec.pipeline.codec.type) { + codec = structuredClone(CODECS[sub].initial) + } + if (!codec) return spec + return { ...withPipeline(spec, { codec }), transport: cotFraming(spec.transport, codec) } + } + case 'stage': { + const s = STAGES[sub as keyof typeof STAGES] + if (!s || stepsOf(spec).includes(sub as StepId)) return spec + return s.add(spec) + } + case 'output': { + if (sub !== 'tracks' && sub !== 'detections') return spec + const next: SourceSpec = { ...spec, reports: sub } + if (sub === 'tracks') delete next.reports + return next + } + } + return spec +} + +/** The source without this stage. Source, Decode, Map and Output stay. */ +export function removeStep(spec: SourceSpec, step: StepId): SourceSpec { + if (REQUIRED.has(step)) return spec + return STAGES[step as keyof typeof STAGES].remove(spec) +} + +/** Where each step sits: one column, top to bottom. */ +export const STEP_GAP = 96 +export const positionOf = (index: number) => ({ x: 0, y: index * STEP_GAP }) + +/** A new source: its transport, an empty JSON decode and mapping, reporting tracks. */ +export function newSourceSpec(id: string, name: string, transport: string): SourceSpec { + return { + id, + name, + transport: structuredClone(TRANSPORTS[transport]?.initial ?? TRANSPORTS.http_poll.initial), + pipeline: { codec: { type: 'json' }, mapping: { rules: [] } }, + } +} + +/** A source id from its name: lowercase letters, digits, `-` and `_`, at most 64 (crates/ot-source/src/source.rs). */ +export function idFromName(name: string): string { + return name + .toLowerCase() + .normalize('NFKD') + .replace(/[^a-z0-9_-]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 64) +} + +export const validId = (id: string) => /^[a-z0-9_-]{1,64}$/.test(id) diff --git a/ui/src/pages/sources/flow/stepIcons.tsx b/ui/src/pages/sources/flow/stepIcons.tsx new file mode 100644 index 0000000..1198754 --- /dev/null +++ b/ui/src/pages/sources/flow/stepIcons.tsx @@ -0,0 +1,76 @@ +import type { ReactNode } from 'react' +import { + TbAddressBook, + TbApi, + TbArrowsExchange, + TbBan, + TbBinary, + TbBraces, + TbBroadcast, + TbClockPause, + TbCode, + TbFile, + TbFilter, + TbFlag, + TbLink, + TbMessages, + TbNetwork, + TbPlugConnected, + TbPuzzle, + TbRadar, + TbRoute, + TbSend, + TbServer, + TbTargetArrow, + TbWorldDownload, +} from 'react-icons/tb' +import type { SourceSpec } from '../../../api/client' +import type { StepId } from './sourceFlow' + +const TRANSPORT_ICONS: Record = { + tcp_client: , + tcp_server: , + udp: , + http_poll: , + websocket: , + mqtt: , + grpc_client: , + grpc_server: , + file: , +} + +const CODEC_ICONS: Record = { + json: , + cot_xml: , + xml: , + protobuf: , + plugin: , +} + +const STAGE_ICONS: Partial> = { + reject: , + map: , + join: , + registry: , + affiliation: , + filter: , + tracker: , + throttle: , + output: , +} + +/** A step's icon, for the source as it is. */ +export function stepIcon(spec: SourceSpec, step: StepId): ReactNode { + if (step === 'source') return TRANSPORT_ICONS[spec.transport.type] + if (step === 'decode') return CODEC_ICONS[spec.pipeline.codec.type] + return STAGE_ICONS[step] +} + +/** A palette item's icon, from its type (`source:udp`, `decode:plugin:x`, `stage:filter`, `output:tracks`). */ +export function entryIcon(type: string): ReactNode { + const [kind, sub] = type.split(':') + if (kind === 'source') return TRANSPORT_ICONS[sub] + if (kind === 'decode') return CODEC_ICONS[sub] + if (kind === 'stage') return STAGE_ICONS[sub as StepId] + return STAGE_ICONS.output +} From ce510560ab6375c31c2c6b811dca1255967e3be8 Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Wed, 21 Oct 2026 16:21:03 -0500 Subject: [PATCH 07/16] Sources: review fixes for the flow editor (#104) - The JSON view takes only text with a source's shape, so a half-typed pipeline cannot crash the editor. - A step taken out another way (JSON, emptying the Reject rules) is no longer selected, so its settings are not shown without a value. - Switching codec takes away the framing the old one set (CoT's end tag, a plugin's own); framing set by hand stays. - The dry run says why a pipeline cannot run when the checks did not catch it, on its stage, instead of 'invalid source'. - The sources list stops polling while a source is open. - The editor's state badge is sourceState's; the dead Transport-tab codec helpers are gone. Co-Authored-By: Claude Opus 5.5 --- crates/ot-server/src/api.rs | 23 ++++++++++++------ ui/src/lib/pipeline.test.ts | 17 +------------ ui/src/lib/pipeline.ts | 16 ------------- ui/src/pages/sources/SourceEditor.tsx | 25 ++++++++++++++------ ui/src/pages/sources/SourcesPage.tsx | 14 ++++------- ui/src/pages/sources/flow/sourceFlow.test.ts | 5 ++++ ui/src/pages/sources/flow/sourceFlow.ts | 18 +++++++++++++- 7 files changed, 62 insertions(+), 56 deletions(-) diff --git a/crates/ot-server/src/api.rs b/crates/ot-server/src/api.rs index 519a185..e603984 100644 --- a/crates/ot-server/src/api.rs +++ b/crates/ot-server/src/api.rs @@ -107,8 +107,8 @@ impl Checked { fn dry_run(&self) -> Result { self.clone().savable()?; let pipeline = - Pipeline::new(self.spec.id.clone(), self.spec.pipeline.clone()).map_err(|_| { - let blocking: Vec = self + Pipeline::new(self.spec.id.clone(), self.spec.pipeline.clone()).map_err(|e| { + let mut blocking: Vec = self .problems .iter() .filter(|p| { @@ -116,11 +116,20 @@ impl Checked { }) .cloned() .collect(); - ApiError::source_problems(if blocking.is_empty() { - &self.problems - } else { - &blocking - }) + // What the checks did not see (a tracker plugin not loaded, say), on its stage. + if blocking.is_empty() { + blocking.push(Problem { + kind: ProblemKind::Incomplete, + stage: Some(match e { + ot_source::pipeline::PipelineError::Mapping(_) => Stage::Map, + ot_source::pipeline::PipelineError::Tracker(_) => Stage::Tracker, + ot_source::pipeline::PipelineError::Codec(_) => Stage::Decode, + }), + field: None, + message: e.to_string(), + }); + } + ApiError::source_problems(&blocking) })?; Ok(match &self.schema { Some(schema) => pipeline.with_schema(schema.clone()), diff --git a/ui/src/lib/pipeline.test.ts b/ui/src/lib/pipeline.test.ts index 7ec2a8e..b596add 100644 --- a/ui/src/lib/pipeline.test.ts +++ b/ui/src/lib/pipeline.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import type { SchemaOverview, SourceSpec } from '../api/client' -import { describeCondition, describeValue, destinations, fieldMap, pipelineStages, transportCodec, valueSources } from './pipeline' +import { describeCondition, describeValue, destinations, fieldMap, pipelineStages, valueSources } from './pipeline' const schema = { published_core: [], @@ -105,18 +105,3 @@ describe('pipelineStages', () => { }) }) -describe('transportCodec', () => { - it('leaves the codecs the Decode stage edits to it', () => { - expect(transportCodec(undefined)).toBeNull() - expect(transportCodec({ type: 'json', records: 'ac' })).toBeNull() - expect(transportCodec({ type: 'cot_xml' })).toBeNull() - expect(transportCodec({ type: 'xml', record_element: 'r' })).toBeNull() - }) - - it('shows protobuf and plugin codecs with their message or plugin', () => { - expect(transportCodec({ type: 'protobuf', files: {}, message: 'acme.v1.Batch' })).toEqual({ type: 'protobuf', name: 'acme.v1.Batch' }) - expect(transportCodec({ type: 'protobuf', files: {}, message: '' })).toEqual({ type: 'protobuf' }) - expect(transportCodec({ type: 'plugin', plugin: 'sapient', options: {} })).toEqual({ type: 'plugin', name: 'sapient' }) - expect(transportCodec({ type: 'future_codec' })).toEqual({ type: 'future_codec' }) - }) -}) diff --git a/ui/src/lib/pipeline.ts b/ui/src/lib/pipeline.ts index ec905ad..2aee9dc 100644 --- a/ui/src/lib/pipeline.ts +++ b/ui/src/lib/pipeline.ts @@ -284,22 +284,6 @@ export function pipelineStages(spec: SourceSpec): Stage[] { return out } -// --- Decode stage ------------------------------------------------------------------------ - -/** The codecs the Decode stage edits; the others (protobuf, plugins) are chosen on the Transport tab. */ -export const DECODE_CODECS = ['json', 'cot_xml', 'xml'] - -/** - * A codec the Decode stage only shows: its type and its message or plugin, or null for one it - * edits. Choosing a Decode format would replace such a codec's schema, plugin and options. - */ -export function transportCodec(codec: Obj | undefined): { type: string; name?: string } | null { - const type = String(codec?.type ?? 'json') - if (DECODE_CODECS.includes(type)) return null - const name = type === 'protobuf' ? codec?.message : type === 'plugin' ? codec?.plugin : undefined - return typeof name === 'string' && name ? { type, name } : { type } -} - // --- Entity links -------------------------------------------------------------------------- export interface EntityLink { diff --git a/ui/src/pages/sources/SourceEditor.tsx b/ui/src/pages/sources/SourceEditor.tsx index 29c6f70..c7b4789 100644 --- a/ui/src/pages/sources/SourceEditor.tsx +++ b/ui/src/pages/sources/SourceEditor.tsx @@ -43,6 +43,15 @@ import { } from './flow/sourceFlow' const REFRESH_MS = 5000 + +/** Whether parsed JSON has a source's shape, enough to show as steps (the server checks the rest). */ +function isSpecShaped(v: unknown): v is SourceSpec { + const o = (x: unknown): x is Record => typeof x === 'object' && x !== null && !Array.isArray(x) + if (!o(v) || typeof v.id !== 'string' || typeof v.name !== 'string' || !o(v.transport) || typeof v.transport.type !== 'string') return false + const p = v.pipeline + if (!o(p) || !o(p.codec) || typeof p.codec.type !== 'string' || !o(p.mapping) || !Array.isArray(p.mapping.rules)) return false + return p.mapping.reject === undefined || Array.isArray(p.mapping.reject) +} const PREVIEW_DELAY_MS = 600 const NODE_TYPES = { step: StepNode } @@ -180,6 +189,8 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o const problems = useMemo(() => live.problems ?? row?.problems ?? [], [live.problems, row?.problems]) const steps = useMemo(() => (spec ? stepsOf(spec) : []), [spec]) const byStep = useMemo(() => problemsByStep(problems, steps), [problems, steps]) + // A step taken out another way (the JSON view, emptying the Reject rules) is no longer selected. + const current = selected && steps.includes(selected) ? selected : null const edit = (next: SourceSpec) => { setSpec(next) @@ -194,7 +205,7 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o id: step, type: 'step', position: positionOf(i), - selected: step === selected, + selected: step === current, draggable: false, deletable: !REQUIRED.has(step), data: { @@ -205,7 +216,7 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o problems: byStep.byStep[step] ?? [], } satisfies StepView, })) - }, [spec, steps, selected, byStep]) + }, [spec, steps, current, byStep]) const edges: FlowCanvasEdge[] = useMemo( () => steps.slice(1).map((step, i) => ({ id: `${steps[i]}->${step}`, source: steps[i], target: step, deletable: false })), @@ -319,7 +330,7 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o ) } - const state = row.draft ? { label: 'draft', color: 'grey' as const } : sourceState(row) + const state = sourceState(row) const general = [...(notice ? [notice] : []), ...byStep.general] return ( @@ -374,11 +385,11 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o style={{ minHeight: 0 }} />
{step.summary && {step.summary}} - {step.problems.map((message) => ( - - {message} + {failing && ( + + {step.problems[0]} - ))} + )} + {step.problems.length > 1 && +{step.problems.length - 1} more} {step.kind !== 'output' && }
) diff --git a/ui/src/pages/sources/flow/sourceFlow.ts b/ui/src/pages/sources/flow/sourceFlow.ts index bb7ba1c..96bed3e 100644 --- a/ui/src/pages/sources/flow/sourceFlow.ts +++ b/ui/src/pages/sources/flow/sourceFlow.ts @@ -283,7 +283,7 @@ export function removeStep(spec: SourceSpec, step: StepId): SourceSpec { } /** Where each step sits: one column, top to bottom. */ -export const STEP_GAP = 96 +export const STEP_GAP = 112 export const positionOf = (index: number) => ({ x: 0, y: index * STEP_GAP }) /** A new source: its transport, an empty JSON decode and mapping, reporting tracks. */ From e1a429e5bf2579f8fbb4f0d32508cf10c5c125dc Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Wed, 21 Oct 2026 16:58:09 -0500 Subject: [PATCH 10/16] Sources: the JSON view fills the page's height (#104) Co-Authored-By: Claude Opus 5.5 --- ui/src/index.css | 2 +- ui/src/pages/sources/SourceEditor.tsx | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/ui/src/index.css b/ui/src/index.css index 1cc69b2..f90431c 100644 --- a/ui/src/index.css +++ b/ui/src/index.css @@ -948,7 +948,7 @@ body { .source-editor-json { flex: 1; min-height: 0; - overflow: auto; + overflow: hidden; } /* Settings beside their samples; stacked on narrower screens. */ diff --git a/ui/src/pages/sources/SourceEditor.tsx b/ui/src/pages/sources/SourceEditor.tsx index 5982dce..7e49c08 100644 --- a/ui/src/pages/sources/SourceEditor.tsx +++ b/ui/src/pages/sources/SourceEditor.tsx @@ -94,6 +94,25 @@ function SourcePanel({ spec, row, readOnly, onChange }: { spec: SourceSpec; row: ) } +/** The height of the element the ref is on, as it resizes: the JSON view fills the page with it. */ +function useBoxHeight(): [(el: HTMLElement | null) => void, number] { + const [height, setHeight] = useState(400) + const observer = useRef(null) + const ref = useCallback((el: HTMLElement | null) => { + observer.current?.disconnect() + observer.current = null + if (!el || typeof ResizeObserver === 'undefined') return + const ro = new ResizeObserver(() => { + const h = Math.floor(el.getBoundingClientRect().height) + if (h > 0) setHeight((old) => (old === h ? old : h)) + }) + ro.observe(el) + observer.current = ro + }, []) + useEffect(() => () => observer.current?.disconnect(), []) + return [ref, height] +} + /** * A dry run of the source as edited over its stored samples, re-run shortly after it stops changing: * what stops it running (each problem on its stage) and its samples stage by stage. A draft runs as @@ -145,6 +164,7 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o const [selected, setSelected] = useState(null) const [view, setView] = useState<'flow' | 'json'>('flow') const [jsonText, setJsonText] = useState('') + const [jsonBox, jsonHeight] = useBoxHeight() const [saving, setSaving] = useState(false) const [saved, setSaved] = useState(false) const [notice, setNotice] = useState(null) @@ -405,9 +425,11 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o
) : ( -
+
{ From 04a210be488545696a901222d7f596da8b739083 Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Wed, 21 Oct 2026 17:14:09 -0500 Subject: [PATCH 11/16] ot-source: a source's processing as a flow of steps (#104) The pipeline is a flow: one input (transport and codec), then Reject, Map, Filter, Entity links, Affiliation, Tracker and Throttle steps the user connects, any number of each, branching to the source's one Output (system tracks) and any NATS topics (mapped data as JSON). The older fixed form reads as the same stages in a line, so every existing spec, example and test runs unchanged. Problems name their step. Co-Authored-By: Claude Opus 5.5 --- crates/ot-source/src/flow.rs | 668 ++++++++++ crates/ot-source/src/lib.rs | 1 + crates/ot-source/src/pipeline.rs | 1166 ++++++++++------- crates/ot-source/src/source.rs | 267 ++-- crates/ot-source/src/trace.rs | 18 +- .../tests/sapient_acoustic_example.rs | 2 +- crates/ot-source/tests/stanag4607.rs | 6 +- 7 files changed, 1554 insertions(+), 574 deletions(-) create mode 100644 crates/ot-source/src/flow.rs diff --git a/crates/ot-source/src/flow.rs b/crates/ot-source/src/flow.rs new file mode 100644 index 0000000..92244f2 --- /dev/null +++ b/crates/ot-source/src/flow.rs @@ -0,0 +1,668 @@ +//! A source's processing as a flow of steps (kineticlogic-io/OpenTrack#104): +//! one input (the transport and its codec), then steps the user connects, +//! each taking one input and feeding any number of others, ending at the +//! source's one Output (system tracks, to correlation) and any NATS topics +//! (its mapped data, published as JSON without correlation). +//! +//! Records flow from the input through Reject steps to a Map, which turns +//! them into observations; Filter, Entity links, Affiliation, Tracker and +//! Throttle steps work on observations; Output and NATS topics take them. +//! +//! A spec in the older fixed form (`mapping`, `filter`, `tracker`… under +//! `pipeline`) still reads: it becomes the same steps in a line. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; + +use crate::codec::CodecConfig; +use crate::mapping::{MappingSpec, RejectRule}; +use crate::pipeline::{AffiliationStage, FilterSpec, StaticJoinSpec, ThrottleSpec}; +use crate::registry::RegistryStage; +use crate::tracker::TrackerSpec; + +/// The input step's id: the transport and codec, where every flow starts. +pub const INPUT: &str = "input"; + +/// The most steps a flow may have. +pub const MAX_NODES: usize = 200; + +/// Everything between the transport and the source's outputs. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PipelineSpec { + /// How the input's frames become records. + pub codec: CodecConfig, + pub nodes: Vec, + pub edges: Vec, +} + +/// One step of a flow. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct FlowNode { + /// 1-64 of A-Z, a-z, 0-9, `-` and `_`; the input is `input`. + pub id: String, + #[serde(flatten)] + pub step: Step, + /// Where the editor draws it; nothing runs on it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub position: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Position { + pub x: f64, + pub y: f64, +} + +/// A connection: `target` takes what `source` passes on. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FlowEdge { + pub source: String, + pub target: String, +} + +/// What a step does, with its settings. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", content = "config", rename_all = "snake_case")] +pub enum Step { + /// The transport and codec (set on the source); only `input`. + Input, + /// Drop records by condition, each rule with a reason. + Reject(Vec), + /// Turn records into observations. + Map(MapStep), + /// Keep or drop observations. + Filter(FilterSpec), + /// Find each track's entity by its identifiers (Entity links). + Registry(RegistryStage), + Affiliation(AffiliationStage), + /// Form tracks from detections. + Tracker(TrackerSpec), + Throttle(ThrottleSpec), + /// The source's system tracks, to correlation. + Output, + /// Publish what reaches it, as JSON, on a NATS subject, without + /// correlation. + Nats(NatsStep), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MapStep { + pub mapping: MappingSpec, + /// How long identity fields from static rules are kept, to fill the + /// same track's later reports. + #[serde(default)] + pub static_join: StaticJoinSpec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NatsStep { + pub subject: String, +} + +/// What passes along a connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Data { + /// Decoded records, before a Map. + Records, + /// Observations, from a Map on. + Observations, +} + +impl Step { + /// What it takes; `None`: nothing (the input). + pub fn takes(&self) -> Option { + match self { + Step::Input => None, + Step::Reject(_) | Step::Map(_) => Some(Data::Records), + _ => Some(Data::Observations), + } + } + + /// What it passes on; `None`: nothing (an output). + pub fn gives(&self) -> Option { + match self { + Step::Input | Step::Reject(_) => Some(Data::Records), + Step::Output | Step::Nats(_) => None, + _ => Some(Data::Observations), + } + } + + /// The word for it in messages. + pub fn name(&self) -> &'static str { + match self { + Step::Input => "input", + Step::Reject(_) => "reject", + Step::Map(_) => "map", + Step::Filter(_) => "filter", + Step::Registry(_) => "entity links", + Step::Affiliation(_) => "affiliation", + Step::Tracker(_) => "tracker", + Step::Throttle(_) => "throttle", + Step::Output => "output", + Step::Nats(_) => "NATS topic", + } + } +} + +/// A problem with a flow's shape, on the step it concerns (`None`: the +/// flow as a whole). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FlowProblem { + /// It cannot be saved (rather than only not run yet). + pub invalid: bool, + pub node: Option, + pub message: String, +} + +impl PipelineSpec { + /// A step by id. + pub fn node(&self, id: &str) -> Option<&FlowNode> { + self.nodes.iter().find(|n| n.id == id) + } + + /// Every Map's mapping, in step order. + pub fn mappings(&self) -> impl Iterator { + self.nodes.iter().filter_map(|n| match &n.step { + Step::Map(m) => Some(&m.mapping), + _ => None, + }) + } + + /// The extension schema version the flow's mappings target (they share + /// one; see [`Self::shape_problems`]); 1 without a Map. + pub fn schema_version(&self) -> u32 { + self.mappings().next().map_or(1, |m| m.schema_version) + } + + /// The Output step, if the flow has one. + pub fn output(&self) -> Option<&FlowNode> { + self.nodes.iter().find(|n| matches!(n.step, Step::Output)) + } + + /// The steps from the input to `id`, input first (empty if `id` is not + /// reached from the input). + pub fn path_to(&self, id: &str) -> Vec<&FlowNode> { + let parent: HashMap<&str, &str> = self + .edges + .iter() + .map(|e| (e.target.as_str(), e.source.as_str())) + .collect(); + let mut out = Vec::new(); + let mut at = id; + let mut seen = HashSet::new(); + loop { + if !seen.insert(at) { + return Vec::new(); + } + match self.node(at) { + Some(n) => out.push(n), + None if at == INPUT => {} + None => return Vec::new(), + } + if at == INPUT { + break; + } + match parent.get(at) { + Some(p) => at = p, + None => return Vec::new(), + } + } + out.reverse(); + out + } + + /// The Entity links settings the source's system tracks pass through: + /// the last such step before the Output, else the defaults. + pub fn output_registry(&self) -> RegistryStage { + self.output() + .map(|o| self.path_to(&o.id)) + .unwrap_or_default() + .into_iter() + .rev() + .find_map(|n| match &n.step { + Step::Registry(r) => Some(r.clone()), + _ => None, + }) + .unwrap_or_default() + } + + /// Whether the source's system tracks come through a Tracker. + pub fn output_tracked(&self) -> bool { + self.output().is_some_and(|o| { + self.path_to(&o.id) + .iter() + .any(|n| matches!(n.step, Step::Tracker(_))) + }) + } + + /// The steps' children, by id. + pub fn children(&self) -> HashMap<&str, Vec<&str>> { + let mut out: HashMap<&str, Vec<&str>> = HashMap::new(); + for e in &self.edges { + out.entry(e.source.as_str()) + .or_default() + .push(e.target.as_str()); + } + out + } + + /// What is wrong with the flow's shape: ids, connections, what each + /// step takes and gives, and whether it leads to an output. Problems + /// with a step's own settings are the step's (see `SourceSpec::problems`). + pub fn shape_problems(&self) -> Vec { + let mut out = Vec::new(); + let invalid = |node: Option<&str>, m: String| FlowProblem { + invalid: true, + node: node.map(str::to_owned), + message: m, + }; + let incomplete = |node: Option<&str>, m: String| FlowProblem { + invalid: false, + node: node.map(str::to_owned), + message: m, + }; + if self.nodes.len() > MAX_NODES { + out.push(invalid( + None, + format!("a flow has at most {MAX_NODES} steps"), + )); + } + let mut ids = HashSet::new(); + for n in &self.nodes { + let ok = !n.id.is_empty() + && n.id.len() <= 64 + && n.id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_'); + if !ok { + out.push(invalid( + Some(&n.id), + format!( + "step id {:?}: 1-64 of letters, digits, - and _", + n.id + ), + )); + } + if !ids.insert(n.id.as_str()) { + out.push(invalid(Some(&n.id), format!("two steps are called {:?}", n.id))); + } + if (n.id == INPUT) != matches!(n.step, Step::Input) { + out.push(invalid( + Some(&n.id), + "only the step called \"input\" is the input, and it is the input".into(), + )); + } + } + let outputs = self + .nodes + .iter() + .filter(|n| matches!(n.step, Step::Output)) + .count(); + if outputs > 1 { + out.push(invalid(None, "a source has one Output".into())); + } + let gives = |id: &str| match self.node(id) { + Some(n) => n.step.gives(), + None if id == INPUT => Some(Data::Records), + None => None, + }; + let mut incoming: HashMap<&str, usize> = HashMap::new(); + let mut pairs = HashSet::new(); + for e in &self.edges { + let (from, to) = (self.node(&e.source), self.node(&e.target)); + if from.is_none() && e.source != INPUT { + out.push(invalid(None, format!("a connection from {:?}, which is not a step", e.source))); + continue; + } + let Some(to) = to else { + out.push(invalid(None, format!("a connection to {:?}, which is not a step", e.target))); + continue; + }; + if !pairs.insert((e.source.as_str(), e.target.as_str())) { + out.push(invalid(Some(&to.id), "it is connected twice from one step".into())); + continue; + } + *incoming.entry(e.target.as_str()).or_default() += 1; + match (gives(&e.source), to.step.takes()) { + (_, None) => out.push(invalid(Some(&to.id), "nothing connects into the input".into())), + (None, _) => out.push(invalid( + Some(&e.source), + format!("a {} passes nothing on", from.map_or("step", |f| f.step.name())), + )), + (Some(Data::Records), Some(Data::Observations)) => out.push(invalid( + Some(&to.id), + format!("a {} takes observations: put a Map before it", to.step.name()), + )), + (Some(Data::Observations), Some(Data::Records)) => out.push(invalid( + Some(&to.id), + format!("a {} takes records, before a Map", to.step.name()), + )), + _ => {} + } + } + for (id, n) in &incoming { + if *n > 1 { + out.push(invalid(Some(id), "a step takes one input".into())); + } + } + if !out.is_empty() { + return out; + } + // Every step is reached from the input, and leads to an output. + let children = self.children(); + let mut reached = HashSet::new(); + let mut stack = vec![INPUT]; + while let Some(at) = stack.pop() { + if reached.insert(at) { + stack.extend(children.get(at).into_iter().flatten().copied()); + } + } + // A loop is a step with one input that the input never reaches. + let mut sinks = 0; + for n in &self.nodes { + if n.id == INPUT { + continue; + } + if !reached.contains(n.id.as_str()) { + if incoming.contains_key(n.id.as_str()) { + let looped = self.in_loop(&n.id); + if looped { + out.push(invalid(Some(&n.id), "the flow loops back on itself".into())); + continue; + } + } + out.push(incomplete(Some(&n.id), "this step has no input".into())); + continue; + } + if n.step.gives().is_none() { + sinks += 1; + } else if !children.contains_key(n.id.as_str()) { + out.push(incomplete(Some(&n.id), "this step leads nowhere".into())); + } + } + if out.iter().any(|p| p.invalid) { + return out; + } + if !children.contains_key(INPUT) { + out.push(incomplete(Some(INPUT), "the input leads nowhere".into())); + } + if sinks == 0 { + out.push(incomplete( + None, + "a flow needs an Output or a NATS topic".into(), + )); + } + let versions: BTreeSet = self.mappings().map(|m| m.schema_version).collect(); + if versions.len() > 1 { + out.push(incomplete( + None, + format!("the Maps target different schema versions ({versions:?}); use one"), + )); + } + out + } + + /// Whether following inputs back from `id` comes round to it. + fn in_loop(&self, id: &str) -> bool { + let parent: HashMap<&str, &str> = self + .edges + .iter() + .map(|e| (e.target.as_str(), e.source.as_str())) + .collect(); + let mut at = id; + let mut seen = HashSet::new(); + while let Some(p) = parent.get(at) { + if *p == id { + return true; + } + if !seen.insert(*p) { + return false; + } + at = p; + } + false + } +} + +/// The older fixed form: one of each stage, in a fixed order. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct FixedPipeline { + codec: CodecConfig, + mapping: MappingSpec, + #[serde(default)] + static_join: StaticJoinSpec, + #[serde(default)] + registry: Option, + #[serde(default)] + affiliation: Option, + #[serde(default)] + filter: Option, + #[serde(default)] + tracker: Option, + #[serde(default)] + throttle: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct GraphPipeline { + codec: CodecConfig, + #[serde(default)] + nodes: Vec, + #[serde(default)] + edges: Vec, +} + +impl From for PipelineSpec { + /// The same stages, in a line, with the ids the stages had. The entity + /// stage, which always ran, is a step of its own. + fn from(p: FixedPipeline) -> Self { + let mut mapping = p.mapping; + let reject = std::mem::take(&mut mapping.reject); + let mut nodes = vec![node(INPUT, Step::Input)]; + if !reject.is_empty() { + nodes.push(node("reject", Step::Reject(reject))); + } + nodes.push(node( + "map", + Step::Map(MapStep { + mapping, + static_join: p.static_join, + }), + )); + nodes.push(node( + "registry", + Step::Registry(p.registry.unwrap_or_default()), + )); + if let Some(a) = p.affiliation { + nodes.push(node("affiliation", Step::Affiliation(a))); + } + if let Some(f) = p.filter { + nodes.push(node("filter", Step::Filter(f))); + } + if let Some(t) = p.tracker { + nodes.push(node("tracker", Step::Tracker(t))); + } + if let Some(t) = p.throttle { + nodes.push(node("throttle", Step::Throttle(t))); + } + nodes.push(node("output", Step::Output)); + let edges = nodes + .windows(2) + .map(|w| FlowEdge { + source: w[0].id.clone(), + target: w[1].id.clone(), + }) + .collect(); + PipelineSpec { + codec: p.codec, + nodes, + edges, + } + } +} + +fn node(id: &str, step: Step) -> FlowNode { + FlowNode { + id: id.to_owned(), + step, + position: None, + } +} + +impl<'de> Deserialize<'de> for PipelineSpec { + /// A flow (`nodes` and `edges`), or the older fixed form, read as the + /// same stages in a line. The input step is added if left out. + fn deserialize>(d: D) -> Result { + let v = Value::deserialize(d)?; + let graph = v.get("nodes").is_some() || v.get("edges").is_some(); + let mut spec: PipelineSpec = if graph { + let g: GraphPipeline = serde_json::from_value(v).map_err(serde::de::Error::custom)?; + PipelineSpec { + codec: g.codec, + nodes: g.nodes, + edges: g.edges, + } + } else { + serde_json::from_value::(v) + .map_err(serde::de::Error::custom)? + .into() + }; + if spec.node(INPUT).is_none() { + spec.nodes.insert(0, node(INPUT, Step::Input)); + } + Ok(spec) + } +} + +impl PipelineSpec { + /// A flow in a line through these steps: input, then each, ending at an + /// Output. For tests and simple sources. + pub fn line(codec: CodecConfig, steps: Vec<(&str, Step)>) -> Self { + let mut nodes = vec![node(INPUT, Step::Input)]; + nodes.extend(steps.into_iter().map(|(id, s)| node(id, s))); + let edges = nodes + .windows(2) + .map(|w| FlowEdge { + source: w[0].id.clone(), + target: w[1].id.clone(), + }) + .collect(); + PipelineSpec { + codec, + nodes, + edges, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn spec(v: Value) -> PipelineSpec { + serde_json::from_value(v).unwrap() + } + + fn map() -> Value { + json!({"rules": [{"name": "r", "key": "id", + "fields": {"position.latitude": "lat", "position.longitude": "lon"}}]}) + } + + #[test] + fn the_fixed_form_reads_as_its_stages_in_a_line() { + let p = spec(json!({ + "codec": {"type": "json"}, + "mapping": {"reject": [{"reason": "x", "when": {"path": "a", "exists": false}}], "rules": map()["rules"]}, + "filter": {"drop_if": {"path": "name", "eq": "N"}}, + "throttle": {"min_interval_secs": 5} + })); + let ids: Vec<&str> = p.nodes.iter().map(|n| n.id.as_str()).collect(); + assert_eq!(ids, ["input", "reject", "map", "registry", "filter", "throttle", "output"]); + assert_eq!(p.edges.len(), 6); + assert!(p.shape_problems().is_empty(), "{:?}", p.shape_problems()); + let Step::Map(m) = &p.node("map").unwrap().step else { panic!() }; + assert!(m.mapping.reject.is_empty(), "the rejects moved to their own step"); + // Saved, it reads back the same. + let again: PipelineSpec = serde_json::from_value(serde_json::to_value(&p).unwrap()).unwrap(); + assert_eq!(again, p); + assert!(p.output_registry() == RegistryStage::default() && !p.output_tracked()); + } + + #[test] + fn a_flow_branches_to_an_output_and_nats_topics() { + let p = spec(json!({ + "codec": {"type": "json"}, + "nodes": [ + {"id": "m", "type": "map", "config": {"mapping": map()}}, + {"id": "f1", "type": "filter", "config": {"keep_if": {"path": "name", "exists": true}}}, + {"id": "f2", "type": "filter", "config": {}}, + {"id": "out", "type": "output"}, + {"id": "raw", "type": "nats", "config": {"subject": "raw.feed"}} + ], + "edges": [ + {"source": "input", "target": "m"}, {"source": "m", "target": "f1"}, + {"source": "f1", "target": "f2"}, {"source": "f2", "target": "out"}, + {"source": "m", "target": "raw"} + ] + })); + assert_eq!(p.nodes[0].id, "input", "the input is added"); + assert!(p.shape_problems().is_empty(), "{:?}", p.shape_problems()); + let path: Vec<&str> = p.path_to("out").iter().map(|n| n.id.as_str()).collect(); + assert_eq!(path, ["input", "m", "f1", "f2", "out"]); + } + + #[test] + fn shape_problems_name_their_step() { + let base = || json!({"codec": {"type": "json"}, "nodes": [ + {"id": "m", "type": "map", "config": {"mapping": map()}}, + {"id": "f", "type": "filter", "config": {}}, + {"id": "out", "type": "output"} + ], "edges": [{"source": "input", "target": "m"}, {"source": "m", "target": "f"}, {"source": "f", "target": "out"}]}); + let problems = |v: Value| -> Vec<(bool, Option)> { + spec(v).shape_problems().into_iter().map(|p| (p.invalid, p.node)).collect() + }; + // A filter before the map. + let mut v = base(); + v["edges"] = json!([{"source": "input", "target": "f"}, {"source": "f", "target": "m"}, {"source": "m", "target": "out"}]); + assert_eq!(problems(v), [(true, Some("f".into())), (true, Some("m".into()))]); + // Two inputs into one step. + let mut v = base(); + v["edges"].as_array_mut().unwrap().push(json!({"source": "input", "target": "f"})); + assert!(problems(v).contains(&(true, Some("f".into())))); + // A loop. + let mut v = base(); + v["nodes"].as_array_mut().unwrap().push(json!({"id": "g", "type": "filter", "config": {}})); + v["nodes"].as_array_mut().unwrap().push(json!({"id": "h", "type": "filter", "config": {}})); + v["edges"].as_array_mut().unwrap().extend([json!({"source": "g", "target": "h"}), json!({"source": "h", "target": "g"})]); + assert!(problems(v).contains(&(true, Some("g".into())))); + // Two outputs. + let mut v = base(); + v["nodes"].as_array_mut().unwrap().push(json!({"id": "out2", "type": "output"})); + assert!(problems(v).contains(&(true, None))); + // Loose ends are only incomplete: a draft. + let mut v = base(); + v["nodes"].as_array_mut().unwrap().push(json!({"id": "t", "type": "throttle", "config": {}})); + v["edges"].as_array_mut().unwrap().push(json!({"source": "f", "target": "t"})); + assert_eq!(problems(v), [(false, Some("t".into()))]); + let mut v = base(); + v["edges"] = json!([{"source": "input", "target": "m"}]); + assert_eq!( + problems(v), + [ + (false, Some("m".into())), + (false, Some("f".into())), + (false, Some("out".into())), + (false, None) + ] + ); + } +} diff --git a/crates/ot-source/src/lib.rs b/crates/ot-source/src/lib.rs index 7c17b5e..15154f6 100644 --- a/crates/ot-source/src/lib.rs +++ b/crates/ot-source/src/lib.rs @@ -6,6 +6,7 @@ pub mod codec; pub mod expr; +pub mod flow; pub mod frame; pub mod grpc; pub mod mapping; diff --git a/crates/ot-source/src/pipeline.rs b/crates/ot-source/src/pipeline.rs index c1c8c39..7964718 100644 --- a/crates/ot-source/src/pipeline.rs +++ b/crates/ot-source/src/pipeline.rs @@ -1,10 +1,9 @@ -//! The per-source processing pipeline, free of I/O: +//! The per-source processing pipeline, free of I/O: a frame is decoded into +//! records, and each record follows the source's flow of steps +//! ([`crate::flow`]) to its Output (system tracks) and NATS topics. //! -//! frame → codec → records → reject → mapping → static join → registry → -//! affiliation → filter → tracker → throttle → observation. -//! -//! The tracker stage is optional: it turns a detection feed's plots into -//! source tracks (see [`crate::tracker`]). +//! A Tracker step turns a detection feed's plots into source tracks (see +//! [`crate::tracker`]). //! //! The worker feeds frames in and ships observations out; everything here is //! deterministic given its inputs, so it can be replayed in tests and probes. @@ -17,37 +16,15 @@ use ot_core::{Identifier, Observation}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::codec::{Codec, CodecConfig}; +use crate::codec::Codec; use crate::expr::{Condition, ValueSpec, as_string}; use crate::frame::Frame; use crate::mapping::{self, MapOutcome, MappingSpec, RuleKind}; use crate::registry::{RegistryLookup, RegistryStage}; use crate::trace::{SampleTrace, Trace}; -/// Everything between the transport and the observation stream. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PipelineSpec { - pub codec: CodecConfig, - pub mapping: MappingSpec, - #[serde(default)] - pub static_join: StaticJoinSpec, - #[serde(default, skip_serializing_if = "Option::is_none")] - /// Registry resolution settings. Without them the registry still runs - /// with defaults: identifiers are resolved and graded under - /// `ext.registry`, and the entity's OTH-GOLD minimum populates the track - /// (see [`crate::registry::default_links`]). - pub registry: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub affiliation: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub filter: Option, - /// Form tracks from detections (GNN or MHT) before they leave the source. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tracker: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub throttle: Option, -} +pub use crate::flow::PipelineSpec; +use crate::flow::{INPUT, Step}; #[derive(Debug, Clone, PartialEq, thiserror::Error)] pub enum PipelineError { @@ -57,6 +34,9 @@ pub enum PipelineError { Tracker(String), #[error("{0}")] Codec(String), + /// The flow's shape cannot run (a loop, a step fed the wrong data). + #[error("{0}")] + Flow(String), } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -248,7 +228,9 @@ pub struct Counts { /// Runs of a tracker plugin that failed. pub tracker_errors: u64, pub throttled: u64, + /// Observations sent to the Output, and items published on NATS topics. pub emitted: u64, + pub published: u64, pub grades: HashMap<&'static str, u64>, } @@ -268,6 +250,7 @@ impl Counts { ("tracker_error", self.tracker_errors), ("throttled", self.throttled), ("emitted", self.emitted), + ("published", self.published), ] .into_iter() .filter(|(_, n)| *n > 0) @@ -298,30 +281,78 @@ pub struct Output { pub entity_updates: Vec<(String, Vec<(String, Value)>)>, /// The last decode or validation error seen, for status reporting. pub last_error: Option, - /// Plots given to the tracker stage, when [`Pipeline::keep_plots`] is on. + /// Plots given to a Tracker step, when [`Pipeline::keep_plots`] is on. pub plots: Vec, + /// What reached each NATS topic step: its subject and the item as JSON. + pub published: Vec<(String, Value)>, } pub struct Pipeline { source_id: String, spec: PipelineSpec, codec: Codec, - statics: HashMap, - throttle: HashMap, - /// Source tracks whose static details changed since their last write. - details_changed: HashSet, - /// Extension field types for the mapping's schema version. + /// Each step's state, in `spec.nodes` order. + steps: Vec, + /// Each step's children, by index. + children: Vec>, + /// Whether an Entity links step comes before each step. + linked: Vec, + input: usize, + /// The Map whose static cache is stored under the plain track key. + primary_map: Option, + /// The steps a trace records, input first, in the order they are met. + order: Vec, + /// Extension field types for the mappings' schema version. schema: Option, pub counts: Counts, - /// Used when the spec configures no registry stage. + /// Run before the Output when no Entity links step comes before it. default_registry: RegistryStage, /// The last value sent to each (entity, field), so a feed repeating a /// value writes it once, not until the registry snapshot catches up. entity_sent: HashMap<(String, String), Value>, - tracker: Option, keep_plots: bool, } +/// A step's settings and state. +enum Run { + Input, + Reject(Vec), + Map { + spec: MappingSpec, + ttl_secs: u64, + statics: HashMap, + /// Tracks whose static details changed since their last report. + changed: HashSet, + }, + Filter(FilterSpec), + Registry(RegistryStage), + Affiliation(AffiliationStage), + Tracker(Stage), + Throttle { + spec: ThrottleSpec, + last: HashMap, + /// Tracks whose details changed since they last passed. + pending: HashSet, + }, + Output, + Nats(String), +} + +/// What passes between steps: a record, or an observation's JSON form. +#[derive(Clone)] +struct Item { + value: Value, + /// The mapping rule's kind, for observations. + kind: RuleKind, + /// Its track's static details changed since its last report. + changed: bool, + /// The entity set its affiliation (the country lists then keep out). + entity_affiliation: bool, + received_at: DateTime, + /// The traced sample it belongs to. + sample: Option, +} + /// The tracker stage: OpenTrack's own tracker, or a plugin's. enum Stage { Builtin(Box), @@ -355,24 +386,90 @@ impl Stage { } } +/// Separates a Map's id from a track key in a stored static cache key; the +/// primary Map's keys are plain. +const STATIC_KEY_SEP: char = '\u{1f}'; + impl Pipeline { pub fn new(source_id: impl Into, spec: PipelineSpec) -> Result { - spec.mapping.validate()?; - let tracker = spec - .tracker - .clone() - .map(Stage::new) - .transpose() - .map_err(PipelineError::Tracker)?; + if let Some(p) = spec.shape_problems().into_iter().find(|p| p.invalid) { + return Err(PipelineError::Flow(match p.node { + Some(n) => format!("step {n}: {}", p.message), + None => p.message, + })); + } + let index: HashMap<&str, usize> = spec + .nodes + .iter() + .enumerate() + .map(|(i, n)| (n.id.as_str(), i)) + .collect(); + let mut steps = Vec::with_capacity(spec.nodes.len()); + for n in &spec.nodes { + steps.push(match &n.step { + Step::Input => Run::Input, + Step::Reject(r) => Run::Reject(r.clone()), + Step::Map(m) => { + m.mapping.validate()?; + Run::Map { + spec: m.mapping.clone(), + ttl_secs: m.static_join.ttl_secs, + statics: HashMap::new(), + changed: HashSet::new(), + } + } + Step::Filter(f) => Run::Filter(f.clone()), + Step::Registry(r) => Run::Registry(r.clone()), + Step::Affiliation(a) => Run::Affiliation(a.clone()), + Step::Tracker(t) => Run::Tracker(Stage::new(t.clone()).map_err(PipelineError::Tracker)?), + Step::Throttle(t) => Run::Throttle { + spec: t.clone(), + last: HashMap::new(), + pending: HashSet::new(), + }, + Step::Output => Run::Output, + Step::Nats(n) => Run::Nats(n.subject.clone()), + }); + } + let mut children = vec![Vec::new(); spec.nodes.len()]; + for e in &spec.edges { + if let (Some(&from), Some(&to)) = (index.get(e.source.as_str()), index.get(e.target.as_str())) { + children[from].push(to); + } + } + let input = index[INPUT]; + // Breadth first from the input: the order steps are met in, and + // whether an Entity links step comes before each. + let mut order = Vec::new(); + let mut linked = vec![false; spec.nodes.len()]; + let mut queue = std::collections::VecDeque::from([input]); + let mut seen = HashSet::new(); + while let Some(at) = queue.pop_front() { + if !seen.insert(at) { + continue; + } + order.push(spec.nodes[at].id.clone()); + let here = linked[at] || matches!(spec.nodes[at].step, Step::Registry(_)); + for &c in &children[at] { + linked[c] = here; + queue.push_back(c); + } + } + let primary_map = index + .get("map") + .copied() + .filter(|&i| matches!(steps[i], Run::Map { .. })) + .or_else(|| steps.iter().position(|s| matches!(s, Run::Map { .. }))); Ok(Self { - tracker, source_id: source_id.into(), - codec: Codec::new(spec.codec.clone()) - .map_err(|e| PipelineError::Codec(e.to_string()))?, + codec: Codec::new(spec.codec.clone()).map_err(|e| PipelineError::Codec(e.to_string()))?, spec, - statics: HashMap::new(), - throttle: HashMap::new(), - details_changed: HashSet::new(), + steps, + children, + linked, + input, + primary_map, + order, schema: None, counts: Counts::default(), default_registry: RegistryStage::default(), @@ -381,7 +478,7 @@ impl Pipeline { }) } - /// Also return the plots the tracker stage is given (benchmarks score + /// Also return the plots the Tracker steps are given (benchmarks score /// what the sensor saw as well as what the tracker made of it). pub fn keep_plots(mut self, on: bool) -> Self { self.keep_plots = on; @@ -402,9 +499,47 @@ impl Pipeline { &self.spec } - /// Seed the static cache (e.g. from Redis after a restart). + /// Whether it has a Tracker step (the worker then runs it on a timer). + pub fn has_tracker(&self) -> bool { + self.steps.iter().any(|s| matches!(s, Run::Tracker(_))) + } + + /// The key a Map's static cache entry is stored under. + fn static_key(&self, map: usize, key: &str) -> String { + if Some(map) == self.primary_map { + key.to_owned() + } else { + format!("{}{STATIC_KEY_SEP}{key}", self.spec.nodes[map].id) + } + } + + /// Seed a Map's static cache (e.g. from Redis after a restart), by the + /// key [`Output::statics`] gave it. pub fn load_static(&mut self, key: String, entry: StaticEntry) { - self.statics.insert(key, entry); + let (map, key) = match key.split_once(STATIC_KEY_SEP) { + Some((id, k)) => match self.spec.nodes.iter().position(|n| n.id == id) { + Some(i) => (Some(i), k.to_owned()), + None => return, + }, + None => (self.primary_map, key), + }; + if let Some(Run::Map { statics, .. }) = map.and_then(|i| self.steps.get_mut(i)) { + statics.insert(key, entry); + } + } + + /// The longest any Map keeps its static details. + pub fn static_ttl(&self) -> Duration { + let secs = self + .steps + .iter() + .filter_map(|s| match s { + Run::Map { ttl_secs, .. } => Some(*ttl_secs), + _ => None, + }) + .max() + .unwrap_or(0); + Duration::from_secs(secs) } /// Take and reset the counters. @@ -412,25 +547,29 @@ impl Pipeline { std::mem::take(&mut self.counts) } - /// Forget throttle and static state older than the static TTL. + /// Forget throttle and static state older than they are kept. pub fn prune(&mut self, now: DateTime) { - let ttl = chrono::Duration::from_std(Duration::from_secs(self.spec.static_join.ttl_secs)) - .unwrap_or(chrono::Duration::MAX); - self.statics.retain(|_, e| now - e.updated_at < ttl); - let heartbeat = self - .spec - .throttle - .as_ref() - .map_or(0.0, |t| t.heartbeat_secs); - let keep = chrono::Duration::milliseconds((heartbeat * 1000.0) as i64 * 2); - self.throttle.retain(|_, w| now - w.at < keep); + for s in &mut self.steps { + match s { + Run::Map { ttl_secs, statics, .. } => { + let ttl = chrono::Duration::from_std(Duration::from_secs(*ttl_secs)) + .unwrap_or(chrono::Duration::MAX); + statics.retain(|_, e| now - e.updated_at < ttl); + } + Run::Throttle { spec, last, .. } => { + let keep = chrono::Duration::milliseconds((spec.heartbeat_secs * 1000.0) as i64 * 2); + last.retain(|_, w| now - w.at < keep); + } + _ => {} + } + } } pub fn process(&mut self, frame: &Frame, registry: &dyn RegistryLookup) -> Output { self.run_frame(frame, registry, None) } - /// [`Self::process`], recording in `trace` what each stage made of the + /// [`Self::process`], recording in `trace` what each step made of the /// frame's records while the trace has room (see [`crate::trace`]). pub fn process_traced( &mut self, @@ -441,41 +580,6 @@ impl Pipeline { self.run_frame(frame, registry, Some(trace)) } - /// The stages a trace records, after the transport, in the order they - /// run, with the ids the UI gives them. - fn trace_stages(&self) -> Vec<&'static str> { - let s = &self.spec; - let mut out = vec!["decode"]; - if !s.mapping.reject.is_empty() { - out.push("reject"); - } - out.push("map"); - if self.has_join() { - out.push("join"); - } - out.push("registry"); - for (id, on) in [ - ("affiliation", s.affiliation.is_some()), - ("filter", s.filter.is_some()), - ("tracker", s.tracker.is_some()), - ("throttle", s.throttle.is_some()), - ] { - if on { - out.push(id); - } - } - out - } - - /// Whether static rules feed an identity join. - fn has_join(&self) -> bool { - self.spec - .mapping - .rules - .iter() - .any(|r| r.kind == RuleKind::Static) - } - fn run_frame( &mut self, frame: &Frame, @@ -484,288 +588,357 @@ impl Pipeline { ) -> Output { let mut out = Output::default(); self.counts.frames += 1; - let stages = match trace.as_deref_mut() { - Some(t) => { - t.start_frame(); - self.trace_stages() - } - None => Vec::new(), - }; + if let Some(t) = trace.as_deref_mut() { + t.start_frame(); + } let records = match self.codec.decode(frame) { Ok(r) => r, Err(e) => { self.counts.decode_errors += 1; // A frame that does not decode is a sample of its own. if let Some(t) = trace.as_deref_mut() - && let Some(at) = t.begin(frame, &stages) + && let Some(at) = t.begin(frame, &self.order) { - t.samples[at].dropped("decode", format!("decode error: {e}")); + t.samples[at].dropped(INPUT, format!("decode error: {e}")); } out.last_error = Some(e.to_string()); return out; } }; // A tracker timed by the sensor follows what the codec has learnt. - if let Some(Stage::Builtin(t)) = self.tracker.as_mut() - && let Some(h) = self.codec.hints() - { - t.set_revisit(h.revisit_secs, h.revisit_source); + if let Some(h) = self.codec.hints() { + for s in &mut self.steps { + if let Run::Tracker(Stage::Builtin(t)) = s { + t.set_revisit(h.revisit_secs, h.revisit_source); + } + } } for record in records { - let mut f = match trace.as_deref_mut() { - Some(t) => t.begin(frame, &stages).map(|at| &mut t.samples[at]), - None => None, - }; - if let Some(f) = f.as_deref_mut() { - f.item("decode", || record.clone()); + self.counts.records += 1; + let sample = trace.as_deref_mut().and_then(|t| t.begin(frame, &self.order)); + if let (Some(t), Some(at)) = (trace.as_deref_mut(), sample) { + t.samples[at].item(INPUT, || record.clone()); } - self.process_record(&record, frame.received_at, registry, &mut out, f); + let item = Item { + value: record, + kind: RuleKind::Observation, + changed: false, + entity_affiliation: false, + received_at: frame.received_at, + sample, + }; + self.pass_on(self.input, item, registry, &mut out, &mut trace); } - self.run_tracker(frame.received_at, false, &mut out, trace.as_deref_mut()); + self.run_trackers(frame.received_at, false, registry, &mut out, &mut trace); // Plots still waiting for the rest of their scan. - if let Some(Stage::Builtin(t)) = self.tracker.as_ref() - && let Some(tr) = trace - { - for f in &mut tr.samples { - if !f.plots.is_empty() - && t.waiting().any(|w| f.plots.contains(w)) - && let Some(s) = f.stage("tracker") - { - s.held = true; + if let Some(tr) = trace { + for (i, s) in self.steps.iter().enumerate() { + let Run::Tracker(Stage::Builtin(t)) = s else { continue }; + let id = &self.spec.nodes[i].id; + for f in &mut tr.samples { + let mine: Vec<&Observation> = f.plots.iter().filter(|(n, _)| n == id).map(|(_, o)| o).collect(); + if !mine.is_empty() + && t.waiting().any(|w| mine.contains(&w)) + && let Some(st) = f.stage(id) + { + st.held = true; + } } } } out } - /// The windows a tracker with `auto_timing` is using, once it knows the - /// sensor's revisit period. - pub fn tracker_timing(&self) -> Option<&crate::tracker::Timing> { - match self.tracker.as_ref()? { - Stage::Builtin(t) => t.timing(), - Stage::Plugin { .. } => None, - } - } - - /// Run the tracker on scans still waiting (plots grouped by time wait - /// `scan_hold_secs` for more; `force` runs them regardless). The worker - /// calls this on a timer; a dry run calls it with `force` at the end. - pub fn flush(&mut self, now: DateTime, force: bool) -> Output { - let mut out = Output::default(); - self.run_tracker(now, force, &mut out, None); - out - } - - /// [`Self::flush`], adding the reports to the traced samples whose plots - /// they came from. With `force` (the end of a dry run), a sample whose - /// plots gave no report is marked so. - pub fn flush_traced(&mut self, now: DateTime, force: bool, trace: &mut Trace) -> Output { - let mut out = Output::default(); - self.run_tracker(now, force, &mut out, Some(trace)); - if force { - let why = match self.tracker { - Some(Stage::Plugin { .. }) => { - "a tracker plugin's reports are not traced back to the records behind them" - } - _ => { - "no track report: the tracker has not confirmed a track from these plots (or merged them into a nearby plot, or they came too late for their scan)" - } - }; - for f in &mut trace.samples { - let plots = !f.plots.is_empty(); - if let Some(s) = f.stage("tracker") - && plots - && s.items.is_empty() - && s.dropped.is_empty() - { - s.dropped.push(why.to_owned()); - } + /// Give `item` to each of `at`'s children. + fn pass_on( + &mut self, + at: usize, + item: Item, + registry: &dyn RegistryLookup, + out: &mut Output, + trace: &mut Option<&mut Trace>, + ) { + let n = self.children[at].len(); + for i in 0..n { + let c = self.children[at][i]; + if i + 1 < n { + self.step(c, item.clone(), registry, out, trace); + } else { + self.step(c, item, registry, out, trace); + return; } } - out } - /// Run the tracker stage. When tracing, each report of the built-in - /// tracker is added to the sample whose detection it came from. - fn run_tracker( + /// One step's work on one item. + fn step( &mut self, - now: DateTime, - force: bool, + at: usize, + mut item: Item, + registry: &dyn RegistryLookup, out: &mut Output, - mut trace: Option<&mut Trace>, + trace: &mut Option<&mut Trace>, ) { - // The detection behind each report, only when tracing. - let mut dets: Vec = Vec::new(); - let reports = match self.tracker.as_mut() { - None => return, - Some(Stage::Builtin(t)) => { - let force = force || t.spec().scans == crate::tracker::ScanGrouping::Frame; - let reports = t.run(now, force); - self.counts.late_plots += std::mem::take(&mut t.late); - if trace.is_some() { - dets = reports.iter().map(|(_, d)| d.clone()).collect(); + let id = self.spec.nodes[at].id.clone(); + let mut sample = match (trace.as_deref_mut(), item.sample) { + (Some(t), Some(s)) => t.samples.get_mut(s), + _ => None, + }; + match &mut self.steps[at] { + Run::Input => {} + Run::Reject(rules) => { + if let Some(r) = rules.iter().find(|r| r.when.eval(&item.value)) { + if let Some(f) = sample { + f.dropped(&id, format!("rejected: {}", r.reason)); + } + *self.counts.rejected.entry(r.reason.clone()).or_default() += 1; + return; + } + if let Some(f) = sample { + f.item(&id, || item.value.clone()); } - reports.into_iter().map(|(obs, _)| obs).collect() } - Some(Stage::Plugin { tracker, version }) => match tracker.run(now, force) { - Ok(tracks) => { - let mut tracks: Vec = tracks; - for t in &mut tracks { - t.source_id = self.source_id.clone(); - t.provenance.tracker = Some(version.clone()); + Run::Map { .. } => return self.map(at, &id, item, registry, out, trace), + Run::Filter(filter) => { + if let Some(why) = filter.verdict(&item.value) { + self.counts.filtered += 1; + if let Some(f) = sample { + f.dropped(&id, why); } - tracks + return; } - Err(e) => { - self.counts.tracker_errors += 1; - out.last_error = Some(format!("tracker plugin: {e}")); - Vec::new() + if let Some(f) = sample { + f.item(&id, || item.value.clone()); } - }, - }; - for (i, obs) in reports.into_iter().enumerate() { - let mut f = None; - if let Some(t) = trace.as_deref_mut() { - f = dets - .get(i) - .and_then(|d| t.samples.iter_mut().find(|f| f.plots.contains(d))); - if let Some(f) = f.as_deref_mut() { - f.obs("tracker", &obs); + } + Run::Registry(stage) => { + let stage = stage.clone(); + self.link(&stage, &mut item, registry, out); + if let Some(f) = sample { + f.item(&id, || item.value.clone()); + } + } + Run::Affiliation(stage) => { + if !item.entity_affiliation { + stage.run(&mut item.value); + } + if let Some(f) = sample { + f.item(&id, || item.value.clone()); + if item.entity_affiliation { + f.note(&id, "kept the entity's affiliation"); + } + } + } + Run::Tracker(_) => { + let Some(obs) = self.typed(&id, &item, out, sample.as_deref_mut()) else { return }; + if item.kind == RuleKind::Observation { + self.counts.plots += 1; + if self.keep_plots { + out.plots.push(obs.clone()); + } + if let Some(f) = sample { + f.plots.push((id.clone(), obs.clone())); + } + if let Run::Tracker(t) = &mut self.steps[at] { + t.push(obs, item.received_at); + } + return; + } + // A report with its own identity passes the tracker by. + if let Some(f) = sample { + f.obs(&id, &obs); + } + } + Run::Throttle { .. } => { + let Some(obs) = self.typed(&id, &item, out, sample.as_deref_mut()) else { return }; + let Run::Throttle { spec, last, pending } = &mut self.steps[at] else { unreachable!() }; + let key = obs.source_track_key.clone(); + if obs.state == Some(ot_core::TrackState::Dropped) { + // A source track's end always passes. + last.remove(&key); + pending.remove(&key); + } else { + if item.changed { + pending.insert(key.clone()); + } + let prev = last.get(&key); + if !spec.due(prev, &obs, pending.contains(&key)) { + self.counts.throttled += 1; + if let Some(f) = sample { + f.dropped(&id, spec.why_not(prev, &obs)); + } + return; + } + pending.remove(&key); + last.insert( + key, + LastWrite { + at: obs.observed_at, + lat: obs.position.latitude, + lon: obs.position.longitude, + }, + ); + } + if let Some(f) = sample { + f.obs(&id, &obs); } } - self.emit(obs, out, f); + Run::Output => { + // Tracks reach correlation with their entity looked up. + if !self.linked[at] { + let stage = self.default_registry.clone(); + self.link(&stage, &mut item, registry, out); + } + let Some(obs) = self.typed(&id, &item, out, sample.as_deref_mut()) else { return }; + self.counts.emitted += 1; + if let Some(f) = sample { + f.obs(&id, &obs); + f.emitted.push(obs.clone()); + } + out.observations.push(obs); + return; + } + Run::Nats(subject) => { + self.counts.published += 1; + if let Some(f) = sample { + f.item(&id, || item.value.clone()); + } + out.published.push((subject.clone(), item.value)); + return; + } } + self.pass_on(at, item, registry, out, trace); } - /// Throttle, then ship. A source track's end always ships. - fn emit(&mut self, obs: Observation, out: &mut Output, trace: Option<&mut SampleTrace>) { - if obs.state == Some(ot_core::TrackState::Dropped) { - self.throttle.remove(&obs.source_track_key); - self.details_changed.remove(&obs.source_track_key); - } else if let Some(t) = &self.spec.throttle { - let key = obs.source_track_key.clone(); - let last = self.throttle.get(&key); - if !t.due(last, &obs, self.details_changed.contains(&key)) { - self.counts.throttled += 1; - if let Some(f) = trace { - f.dropped("throttle", t.why_not(last, &obs)); + /// An item as an observation, or why not (counted, and noted on the step). + fn typed( + &mut self, + id: &str, + item: &Item, + out: &mut Output, + sample: Option<&mut SampleTrace>, + ) -> Option { + match serde_json::from_value::(item.value.clone()) { + Ok(o) => Some(o), + Err(e) => { + self.counts.invalid += 1; + if let Some(f) = sample { + f.dropped(id, format!("invalid: {e}")); } - return; + out.last_error = Some(e.to_string()); + None } - self.details_changed.remove(&key); - self.throttle.insert( - key, - LastWrite { - at: obs.observed_at, - lat: obs.position.latitude, - lon: obs.position.longitude, - }, - ); } - self.counts.emitted += 1; - if let Some(f) = trace { - f.obs("throttle", &obs); - f.emitted.push(obs.clone()); + } + + /// Look up the item's entity with these Entity links: grades it, passes + /// fields between entity and track, and queues the entity's updates. + fn link(&mut self, stage: &RegistryStage, item: &mut Item, registry: &dyn RegistryLookup, out: &mut Output) { + let matched = stage.run(&mut item.value, registry); + // Values mapped to the entity go no further than here. + if let Some(ext) = item.value.get_mut("ext").and_then(Value::as_object_mut) { + ext.remove(crate::registry::ENTITY_EXT); } - out.observations.push(obs); + let grade = matched.as_ref().map_or("none", |m| m.grade.as_str()); + // An affiliation the entity set stands: the entity is the authority + // over the country lists. + item.entity_affiliation = matched + .as_ref() + .is_some_and(|m| m.set.iter().any(|f| f == "classification.affiliation")); + if let Some(m) = matched { + let fresh: Vec<(String, Value)> = m + .updates + .into_iter() + .filter(|(field, v)| self.entity_sent.get(&(m.entity_id.clone(), field.clone())) != Some(v)) + .collect(); + if !fresh.is_empty() { + if self.entity_sent.len() > 100_000 { + self.entity_sent.clear(); + } + for (field, v) in &fresh { + self.entity_sent.insert((m.entity_id.clone(), field.clone()), v.clone()); + } + out.entity_updates.push((m.entity_id, fresh)); + } + } + *self.counts.grades.entry(grade).or_default() += 1; } - fn process_record( + /// A Map step: the record's rules, static details kept and joined, each + /// report finalised into an observation. + fn map( &mut self, - record: &Value, - received_at: DateTime, + at: usize, + id: &str, + item: Item, registry: &dyn RegistryLookup, out: &mut Output, - mut trace: Option<&mut SampleTrace>, + trace: &mut Option<&mut Trace>, ) { - self.counts.records += 1; - let mapped = match self.spec.mapping.apply(record) { + let Run::Map { spec, .. } = &self.steps[at] else { return }; + let schema_version = spec.schema_version; + let outcome = spec.apply(&item.value); + let mut sample = match (trace.as_deref_mut(), item.sample) { + (Some(t), Some(s)) => t.samples.get_mut(s), + _ => None, + }; + let mapped = match outcome { MapOutcome::Rejected(reason) => { - if let Some(f) = trace { - f.dropped("reject", format!("rejected: {reason}")); + if let Some(f) = sample { + f.dropped(id, format!("rejected: {reason}")); } *self.counts.rejected.entry(reason).or_default() += 1; return; } MapOutcome::Unmatched => { - if let Some(f) = trace { - f.item("reject", || record.clone()); - f.dropped( - "map", - "no mapping rule matched it (or the rule's key was empty)", - ); + if let Some(f) = sample { + f.dropped(id, "no mapping rule matched it (or the rule's key was empty)"); } self.counts.unmatched += 1; return; } MapOutcome::Mapped(m) => m, }; - // Where a mapped report is finalised into an observation. - let join = self.has_join(); - let made = if join { "join" } else { "map" }; - if let Some(f) = trace.as_deref_mut() { - f.item("reject", || record.clone()); - for m in &mapped { - if m.kind == RuleKind::Static { - f.item("map", || mapped_json(m)); - f.note( - "join", - format!( - "kept as the static identity of {}, to fill its later reports", - m.key - ), - ); - } else if join { - // Before the join fills it in. - let before = mapping::finalize( - &self.source_id, - self.spec.mapping.schema_version, - m, - received_at, - ); - match before { - Ok(o) => f.obs("map", &o), - Err(_) => f.item("map", || mapped_json(m)), - } - } - } - } // Static rules first, so a record carrying both (AIS msg 19) joins // its own identity fields. for m in mapped.iter().filter(|m| m.kind == RuleKind::Static) { self.counts.statics += 1; - let entry = self - .statics - .entry(m.key.clone()) - .or_insert_with(|| StaticEntry { - fields: Value::Object(Default::default()), - identifiers: Vec::new(), - updated_at: received_at, - }); + if let Some(f) = sample.as_deref_mut() { + f.item(id, || mapped_json(m)); + f.note(id, format!("kept the static identity of {}, to fill its later reports", m.key)); + } + let stored = self.static_key(at, &m.key); + let Run::Map { statics, changed, .. } = &mut self.steps[at] else { return }; + let entry = statics.entry(m.key.clone()).or_insert_with(|| StaticEntry { + fields: Value::Object(Default::default()), + identifiers: Vec::new(), + updated_at: item.received_at, + }); let before = entry.fields.clone(); let ids_before = entry.identifiers.len(); merge_over(&mut entry.fields, &m.fields); - for id in &m.identifiers { - if !entry.identifiers.contains(id) { - entry.identifiers.push(id.clone()); + for i in &m.identifiers { + if !entry.identifiers.contains(i) { + entry.identifiers.push(i.clone()); } } - entry.updated_at = received_at; + entry.updated_at = item.received_at; if entry.fields != before || entry.identifiers.len() != ids_before { - self.details_changed.insert(m.key.clone()); + changed.insert(m.key.clone()); } - out.statics.push((m.key.clone(), entry.clone())); + out.statics.push((stored, entry.clone())); } - for m in mapped.into_iter().filter(|m| m.kind.reports()) { - let mut m = m; - if let Some(s) = self.statics.get(&m.key) { + for mut m in mapped.into_iter().filter(|m| m.kind.reports()) { + let Run::Map { statics, changed, .. } = &mut self.steps[at] else { return }; + if let Some(s) = statics.get(&m.key) { mapping::fill_missing(&mut m.fields, &s.fields); - for id in &s.identifiers { - if !m.identifiers.contains(id) { - m.identifiers.push(id.clone()); + for i in &s.identifiers { + if !m.identifiers.contains(i) { + m.identifiers.push(i.clone()); } } } + let details_changed = changed.remove(&m.key); if let Some(schema) = &self.schema { let ext = m .fields @@ -777,138 +950,165 @@ impl Pipeline { && let Err(e) = schema.apply(ext) { self.counts.invalid += 1; - if let Some(f) = trace.as_deref_mut() { - f.dropped(made, format!("invalid: {e}")); + if let Some(f) = sample.as_deref_mut() { + f.dropped(id, format!("invalid: {e}")); } out.last_error = Some(e); continue; } } - // Stages below work on the observation's JSON form. - let mut obs = match mapping::finalize( - &self.source_id, - self.spec.mapping.schema_version, - &m, - received_at, - ) - .and_then(|o| { + // Later steps work on the observation's JSON form. + let obs = match mapping::finalize(&self.source_id, schema_version, &m, item.received_at).and_then(|o| { serde_json::to_value(&o).map_err(|e| mapping::FinalizeError::Invalid(e.to_string())) }) { Ok(v) => v, Err(e) => { self.counts.invalid += 1; - if let Some(f) = trace.as_deref_mut() { - f.dropped(made, format!("invalid: {e}")); + if let Some(f) = sample.as_deref_mut() { + f.dropped(id, format!("invalid: {e}")); } out.last_error = Some(e.to_string()); continue; } }; - if let Some(f) = trace.as_deref_mut() { - f.item(made, || obs.clone()); + if let Some(f) = sample.as_deref_mut() { + f.item(id, || obs.clone()); } - let stage = self - .spec - .registry - .as_ref() - .unwrap_or(&self.default_registry); - let matched = stage.run(&mut obs, registry); - // Values mapped to the entity go no further than here. - if let Some(ext) = obs.get_mut("ext").and_then(Value::as_object_mut) { - ext.remove(crate::registry::ENTITY_EXT); - } - if let Some(f) = trace.as_deref_mut() { - f.item("registry", || obs.clone()); - } - let grade = matched.as_ref().map_or("none", |m| m.grade.as_str()); - // An affiliation the entity set stands: the entity is the authority - // over the country lists. - let entity_affiliation = matched - .as_ref() - .is_some_and(|m| m.set.iter().any(|f| f == "classification.affiliation")); - if let Some(m) = matched { - let fresh: Vec<(String, Value)> = m - .updates - .into_iter() - .filter(|(field, v)| { - self.entity_sent.get(&(m.entity_id.clone(), field.clone())) != Some(v) - }) - .collect(); - if !fresh.is_empty() { - if self.entity_sent.len() > 100_000 { - self.entity_sent.clear(); + let next = Item { + value: obs, + kind: m.kind, + changed: details_changed, + entity_affiliation: false, + received_at: item.received_at, + sample: item.sample, + }; + self.pass_on(at, next, registry, out, trace); + sample = match (trace.as_deref_mut(), item.sample) { + (Some(t), Some(s)) => t.samples.get_mut(s), + _ => None, + }; + } + } + + /// The windows a Tracker step with `auto_timing` is using, once it knows + /// the sensor's revisit period (the first such step's). + pub fn tracker_timing(&self) -> Option<&crate::tracker::Timing> { + self.steps.iter().find_map(|s| match s { + Run::Tracker(Stage::Builtin(t)) => t.timing(), + _ => None, + }) + } + + /// Run the Tracker steps on scans still waiting (plots grouped by time + /// wait `scan_hold_secs` for more; `force` runs them regardless). The + /// worker calls this on a timer; a dry run calls it with `force` at the + /// end. + pub fn flush(&mut self, now: DateTime, force: bool, registry: &dyn RegistryLookup) -> Output { + let mut out = Output::default(); + self.run_trackers(now, force, registry, &mut out, &mut None); + out + } + + /// [`Self::flush`], adding the reports to the traced samples whose plots + /// they came from. With `force` (the end of a dry run), a sample whose + /// plots gave no report is marked so. + pub fn flush_traced( + &mut self, + now: DateTime, + force: bool, + registry: &dyn RegistryLookup, + trace: &mut Trace, + ) -> Output { + let mut out = Output::default(); + let mut t = Some(&mut *trace); + self.run_trackers(now, force, registry, &mut out, &mut t); + if force { + for (i, s) in self.steps.iter().enumerate() { + let Run::Tracker(stage) = s else { continue }; + let why = match stage { + Stage::Plugin { .. } => "a tracker plugin's reports are not traced back to the records behind them", + Stage::Builtin(_) => { + "no track report: the tracker has not confirmed a track from these plots (or merged them into a nearby plot, or they came too late for their scan)" } - for (field, v) in &fresh { - self.entity_sent - .insert((m.entity_id.clone(), field.clone()), v.clone()); + }; + let id = &self.spec.nodes[i].id; + for f in &mut trace.samples { + let plots = f.plots.iter().any(|(n, _)| n == id); + if let Some(st) = f.stage(id) + && plots + && st.items.is_empty() + && st.dropped.is_empty() + { + st.dropped.push(why.to_owned()); } - out.entity_updates.push((m.entity_id, fresh)); } } - *self.counts.grades.entry(grade).or_default() += 1; - if let Some(stage) = &self.spec.affiliation { - if !entity_affiliation { - stage.run(&mut obs); - } - if let Some(f) = trace.as_deref_mut() { - f.item("affiliation", || obs.clone()); - if entity_affiliation { - f.note("affiliation", "kept the entity's affiliation"); + } + out + } + + /// Run each Tracker step, and pass its reports on. When tracing, each + /// report of the built-in tracker joins the sample its detection came + /// from. + fn run_trackers( + &mut self, + now: DateTime, + force: bool, + registry: &dyn RegistryLookup, + out: &mut Output, + trace: &mut Option<&mut Trace>, + ) { + for at in 0..self.steps.len() { + let tracing = trace.is_some(); + // The detection behind each report, only when tracing. + let mut dets: Vec = Vec::new(); + let reports: Vec = match &mut self.steps[at] { + Run::Tracker(Stage::Builtin(t)) => { + let force = force || t.spec().scans == crate::tracker::ScanGrouping::Frame; + let reports = t.run(now, force); + self.counts.late_plots += std::mem::take(&mut t.late); + if tracing { + dets = reports.iter().map(|(_, d)| d.clone()).collect(); } + reports.into_iter().map(|(obs, _)| obs).collect() } - } - if let Some(filter) = &self.spec.filter { - if let Some(why) = filter.verdict(&obs) { - self.counts.filtered += 1; - if let Some(f) = trace.as_deref_mut() { - f.dropped("filter", why); + Run::Tracker(Stage::Plugin { tracker, version }) => match tracker.run(now, force) { + Ok(mut tracks) => { + for t in &mut tracks { + t.source_id = self.source_id.clone(); + t.provenance.tracker = Some(version.clone()); + } + tracks } - continue; - } - if let Some(f) = trace.as_deref_mut() { - f.item("filter", || obs.clone()); - } - } - let obs: Observation = match serde_json::from_value(obs) { - Ok(o) => o, - Err(e) => { - self.counts.invalid += 1; - if let Some(f) = trace.as_deref_mut() { - f.dropped(self.last_json_stage(), format!("invalid: {e}")); + Err(e) => { + self.counts.tracker_errors += 1; + out.last_error = Some(format!("tracker plugin: {e}")); + Vec::new() } - out.last_error = Some(e.to_string()); - continue; - } + }, + _ => continue, }; - if let Some(t) = self.tracker.as_mut() { - if m.kind == RuleKind::Observation { - self.counts.plots += 1; - if self.keep_plots { - out.plots.push(obs.clone()); - } - if let Some(f) = trace.as_deref_mut() { - f.plots.push(obs.clone()); + let id = self.spec.nodes[at].id.clone(); + for (i, obs) in reports.into_iter().enumerate() { + let mut sample = None; + if let Some(t) = trace.as_deref_mut() { + sample = dets + .get(i) + .and_then(|d| t.samples.iter().position(|f| f.plots.iter().any(|(n, p)| n == &id && p == d))); + if let Some(f) = sample.and_then(|s| t.samples.get_mut(s)) { + f.obs(&id, &obs); } - t.push(obs, received_at); - continue; - } - // A report with its own identity passes the tracker by. - if let Some(f) = trace.as_deref_mut() { - f.obs("tracker", &obs); } + let item = Item { + value: serde_json::to_value(&obs).unwrap_or(Value::Null), + kind: RuleKind::Track, + changed: false, + entity_affiliation: false, + received_at: now, + sample, + }; + self.pass_on(at, item, registry, out, trace); } - self.emit(obs, out, trace.as_deref_mut()); - } - } - - /// The last stage working on an observation's JSON form. - fn last_json_stage(&self) -> &'static str { - if self.spec.filter.is_some() { - "filter" - } else if self.spec.affiliation.is_some() { - "affiliation" - } else { - "registry" } } } @@ -950,7 +1150,11 @@ mod tests { type Reg = BTreeMap<(String, String), RegistryEntry>; fn spec() -> PipelineSpec { - serde_json::from_value(json!({ + serde_json::from_value(spec_json()).unwrap() + } + + fn spec_json() -> Value { + json!({ "codec": { "type": "json" }, "mapping": { "rules": [ { "name": "static", "kind": "static", "when": { "path": "t", "eq": "static" }, @@ -969,8 +1173,7 @@ mod tests { { "path": "ext.registry.entity_id", "exists": true }, { "path": "classification.cot_type", "matches": "^a-.-S-C" } ] } }, "throttle": { "min_interval_secs": 30, "heartbeat_secs": 600, "min_move_m": 50 } - })) - .unwrap() + }) } fn reg() -> Reg { @@ -994,6 +1197,98 @@ mod tests { json!({"t": "pos", "id": id, "lat": lat, "lon": -117.0, "ts": ts}) } + /// A flow from steps and connections, after the input. + fn flow(nodes: Value, edges: Value) -> PipelineSpec { + serde_json::from_value(json!({"codec": {"type": "json"}, "nodes": nodes, "edges": edges})).unwrap() + } + + fn pos_map() -> Value { + json!({"mapping": {"rules": [{"name": "pos", "key": "id", + "identifiers": [{"scheme": "mmsi", "value": "id"}], + "fields": {"position.latitude": "lat", "position.longitude": "lon", "name": "name", + "observed_at": {"path": "ts", "transforms": ["time"]}}}]}}) + } + + #[test] + fn a_flow_branches_to_its_output_and_nats_topics() { + // input → map → { filter → filter → output, nats "all" }, map → filter → nats "north" + let spec = flow( + json!([ + {"id": "m", "type": "map", "config": pos_map()}, + {"id": "named", "type": "filter", "config": {"keep_if": {"path": "name", "exists": true}}}, + {"id": "south", "type": "filter", "config": {"keep_if": {"path": "position.latitude", "lt": 33}}}, + {"id": "out", "type": "output"}, + {"id": "all", "type": "nats", "config": {"subject": "feed.all"}}, + {"id": "north", "type": "filter", "config": {"keep_if": {"path": "position.latitude", "gte": 33}}}, + {"id": "pub", "type": "nats", "config": {"subject": "feed.north"}} + ]), + json!([ + {"source": "input", "target": "m"}, + {"source": "m", "target": "named"}, {"source": "named", "target": "south"}, + {"source": "south", "target": "out"}, + {"source": "m", "target": "all"}, + {"source": "m", "target": "north"}, {"source": "north", "target": "pub"} + ]), + ); + let mut p = Pipeline::new("ais", spec).unwrap(); + let reg = reg(); + let rec = |id: u32, lat: f64, name: Option<&str>| { + let mut r = json!({"id": id, "lat": lat, "lon": -117.0, "ts": "2026-09-24T12:00:00Z"}); + if let Some(n) = name { + r["name"] = json!(n); + } + r + }; + let a = feed(&mut p, ®, rec(1, 32.0, Some("ALPHA"))); + assert_eq!(a.observations.len(), 1, "named and south: to correlation"); + assert_eq!(a.published.iter().map(|(s, _)| s.as_str()).collect::>(), ["feed.all"]); + // The Output looks the entity up, with no Entity links step before it. + assert_eq!(a.observations[0].ext["registry"]["entity_id"], "e1"); + assert!(a.published[0].1["ext"].get("registry").is_none(), "a topic gets the mapped data"); + let b = feed(&mut p, ®, rec(2, 34.0, None)); + assert!(b.observations.is_empty(), "unnamed, and north"); + assert_eq!(b.published.iter().map(|(s, _)| s.as_str()).collect::>(), ["feed.all", "feed.north"]); + assert_eq!(b.published[1].1["position"]["latitude"], 34.0); + assert_eq!((p.counts.emitted, p.counts.published, p.counts.filtered), (1, 3, 2)); + } + + #[test] + fn a_trace_follows_each_branch() { + let spec = flow( + json!([ + {"id": "m", "type": "map", "config": pos_map()}, + {"id": "f", "type": "filter", "config": {"drop_if": {"path": "position.latitude", "gt": 33}}}, + {"id": "out", "type": "output"}, + {"id": "raw", "type": "nats", "config": {"subject": "raw"}} + ]), + json!([ + {"source": "input", "target": "m"}, {"source": "m", "target": "f"}, + {"source": "f", "target": "out"}, {"source": "m", "target": "raw"} + ]), + ); + let mut p = Pipeline::new("ais", spec).unwrap(); + let mut trace = Trace::new(5); + let body = json!([{"id": 1, "lat": 32.0, "lon": 1.0, "ts": "2026-09-24T12:00:00Z"}, + {"id": 2, "lat": 40.0, "lon": 1.0, "ts": "2026-09-24T12:00:00Z"}]); + p.process_traced(&Frame::new(body.to_string().into_bytes()), &Reg::new(), &mut trace); + let ids: Vec<&str> = trace.samples[0].stages.iter().map(|s| s.id.as_str()).collect(); + assert_eq!(ids, ["input", "m", "f", "raw", "out"]); + let (a, b) = (&trace.samples[0], &trace.samples[1]); + assert_eq!((stage(a, "out").items.len(), stage(a, "raw").items.len()), (1, 1)); + assert_eq!(stage(b, "f").dropped, ["drop_if matched"]); + assert!(stage(b, "out").items.is_empty()); + assert_eq!(stage(b, "raw").items.len(), 1, "the other branch still has it"); + } + + #[test] + fn a_loop_or_a_step_fed_the_wrong_data_cannot_run() { + let spec = flow( + json!([{"id": "f", "type": "filter", "config": {}}, {"id": "out", "type": "output"}]), + json!([{"source": "input", "target": "f"}, {"source": "f", "target": "out"}]), + ); + assert!(matches!(Pipeline::new("x", spec), Err(PipelineError::Flow(m)) if m.contains("put a Map before it"))); + } + #[test] fn an_affiliation_the_entity_set_outranks_the_country_lists() { let spec: PipelineSpec = serde_json::from_value(json!({ @@ -1231,7 +1526,7 @@ mod tests { assert_eq!(p.counts.plots, 8); assert_eq!(p.counts.emitted, 6); // Nothing waits: frames are scans. - assert!(p.flush(Utc::now(), true).observations.is_empty()); + assert!(p.flush(Utc::now(), true, &Reg::new()).observations.is_empty()); } fn stage<'a>(f: &'a crate::trace::SampleTrace, id: &str) -> &'a crate::trace::StageTrace { @@ -1257,14 +1552,10 @@ mod tests { #[test] fn a_trace_follows_each_record_through_its_stages() { let reg = reg(); - let mut spec = spec(); - spec.throttle = None; - spec.filter = Some(FilterSpec { - keep_if: None, - drop_if: Some( - serde_json::from_value(json!({"path": "position.latitude", "gt": 33})).unwrap(), - ), - }); + let mut v = spec_json(); + v.as_object_mut().unwrap().remove("throttle"); + v["filter"] = json!({"drop_if": {"path": "position.latitude", "gt": 33}}); + let spec: PipelineSpec = serde_json::from_value(v).unwrap(); let mut p = Pipeline::new("ais", spec).unwrap(); let mut trace = Trace::new(5); let frame = |rec: Value| Frame::new(rec.to_string().into_bytes()); @@ -1280,27 +1571,20 @@ mod tests { let f = &trace.samples; assert_eq!((f.len(), trace.frames.len()), (4, 4)); assert_eq!(f.iter().map(|s| s.frame).collect::>(), [0, 1, 2, 3]); - let ids: Vec<&str> = f[0].stages.iter().map(|s| s.id).collect(); + let ids: Vec<&str> = f[0].stages.iter().map(|s| s.id.as_str()).collect(); assert_eq!( ids, - ["decode", "map", "join", "registry", "affiliation", "filter"], - "only the stages the pipeline has" + ["input", "map", "registry", "affiliation", "filter", "output"], + "the flow's steps, in the order they are met" ); assert_eq!(trace.frames[0].format, "json"); - // A static report is kept by the join. - assert_eq!(stage(&f[0], "map").items[0]["kind"], "static"); - let join = stage(&f[0], "join"); - assert!( - join.dropped.is_empty(), - "keeping a static identity is not a drop" - ); - assert!(join.notes[0].starts_with("kept as the static identity of 1")); + // A static report is kept by the map, to join later reports. + let map = stage(&f[0], "map"); + assert_eq!(map.items[0]["kind"], "static"); + assert!(map.dropped.is_empty(), "keeping a static identity is not a drop"); + assert!(map.notes[0].starts_with("kept the static identity of 1")); // A position report: joined, graded, published. - assert!( - stage(&f[1], "map").items[0].get("name").is_none(), - "before the join" - ); - assert_eq!(stage(&f[1], "join").items[0]["name"], "EXAMPLE"); + assert_eq!(stage(&f[1], "map").items[0]["name"], "EXAMPLE"); assert_eq!( stage(&f[1], "registry").items[0]["ext"]["registry"]["grade"], "exact" @@ -1314,7 +1598,7 @@ mod tests { assert!(f[2].emitted.is_empty()); // A frame that does not decode is a sample of its own. assert_eq!(trace.frames[3].format, "text"); - assert!(stage(&f[3], "decode").dropped[0].starts_with("decode error")); + assert!(stage(&f[3], "input").dropped[0].starts_with("decode error")); } #[test] @@ -1334,7 +1618,7 @@ mod tests { assert!(s.iter().all(|s| s.frame == 0)); assert_eq!(trace.frames[0].content["plots"][2]["n"], 3); for (i, sample) in s.iter().enumerate() { - let decoded = &stage(sample, "decode").items; + let decoded = &stage(sample, "input").items; assert_eq!(decoded.len(), 1); assert_eq!(decoded[0]["n"], i + 1, "decode order"); } @@ -1345,7 +1629,7 @@ mod tests { assert_eq!(stage(&s[2], "map").items[0]["source_track_key"], "3"); // First scan: nothing confirmed yet, and nothing waits (frames are scans). assert!(!stage(&s[0], "tracker").held); - p.flush_traced(Utc::now(), true, &mut trace); + p.flush_traced(Utc::now(), true, &Reg::new(), &mut trace); let s = &trace.samples; assert!(stage(&s[0], "tracker").dropped[0].starts_with("no track report")); assert!( @@ -1379,7 +1663,7 @@ mod tests { // Each frame's scan waits for more plots of its time. assert_eq!(trace.samples.len(), 2); assert!(trace.samples.iter().all(|f| stage(f, "tracker").held)); - let out = p.flush_traced(Utc::now(), true, &mut trace); + let out = p.flush_traced(Utc::now(), true, &Reg::new(), &mut trace); assert_eq!(out.observations.len(), 1, "confirmed on the second scan"); let (a, b) = (&trace.samples[0], &trace.samples[1]); assert!(stage(a, "tracker").items.is_empty()); diff --git a/crates/ot-source/src/source.rs b/crates/ot-source/src/source.rs index 751770c..d234ab9 100644 --- a/crates/ot-source/src/source.rs +++ b/crates/ot-source/src/source.rs @@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize}; +use crate::flow::Step; use crate::pipeline::PipelineSpec; use crate::transport::TransportConfig; @@ -185,6 +186,9 @@ pub enum SourceError { Tracker(String), #[error("mapping targets schema version {wanted}, but it is {found}")] SchemaVersion { wanted: u32, found: String }, + /// The flow's shape, or a step's own setting. + #[error("{0}")] + Flow(String), #[error("rule {rule:?} maps ext.{key}, which schema version {version} does not define")] UndeclaredExtension { rule: String, @@ -228,56 +232,56 @@ impl SourceSpec { /// schema checks): see [`Self::problems`] for all of them. pub fn validate(&self) -> Result<(), SourceError> { match self.checks(None).into_iter().next() { - Some((_, _, e)) => Err(e), + Some(c) => Err(c.error), None => Ok(()), } } - /// Everything that stops this source running, each with the pipeline - /// stage (and, where there is one, the setting) it belongs to. Empty: - /// the source is complete and may be enabled. A problem of kind + /// Everything that stops this source running, each on the step of its + /// flow it belongs to (and, where there is one, the setting). Empty: the + /// source is complete and may be enabled. A problem of kind /// [`ProblemKind::Invalid`] also stops it being saved; the rest leave /// it a draft. pub fn problems(&self, schema: Option<&crate::schema::ExtensionSchema>) -> Vec { self.checks(Some(schema)) .into_iter() - .map(|(stage, field, e)| Problem { - kind: match e { - SourceError::BadId(_) | SourceError::NoName => ProblemKind::Invalid, - _ => ProblemKind::Incomplete, + .map(|c| Problem { + kind: if c.invalid { + ProblemKind::Invalid + } else { + ProblemKind::Incomplete }, - stage, - field: field.map(str::to_owned), - message: e.to_string(), + node: c.node, + field: c.field.map(str::to_owned), + message: c.error.to_string(), }) .collect() } - /// Every check, in order, each failure with its stage and setting. + /// Every check, in order, each failure with its step and setting. /// `schema`: `None` skips the extension schema checks; `Some(None)` - /// means the mapping's schema version is not published. - fn checks( - &self, - schema: Option>, - ) -> Vec<(Option, Option<&'static str>, SourceError)> { + /// means the mappings' schema version is not published. + fn checks(&self, schema: Option>) -> Vec { let mut out = Vec::new(); + let input = || Some(crate::flow::INPUT.to_owned()); + let push = |out: &mut Vec, node: Option, field: Option<&'static str>, error: SourceError| { + let invalid = matches!(error, SourceError::BadId(_) | SourceError::NoName); + out.push(Check { + node, + field, + error, + invalid, + }); + }; if let Some(label) = &self.security && let Err(e) = label.validate() { - out.push(( - Some(Stage::Output), - Some("security"), - SourceError::Tracker(e), - )); + push(&mut out, None, Some("security"), SourceError::Tracker(e)); } if let Some(m) = &self.emitter_motion && let Err(e) = m.check() { - out.push(( - Some(Stage::Output), - Some("emitter_motion"), - SourceError::Tracker(e), - )); + push(&mut out, None, Some("emitter_motion"), SourceError::Tracker(e)); } let id_ok = !self.id.is_empty() && self.id.len() <= 64 @@ -286,115 +290,130 @@ impl SourceSpec { .bytes() .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-' || b == b'_'); if !id_ok { - out.push((None, Some("id"), SourceError::BadId(self.id.clone()))); + push(&mut out, None, Some("id"), SourceError::BadId(self.id.clone())); } if self.name.trim().is_empty() { - out.push((None, Some("name"), SourceError::NoName)); + push(&mut out, None, Some("name"), SourceError::NoName); } - for e in self.pipeline.mapping.problems() { - out.push((Some(Stage::Map), None, e.into())); + // The flow's shape, then each step's own settings. + for p in self.pipeline.shape_problems() { + out.push(Check { + node: p.node, + field: None, + error: SourceError::Flow(p.message), + invalid: p.invalid, + }); } - if let Some(t) = &self.pipeline.tracker - && let Err(e) = t.validate() - { - out.push((Some(Stage::Tracker), None, SourceError::Tracker(e))); + for n in &self.pipeline.nodes { + let node = || Some(n.id.clone()); + match &n.step { + Step::Map(m) => { + for e in m.mapping.problems() { + push(&mut out, node(), None, e.into()); + } + } + Step::Tracker(t) => { + if let Err(e) = t.validate() { + push(&mut out, node(), None, SourceError::Tracker(e)); + } + } + Step::Nats(nats) => { + if let Err(e) = check_subject(&nats.subject) { + push(&mut out, node(), Some("subject"), SourceError::Flow(e)); + } + } + _ => {} + } } let grpc = matches!( self.transport, TransportConfig::GrpcClient { .. } | TransportConfig::GrpcServer { .. } ); - let protobuf = matches!( - self.pipeline.codec, - crate::codec::CodecConfig::Protobuf { .. } - ); - let plugin = matches!( - self.pipeline.codec, - crate::codec::CodecConfig::Plugin { .. } - ); + let protobuf = matches!(self.pipeline.codec, crate::codec::CodecConfig::Protobuf { .. }); + let plugin = matches!(self.pipeline.codec, crate::codec::CodecConfig::Plugin { .. }); // A gRPC source's protobuf is compiled (once) by the check below. if (plugin || (protobuf && !grpc)) && let Err(e) = crate::codec::Codec::new(self.pipeline.codec.clone()) { - out.push(( - Some(Stage::Decode), - None, - SourceError::Tracker(e.to_string()), - )); + push(&mut out, input(), Some("codec"), SourceError::Tracker(e.to_string())); } let missing = self.transport.missing(); for (field, what) in &missing { - out.push(( - Some(Stage::Source), - Some(*field), - SourceError::Transport(format!("{what} is not set")), - )); + push(&mut out, input(), Some(*field), SourceError::Transport(format!("{what} is not set"))); } if let Err(e) = self.transport.check() { - out.push((Some(Stage::Source), None, SourceError::Transport(e))); + push(&mut out, input(), None, SourceError::Transport(e)); } // gRPC carries protobuf: the method must fit the codec's message. if grpc { match crate::transport::ProtoContext::of(&self.pipeline.codec) { - Err(e) => out.push((Some(Stage::Decode), None, SourceError::Transport(e))), - Ok(None) => out.push(( - Some(Stage::Decode), - None, + Err(e) => push(&mut out, input(), Some("codec"), SourceError::Transport(e)), + Ok(None) => push( + &mut out, + input(), + Some("codec"), SourceError::Transport("a gRPC source decodes with the protobuf codec".into()), - )), + ), Ok(Some(_)) if !missing.is_empty() => {} Ok(Some(proto)) => { let fit = match &self.transport { - TransportConfig::GrpcClient { - method, request, .. - } => proto.client_request(method, request).map(drop), - TransportConfig::GrpcServer { methods, .. } => { - proto.server_methods(methods).map(drop) + TransportConfig::GrpcClient { method, request, .. } => { + proto.client_request(method, request).map(drop) } + TransportConfig::GrpcServer { methods, .. } => proto.server_methods(methods).map(drop), _ => Ok(()), }; if let Err(e) = fit { - out.push((Some(Stage::Source), None, SourceError::Transport(e))); + push(&mut out, input(), None, SourceError::Transport(e)); } } } } - if let TransportConfig::TcpClient { framing, .. } - | TransportConfig::TcpServer { framing, .. } = &self.transport + if let TransportConfig::TcpClient { framing, .. } | TransportConfig::TcpServer { framing, .. } = + &self.transport && let Err(e) = crate::frame::Framer::new(framing.clone()) { - out.push((Some(Stage::Source), Some("framing"), e.into())); + push(&mut out, input(), Some("framing"), e.into()); } // Last, so that a broken setting is reported before its security. if let Some(e) = self.authentication_problem() { - out.push(( - Some(Stage::Source), - Some("unauthenticated"), - SourceError::Unauthenticated(e), - )); + push(&mut out, input(), Some("unauthenticated"), SourceError::Unauthenticated(e)); } if let Some(schema) = schema { - let wanted = self.pipeline.mapping.schema_version; + let wanted = self.pipeline.schema_version(); + let maps: Vec<&crate::flow::FlowNode> = self + .pipeline + .nodes + .iter() + .filter(|n| matches!(n.step, Step::Map(_))) + .collect(); match schema.filter(|s| s.version == wanted) { - None => out.push(( - Some(Stage::Map), + None if !maps.is_empty() => push( + &mut out, + Some(maps[0].id.clone()), Some("schema_version"), SourceError::SchemaVersion { wanted, found: "not published".into(), }, - )), + ), + None => {} Some(schema) => { - for (key, rules) in crate::schema::mapped_ext_keys(&self.pipeline.mapping) { - if schema.field(&key).is_none() { - out.push(( - Some(Stage::Map), - None, - SourceError::UndeclaredExtension { - rule: rules[0].clone(), - key, - version: wanted, - }, - )); + for n in maps { + let Step::Map(m) = &n.step else { continue }; + for (key, rules) in crate::schema::mapped_ext_keys(&m.mapping) { + if schema.field(&key).is_none() { + push( + &mut out, + Some(n.id.clone()), + None, + SourceError::UndeclaredExtension { + rule: rules[0].clone(), + key, + version: wanted, + }, + ); + } } } } @@ -404,13 +423,41 @@ impl SourceSpec { } } +/// One failed check: the step and setting it concerns, and whether it also +/// stops a save. +struct Check { + node: Option, + field: Option<&'static str>, + error: SourceError, + invalid: bool, +} + +/// A NATS subject a NATS topic step may publish on: concrete (no wildcards), +/// dot-separated tokens without spaces. (The server also keeps it outside +/// the system tracks' subjects.) +pub fn check_subject(subject: &str) -> Result<(), String> { + if subject.trim().is_empty() { + return Err("the subject is not set".into()); + } + let ok = subject + .split('.') + .all(|t| !t.is_empty() && t != "*" && t != ">" && !t.chars().any(char::is_whitespace)); + if ok { + Ok(()) + } else { + Err(format!( + "{subject:?} is not a NATS subject to publish on (dot-separated tokens, no wildcards or spaces)" + )) + } +} + /// One reason a source may not run (see [`SourceSpec::problems`]). #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Problem { pub kind: ProblemKind, - /// The pipeline stage it belongs to; `None` for the source as a whole - /// (its id and name). - pub stage: Option, + /// The step of the flow it belongs to (`input` for the transport and + /// codec); `None` for the source as a whole. + pub node: Option, /// The setting, where there is one (for example `host`). #[serde(default, skip_serializing_if = "Option::is_none")] pub field: Option, @@ -426,26 +473,6 @@ pub enum ProblemKind { Incomplete, } -/// A source's pipeline stages, in the order a frame meets them. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum Stage { - /// The transport and its framing. - Source, - /// The codec. - Decode, - Map, - StaticJoin, - Registry, - Affiliation, - Filter, - Tracker, - Throttle, - /// What the source reports and how: tracks or detections, security - /// label, priority. - Output, -} - #[cfg(test)] mod tests { use super::*; @@ -505,13 +532,13 @@ mod tests { } #[test] - fn problems_name_their_stage_and_a_draft_is_only_incomplete() { + fn problems_name_their_step_and_a_draft_is_only_incomplete() { let ok: SourceSpec = serde_json::from_value(spec("cot")).unwrap(); assert_eq!( ok.problems(None), vec![Problem { kind: ProblemKind::Incomplete, - stage: Some(Stage::Map), + node: Some("map".into()), field: Some("schema_version".into()), message: "mapping targets schema version 1, but it is not published".into(), }] @@ -527,16 +554,16 @@ mod tests { let found: Vec<_> = draft .problems(None) .into_iter() - .map(|p| (p.kind, p.stage, p.field)) + .map(|p| (p.kind, p.node, p.field)) .collect(); let incomplete = ProblemKind::Incomplete; assert_eq!( found, vec![ - (incomplete, Some(Stage::Map), None), - (incomplete, Some(Stage::Source), Some("host".into())), - (incomplete, Some(Stage::Source), Some("port".into())), - (incomplete, Some(Stage::Map), Some("schema_version".into())), + (incomplete, Some("map".into()), None), + (incomplete, Some("input".into()), Some("host".into())), + (incomplete, Some("input".into()), Some("port".into())), + (incomplete, Some("map".into()), Some("schema_version".into())), ] ); assert!(matches!(draft.validate(), Err(SourceError::Mapping(_)))); @@ -552,7 +579,7 @@ mod tests { let found: Vec<_> = s .problems(None) .into_iter() - .map(|p| (p.kind, p.stage, p.message)) + .map(|p| (p.kind, p.node, p.message)) .collect(); assert_eq!( found[0], @@ -563,7 +590,7 @@ mod tests { ) ); assert_eq!( - found.iter().filter(|p| p.1 == Some(Stage::Map)).count(), + found.iter().filter(|p| p.1.as_deref() == Some("map")).count(), 5, "unknown field, no position (twice), duplicate rule, schema: {found:?}" ); diff --git a/crates/ot-source/src/trace.rs b/crates/ot-source/src/trace.rs index 3ca7976..bb1fcad 100644 --- a/crates/ot-source/src/trace.rs +++ b/crates/ot-source/src/trace.rs @@ -1,5 +1,5 @@ -//! A stage-by-stage record of what a pipeline did with its first few -//! records, for the pipeline designer's live preview. +//! A step-by-step record of what a source's flow did with its first few +//! records, for the editor's samples. //! //! Tracing is opt-in: [`crate::pipeline::Pipeline::process_traced`] fills a //! [`Trace`]; [`crate::pipeline::Pipeline::process`] takes no trace and @@ -40,14 +40,14 @@ pub struct Trace { pub struct SampleTrace { /// Index in [`Trace::frames`] of the frame it came from. pub frame: usize, - /// The pipeline's stages after the transport, in order, only those it has. + /// The flow's steps, input first, in the order they are met. pub stages: Vec, /// What the pipeline emitted from this record, to show as published. #[serde(skip)] pub emitted: Vec, - /// Detections this record gave the tracker stage. + /// Detections this record gave each Tracker step, by step id. #[serde(skip)] - pub(crate) plots: Vec, + pub(crate) plots: Vec<(String, Observation)>, } /// A raw frame, readable: JSON when it parses, else text, else escaped bytes. @@ -69,8 +69,8 @@ pub struct FrameView { /// A record after one stage. #[derive(Debug, Serialize)] pub struct StageTrace { - /// The stage id, as the UI names it (`decode`, `map`, `filter`…). - pub id: &'static str, + /// The step's id (`input` holds the decoded record). + pub id: String, /// What this stage passed on: the record up to the reject stage, the /// mapping's outputs, then observations. pub items: Vec, @@ -103,7 +103,7 @@ impl Trace { /// Start tracing a record of `frame` (or the frame itself, when it does /// not decode), if the trace still has room. - pub(crate) fn begin(&mut self, frame: &Frame, stages: &[&'static str]) -> Option { + pub(crate) fn begin(&mut self, frame: &Frame, stages: &[String]) -> Option { if self.samples.len() >= self.limit { return None; } @@ -121,7 +121,7 @@ impl Trace { stages: stages .iter() .map(|id| StageTrace { - id, + id: id.clone(), items: Vec::new(), dropped: Vec::new(), notes: Vec::new(), diff --git a/crates/ot-source/tests/sapient_acoustic_example.rs b/crates/ot-source/tests/sapient_acoustic_example.rs index 6788afa..17532b3 100644 --- a/crates/ot-source/tests/sapient_acoustic_example.rs +++ b/crates/ot-source/tests/sapient_acoustic_example.rs @@ -28,7 +28,7 @@ fn acoustic_example_maps_a_sapient_bearing() { } } }); - let MapOutcome::Mapped(mapped) = source.pipeline.mapping.apply(&record) else { + let MapOutcome::Mapped(mapped) = source.pipeline.mappings().next().unwrap().apply(&record) else { panic!("acoustic record did not match the source mapping"); }; let observation = finalize(&source.id, 1, &mapped[0], Utc::now()).expect("valid bearing"); diff --git a/crates/ot-source/tests/stanag4607.rs b/crates/ot-source/tests/stanag4607.rs index 52fd535..4c17cb4 100644 --- a/crates/ot-source/tests/stanag4607.rs +++ b/crates/ot-source/tests/stanag4607.rs @@ -92,7 +92,7 @@ fn replay(path: &std::path::Path) -> (Vec, Pipeline, usize let last = out.last().map(|o| o.observed_at).unwrap(); out.extend( pipeline - .flush(last + chrono::TimeDelta::hours(1), true) + .flush(last + chrono::TimeDelta::hours(1), true, ®istry) .observations, ); (out, pipeline, packets, dwells) @@ -223,7 +223,7 @@ fn a_slow_revisit_recording_tunes_the_tracker_itself() { fn platform_types_have_valid_symbols() { let spec: SourceSpec = serde_json::from_str(include_str!("../../../docs/examples/stanag4607.json")).unwrap(); - let json = serde_json::to_value(&spec.pipeline.mapping).unwrap(); + let json = serde_json::to_value(spec.pipeline.mappings().next().unwrap()).unwrap(); let rule = json["rules"] .as_array() .unwrap() @@ -276,7 +276,7 @@ fn gmti_plots_carry_ground_error_ellipses() { continue; } located += usize::from(r.get("lat").is_some()); - let ot_source::mapping::MapOutcome::Mapped(mapped) = spec.pipeline.mapping.apply(r) else { + let ot_source::mapping::MapOutcome::Mapped(mapped) = spec.pipeline.mappings().next().unwrap().apply(r) else { continue; }; for m in mapped.iter().filter(|m| m.rule == "target") { From f408c429f40c6253119840e37fb304d69f0cc06c Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Wed, 21 Oct 2026 17:22:15 -0500 Subject: [PATCH 12/16] Sources run their flows; NATS topic steps replace raw output (#104) - The worker runs a source's flow: NATS topic steps publish what reaches them (with the source's security label) on their subjects; a Tracker step anywhere is run on its timer; static caches are kept per Map. - Problems name their step (node) instead of a fixed stage; the dry run says on which step a flow cannot run. - Raw output and its consent are gone: a source saved with a raw-output subject gets a NATS topic step beside its Output, once, on start, and a configuration file's raw output is read the same way. The raw-output endpoints are removed. A NATS topic's subject must lie outside the system tracks' subjects. - The engine reads a source's untracked detections and entity links from its Output's path. Co-Authored-By: Claude Opus 5.5 --- crates/ot-server/src/api.rs | 199 +++++++---------- crates/ot-server/src/config.rs | 9 +- crates/ot-server/src/config_backup.rs | 22 +- crates/ot-server/src/engine.rs | 13 +- crates/ot-server/src/engine/bench.rs | 7 +- crates/ot-server/src/sources.rs | 143 ++++++++---- crates/ot-source/src/flow.rs | 139 +++++++++--- crates/ot-source/src/pipeline.rs | 203 ++++++++++++++---- crates/ot-source/src/source.rs | 73 +++++-- .../tests/sapient_acoustic_example.rs | 3 +- crates/ot-source/tests/stanag4607.rs | 4 +- 11 files changed, 562 insertions(+), 253 deletions(-) diff --git a/crates/ot-server/src/api.rs b/crates/ot-server/src/api.rs index 6c24536..6f17998 100644 --- a/crates/ot-server/src/api.rs +++ b/crates/ot-server/src/api.rs @@ -10,10 +10,11 @@ use axum::extract::{Path, Query, State}; use axum::http::{HeaderMap, StatusCode}; use axum::routing::{get, post, put}; use axum::{Extension, Json, Router}; +use ot_source::flow::{INPUT, Step}; use ot_source::frame::Frame; use ot_source::pipeline::Pipeline; use ot_source::schema::{ExtensionField, ExtensionSchema}; -use ot_source::source::{Problem, ProblemKind, SourceSpec, Stage}; +use ot_source::source::{Problem, ProblemKind, SourceSpec}; use ot_source::trace::Trace; use ot_store::{Entity, SourceRow, SourceWrite}; use serde::Deserialize; @@ -33,10 +34,6 @@ pub fn routes() -> Router { ) .route("/sources/{id}/enable", post(enable_source)) .route("/sources/{id}/disable", post(disable_source)) - .route( - "/sources/{id}/raw-output", - put(set_raw_output).delete(clear_raw_output), - ) .route("/sources/{id}/revisions", get(source_revisions)) .route("/sources/{id}/metrics", get(source_metrics)) .route("/registry", get(registry_resolve)) @@ -102,29 +99,40 @@ impl Checked { } /// Its pipeline, to dry-run over samples: a draft runs too, as far as - /// its decode, map and tracker stages allow (its transport is not used, + /// its shape, maps, trackers and codec allow (its transport is not used, /// and without a published schema its extension fields are not checked). fn dry_run(&self) -> Result { self.clone().savable()?; let pipeline = Pipeline::new(self.spec.id.clone(), self.spec.pipeline.clone()).map_err(|e| { - let mut blocking: Vec = self - .problems - .iter() - .filter(|p| { - matches!(p.stage, Some(Stage::Decode | Stage::Map | Stage::Tracker)) - }) - .cloned() - .collect(); - // What the checks did not see (a tracker plugin not loaded, say), on its stage. + use ot_source::pipeline::PipelineError as E; + let flow = &self.spec.pipeline; + let blocks = |p: &&Problem| { + p.kind == ProblemKind::Invalid + || (p.node.as_deref() == Some(INPUT) && p.field.as_deref() == Some("codec")) + || p.node + .as_deref() + .and_then(|n| flow.node(n)) + .is_some_and(|n| matches!(n.step, Step::Map(_) | Step::Tracker(_))) + }; + let mut blocking: Vec = + self.problems.iter().filter(blocks).cloned().collect(); + // What the checks did not see (a tracker plugin not loaded, say), on its step. if blocking.is_empty() { + let first = |want: fn(&Step) -> bool| { + flow.nodes + .iter() + .find(|n| want(&n.step)) + .map(|n| n.id.clone()) + }; blocking.push(Problem { kind: ProblemKind::Incomplete, - stage: Some(match e { - ot_source::pipeline::PipelineError::Mapping(_) => Stage::Map, - ot_source::pipeline::PipelineError::Tracker(_) => Stage::Tracker, - ot_source::pipeline::PipelineError::Codec(_) => Stage::Decode, - }), + node: match e { + E::Mapping(_) => first(|s| matches!(s, Step::Map(_))), + E::Tracker(_) => first(|s| matches!(s, Step::Tracker(_))), + E::Codec(_) => Some(INPUT.to_owned()), + E::Flow(_) => None, + }, field: None, message: e.to_string(), }); @@ -149,7 +157,7 @@ impl Checked { fn invalid(message: String) -> ApiError { ApiError::source_problems(&[Problem { kind: ProblemKind::Invalid, - stage: None, + node: None, field: None, message, }]) @@ -161,13 +169,27 @@ async fn check_spec(s: &AppState, body: Value) -> Result { crate::probe::file_allowed(&s.common, &spec.transport).map_err(|e| { ApiError::source_problems(&[Problem { kind: ProblemKind::Invalid, - stage: Some(Stage::Source), + node: Some(INPUT.to_owned()), field: Some("path".into()), message: e, }]) })?; - let schema = published_schema(s, spec.pipeline.mapping.schema_version).await?; - let problems = spec.problems(schema.as_ref()); + let schema = published_schema(s, spec.pipeline.schema_version()).await?; + let mut problems = spec.problems(schema.as_ref()); + // A NATS topic publishes outside the system tracks' subjects. + for n in &spec.pipeline.nodes { + if let Step::Nats(t) = &n.step + && ot_source::source::check_subject(&t.subject).is_ok() + && let Err(e) = check_raw_subject(&t.subject, &s.common.nats.tracks_subject) + { + problems.push(Problem { + kind: ProblemKind::Incomplete, + node: Some(n.id.clone()), + field: Some("subject".into()), + message: e, + }); + } + } let normalised = serde_json::to_value(&spec).map_err(|e| ApiError::internal(e.to_string()))?; Ok(Checked { spec, @@ -389,16 +411,6 @@ async fn disable_source( toggle(s, id, false, None, headers).await } -#[derive(Deserialize)] -struct RawOutput { - /// NATS subject for this source's raw tracks; `opentrack.raw.` - /// when omitted. - #[serde(default)] - subject: Option, - /// Must be true: the admin confirms this source's raw data is published. - consent: bool, -} - /// A concrete (wildcard-free) NATS subject outside the system tracks prefix. pub(crate) fn check_raw_subject(subject: &str, tracks_prefix: &str) -> Result<(), String> { let valid = !subject.is_empty() @@ -412,47 +424,12 @@ pub(crate) fn check_raw_subject(subject: &str, tracks_prefix: &str) -> Result<() } if subject == tracks_prefix || subject.starts_with(&format!("{tracks_prefix}.")) { return Err(format!( - "raw output needs its own subject, outside the system tracks subjects ({tracks_prefix}.>)" + "a NATS topic needs its own subject, outside the system tracks subjects ({tracks_prefix}.>)" )); } Ok(()) } -async fn set_raw_output( - State(s): State, - Path(id): Path, - headers: HeaderMap, - Json(body): Json, -) -> Result, ApiError> { - if !body.consent { - return Err(ApiError::unprocessable( - "raw output publishes this source's tracks as received; set consent to true to confirm", - )); - } - let c = match body.subject.as_deref().map(str::trim) { - Some(sub) if !sub.is_empty() => sub.to_owned(), - _ => format!("opentrack.raw.{id}"), - }; - check_raw_subject(&c, &s.common.nats.tracks_subject).map_err(ApiError::unprocessable)?; - let actor = actor(&headers); - let row = s - .with_db(move |db| db.set_raw_output(&id, Some(&c), &actor)) - .await?; - Ok(Json(with_status(&s, row).await)) -} - -async fn clear_raw_output( - State(s): State, - Path(id): Path, - headers: HeaderMap, -) -> Result, ApiError> { - let actor = actor(&headers); - let row = s - .with_db(move |db| db.set_raw_output(&id, None, &actor)) - .await?; - Ok(Json(with_status(&s, row).await)) -} - async fn source_revisions( State(s): State, u: Option>, @@ -580,8 +557,8 @@ async fn validate_source( .last() .map_or_else(chrono::Utc::now, |f| f.received_at); let flushed = match trace.as_mut() { - Some(t) => pipeline.flush_traced(last, true, t), - None => pipeline.flush(last, true), + Some(t) => pipeline.flush_traced(last, true, ®istry, t), + None => pipeline.flush(last, true, ®istry), }; observations.extend(flushed.observations); let counts: BTreeMap = pipeline.take_counts().pairs().into_iter().collect(); @@ -1044,8 +1021,8 @@ mod tests { assert_eq!(st, StatusCode::CREATED, "{body}"); let problem = &body["problems"][0]; assert_eq!( - (&problem["stage"], &problem["field"]), - (&json!("source"), &json!("unauthenticated")), + (&problem["node"], &problem["field"]), + (&json!("input"), &json!("unauthenticated")), "{body}" ); assert!( @@ -1106,7 +1083,7 @@ mod tests { assert_eq!(body["transport"], "http_poll"); assert_eq!(body["draft"], true, "{body}"); assert_eq!(body["problems"][0]["kind"], "incomplete", "{body}"); - assert_eq!(body["problems"][0]["stage"], "map", "{body}"); + assert_eq!(body["problems"][0]["node"], "map", "{body}"); assert!( body["problems"][0]["message"] .as_str() @@ -1120,7 +1097,7 @@ mod tests { body["error"].as_str().unwrap().contains("schema version 2"), "{body}" ); - assert_eq!(body["problems"][0]["stage"], "map", "{body}"); + assert_eq!(body["problems"][0]["node"], "map", "{body}"); // Once the schema is published, it may run as it is. publish_example_schema(&app).await; @@ -1165,7 +1142,7 @@ mod tests { .as_array() .unwrap() .iter() - .inspect(|p| assert_eq!(p["stage"], "map", "{body}")) + .inspect(|p| assert_eq!(p["node"], "map", "{body}")) .map(|p| p["message"].as_str().unwrap()) .collect(); assert!( @@ -1195,44 +1172,28 @@ mod tests { assert_eq!(body["sources"][0]["name"], "adsb.lol (edited)"); assert_eq!(body["sources"][0]["draft"], false); - // Raw output needs explicit consent and a subject outside `tracks.>`. - for bad in [ - json!({"consent": false}), - json!({"subject": "tracks.raw", "consent": true}), - json!({"subject": "tracks", "consent": true}), - json!({"subject": "raw.*", "consent": true}), - json!({"subject": "raw..adsb", "consent": true}), - ] { - let (st, _) = call( - &app, - "PUT", - "/api/v1/sources/adsb-lol/raw-output", - Some(bad.clone()), - ) - .await; - assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "{bad}"); - } - let (st, body) = call( - &app, - "PUT", - "/api/v1/sources/adsb-lol/raw-output", - Some(json!({"consent": true})), - ) - .await; - assert_eq!( - (st, body["raw_subject"].as_str()), - (StatusCode::OK, Some("opentrack.raw.adsb-lol")) - ); - let (st, body) = call( - &app, - "PUT", - "/api/v1/sources/adsb-lol/raw-output", - Some(json!({"subject": "raw.adsb", "consent": true})), - ) - .await; + // A NATS topic publishes outside the system tracks' subjects. + let mut topic = spec.clone(); + topic["pipeline"] = json!({ + "codec": spec["pipeline"]["codec"], + "nodes": [ + {"id": "map", "type": "map", "config": {"mapping": spec["pipeline"]["mapping"]}}, + {"id": "out", "type": "output"}, + {"id": "raw", "type": "nats", "config": {"subject": "tracks.raw"}} + ], + "edges": [{"source": "input", "target": "map"}, {"source": "map", "target": "out"}, + {"source": "map", "target": "raw"}] + }); + topic["name"] = json!("adsb.lol (edited)"); + let (st, body) = call(&app, "PUT", "/api/v1/sources/adsb-lol", Some(topic.clone())).await; + assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "enabled: {body}"); + assert_eq!(body["problems"][0]["node"], "raw", "{body}"); + topic["pipeline"]["nodes"][2]["config"]["subject"] = json!("raw.adsb"); + let (st, body) = call(&app, "PUT", "/api/v1/sources/adsb-lol", Some(topic)).await; + assert_eq!(st, StatusCode::OK, "{body}"); assert_eq!( - (st, body["raw_subject"].as_str()), - (StatusCode::OK, Some("raw.adsb")) + body["spec"]["pipeline"]["nodes"][3]["config"]["subject"], "raw.adsb", + "{body}" ); // Status reports NATS as down without blocking the API: 503, with @@ -1379,7 +1340,7 @@ mod tests { ) .await; assert_eq!(st, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); - assert_eq!(body["problems"][0]["stage"], "map", "{body}"); + assert_eq!(body["problems"][0]["node"], "map", "{body}"); // The others follow, so each step shows what it needs. assert!( body["problems"] @@ -1455,13 +1416,17 @@ mod tests { .iter() .map(|s| s["id"].as_str().unwrap()) .collect(); - assert_eq!(ids, ["decode", "map", "registry", "filter", "publish"]); - let publish = &t[0]["stages"][4]["items"][0]; + assert_eq!( + ids, + ["input", "map", "registry", "filter", "output", "publish"] + ); + let publish = &t[0]["stages"][5]["items"][0]; assert_eq!(publish["name"], "ALPHA", "the published message"); assert_eq!(publish["op"], "upsert"); assert_eq!(publish["track_id"], "assigned by correlation"); assert_eq!(t[1]["stages"][3]["dropped"], json!(["drop_if matched"])); assert_eq!(t[1]["stages"][4]["items"], json!([])); + assert_eq!(t[1]["stages"][5]["items"], json!([])); assert_eq!( frames[2], json!({"format": "text", "bytes": 8, "content": "not json"}) diff --git a/crates/ot-server/src/config.rs b/crates/ot-server/src/config.rs index df46ae6..a557d46 100644 --- a/crates/ot-server/src/config.rs +++ b/crates/ot-server/src/config.rs @@ -219,7 +219,14 @@ impl Common { if let Some(dir) = self.sqlite.parent().filter(|d| !d.as_os_str().is_empty()) { std::fs::create_dir_all(dir)?; } - Ok(ot_store::Db::open(&self.sqlite)?) + let mut db = ot_store::Db::open(&self.sqlite)?; + // Once a process: sources from before flows move their raw output + // to a NATS topic step. + static MIGRATED: std::sync::Once = std::sync::Once::new(); + let mut result = Ok(()); + MIGRATED.call_once(|| result = crate::sources::migrate_raw_outputs(&mut db)); + result?; + Ok(db) } /// Keep one connection open for every later `open_db` of this (and diff --git a/crates/ot-server/src/config_backup.rs b/crates/ot-server/src/config_backup.rs index 0d7efd0..e3ebbff 100644 --- a/crates/ot-server/src/config_backup.rs +++ b/crates/ot-server/src/config_backup.rs @@ -281,7 +281,7 @@ fn check(common: &Common, f: &mut ConfigFile) -> Vec { for s in &mut f.sources { let at = format!("source {}", s.id); p.extend(unique(&mut seen, "source", s.id.clone())); - let spec: SourceSpec = match serde_json::from_value(s.spec.clone()) { + let mut spec: SourceSpec = match serde_json::from_value(s.spec.clone()) { Ok(spec) => spec, Err(e) => { p.push(format!("{at}: invalid source spec: {e}")); @@ -296,8 +296,21 @@ fn check(common: &Common, f: &mut ConfigFile) -> Vec { p.push(format!("{at}: {e}")); continue; } + // A raw output (from before flows) comes back as a NATS topic step. + if let Some(sub) = s.raw_subject.take() { + if let Err(e) = crate::api::check_raw_subject(&sub, &common.nats.tracks_subject) { + p.push(format!("{at}: raw output: {e}")); + continue; + } + if spec.pipeline.add_output_topic(&sub).is_none() { + p.push(format!( + "{at}: raw output: the source has no Output to publish beside" + )); + continue; + } + } // A draft comes back a draft; an enabled source must be complete. - let schema = schemas.get(&spec.pipeline.mapping.schema_version); + let schema = schemas.get(&spec.pipeline.schema_version()); let problems = spec.problems(schema); let refused: Vec<_> = problems .iter() @@ -311,11 +324,6 @@ fn check(common: &Common, f: &mut ConfigFile) -> Vec { Ok(v) => s.problems = v, Err(e) => p.push(format!("{at}: {e}")), } - if let Some(sub) = &s.raw_subject - && let Err(e) = crate::api::check_raw_subject(sub, &common.nats.tracks_subject) - { - p.push(format!("{at}: raw output: {e}")); - } s.name = spec.name.clone(); s.transport = spec.transport.kind().into(); s.codec = spec.pipeline.codec.name().into(); diff --git a/crates/ot-server/src/engine.rs b/crates/ot-server/src/engine.rs index 52659fa..a732937 100644 --- a/crates/ot-server/src/engine.rs +++ b/crates/ot-server/src/engine.rs @@ -765,19 +765,20 @@ impl Engine { .filter(|s| s.enabled) .map(|s| { // A tracker stage turns detections into tracks before they get here. - let detections = s.spec.get("reports").and_then(|r| r.as_str()) - == Some("detections") - && s.spec - .pointer("/pipeline/tracker") - .is_none_or(Value::is_null); let spec = serde_json::from_value::(s.spec.clone()) .ok(); + // Plots reach correlation untracked: no Tracker before the Output. + let detections = s.spec.get("reports").and_then(|r| r.as_str()) + == Some("detections") + && !spec + .as_ref() + .is_some_and(|spec| spec.pipeline.output_tracked()); let alone = spec.as_ref().is_none_or(|spec| spec.publishes_alone()); let confirm = spec.as_ref().and_then(|spec| spec.confirms_after()); let motion = spec.as_ref().and_then(|spec| spec.emitter_motion); let stage = spec - .and_then(|spec| spec.pipeline.registry) + .map(|spec| spec.pipeline.output_registry()) .unwrap_or_default(); (s.id, s.priority, detections, alone, confirm, stage, motion) }) diff --git a/crates/ot-server/src/engine/bench.rs b/crates/ot-server/src/engine/bench.rs index 636e76c..c54711b 100644 --- a/crates/ot-server/src/engine/bench.rs +++ b/crates/ot-server/src/engine/bench.rs @@ -325,7 +325,10 @@ pub async fn run(common: &Common, opts: BenchOptions) -> anyhow::Result<()> { "observations": observations, "sources": specs.iter().map(|(s, _)| json!({ "id": s.id, "reports": s.reports, - "tracker": s.pipeline.tracker.as_ref().map(|t| serde_json::to_value(t).unwrap_or(Value::Null)), + "tracker": s.pipeline.nodes.iter().find_map(|n| match &n.step { + ot_source::flow::Step::Tracker(t) => Some(serde_json::to_value(t).unwrap_or(Value::Null)), + _ => None, + }), })).collect::>(), "correlation": correlation_used, "plugins": plugins, @@ -412,7 +415,7 @@ async fn replay( } let end = now > last; for p in pipelines.values_mut() { - let out = p.flush(now, end); + let out = p.flush(now, end, ®istry); if let Some(e) = &out.last_error { let entry = errors.entry(specs_id(p)).or_insert((0, String::new())); entry.0 += 1; diff --git a/crates/ot-server/src/sources.rs b/crates/ot-server/src/sources.rs index 6f20055..15edd87 100644 --- a/crates/ot-server/src/sources.rs +++ b/crates/ot-server/src/sources.rs @@ -5,8 +5,8 @@ //! backoff) feeding its pipeline, which appends observations to the source's //! Redis stream. The worker also persists the static-join cache, flushes //! per-source metrics, and publishes a live status the API reads. A source -//! with a raw-output subject (set with recorded consent) also publishes each -//! of its observations, before correlation, as JSON on that NATS subject. +//! with NATS topic steps also publishes what reaches each, as JSON on its +//! subject, without correlation. //! Entity fields a feed updates (a pipeline's track → entity links) are //! written to the registry every few seconds. @@ -109,7 +109,7 @@ pub async fn run( shutdown: impl std::future::Future, ) -> anyhow::Result<()> { let redis = common.open_redis().await?; - // Only raw output uses NATS here; the client connects in the background. + // Only NATS topic steps use NATS here; the client connects in the background. let nats = common.connect_nats().await?.client().clone(); let registry = Arc::new(Registry::default()); let mut running: BTreeMap = BTreeMap::new(); @@ -306,15 +306,12 @@ fn reconcile( ); continue; } - let Some(schema) = schemas.get(&spec.pipeline.mapping.schema_version).cloned() else { - tracing::error!(source = %id, version = spec.pipeline.mapping.schema_version, + let Some(schema) = schemas.get(&spec.pipeline.schema_version()).cloned() else { + tracing::error!(source = %id, version = spec.pipeline.schema_version(), "mapping targets an unpublished schema version; not started"); continue; }; - let raw = row.raw_subject.clone().map(|subject| RawOutput { - client: nats.clone(), - subject, - }); + let raw = Some(nats.clone()); let handle = tokio::spawn(run_source( spec, schema, @@ -336,20 +333,13 @@ fn reconcile( refused } -/// Where a source's raw output goes. -#[derive(Clone)] -struct RawOutput { - client: ot_nats::async_nats::Client, - subject: String, -} - /// Run one source forever: a supervised transport and its pipeline. async fn run_source( spec: SourceSpec, schema: ExtensionSchema, revision: i64, redis: RedisStore, - raw: Option, + raw: Option, registry: Arc, ) { let id = spec.id.clone(); @@ -431,10 +421,10 @@ async fn run_source( supervisor.abort(); } -/// Where a pipeline writes: its Redis stream, and its raw feed if it has one. +/// Where a pipeline writes: its Redis stream, and NATS for its topic steps. struct Outputs<'a> { redis: &'a RedisStore, - raw: Option<&'a RawOutput>, + raw: Option<&'a ot_nats::async_nats::Client>, } async fn pipeline_loop( @@ -451,7 +441,7 @@ async fn pipeline_loop( let mut pipeline = Pipeline::new(id, spec.pipeline.clone()) .context("invalid pipeline")? .with_schema(schema.clone()); - let static_ttl = Duration::from_secs(spec.pipeline.static_join.ttl_secs); + let static_ttl = pipeline.static_ttl(); // Warm the static cache so identity survives a restart. let mut loaded = 0; @@ -466,7 +456,7 @@ async fn pipeline_loop( let mut flush = tokio::time::interval(FLUSH_EVERY); let mut last_error: Option = None; let mut totals: HashMap = HashMap::new(); - let (mut raw_published, mut raw_errors) = (0u64, 0u64); + let mut publish_errors = 0u64; // Scans a tracker holds for late plots are run when their hold passes. let mut hold = tokio::time::interval(TRACKER_HOLD_POLL); loop { @@ -476,13 +466,11 @@ async fn pipeline_loop( tracing::info_span!("source.process", source = %id) .in_scope(|| pipeline.process(&frame, registry.as_ref())) } - _ = hold.tick(), if spec.pipeline.tracker.is_some() => pipeline.flush(Utc::now(), false), + _ = hold.tick(), if pipeline.has_tracker() => pipeline.flush(Utc::now(), false, registry.as_ref()), _ = flush.tick() => { let mut counts = pipeline.take_counts().pairs(); - for (k, n) in [("raw_published", &mut raw_published), ("raw_error", &mut raw_errors)] { - if *n > 0 { - counts.push((k.to_owned(), std::mem::take(n))); - } + if publish_errors > 0 { + counts.push(("publish_error".to_owned(), std::mem::take(&mut publish_errors))); } redis.incr_metrics(id, &counts).await?; for (k, n) in &counts { @@ -510,6 +498,12 @@ async fn pipeline_loop( for o in &mut out.observations { o.security = Some(label.clone()); } + let label = serde_json::to_value(label)?; + for (_, v) in &mut out.published { + if let Some(o) = v.as_object_mut() { + o.insert("security".into(), label.clone()); + } + } } redis .append_observations(id, &out.observations) @@ -519,15 +513,12 @@ async fn pipeline_loop( observations = out.observations.len() )) .await?; - if let Some(raw) = raw { - for obs in &out.observations { - let body = serde_json::to_vec(obs)?; - match raw.client.publish(raw.subject.clone(), body.into()).await { - Ok(()) => raw_published += 1, - Err(e) => { - raw_errors += 1; - last_error = Some(format!("raw output: {e}")); - } + if let Some(nats) = raw { + for (subject, v) in &out.published { + let body = serde_json::to_vec(v)?; + if let Err(e) = nats.publish(subject.clone(), body.into()).await { + publish_errors += 1; + last_error = Some(format!("NATS topic {subject}: {e}")); } } } @@ -559,10 +550,90 @@ fn rand_fraction() -> f64 { (h.finish() >> 11) as f64 / (1u64 << 53) as f64 } +/// Sources saved with a raw-output subject (before flows) publish it from a +/// NATS topic step instead: added once, beside the Output, and the subject +/// cleared. Each change is a revision, recorded as the migration's. +pub fn migrate_raw_outputs(db: &mut ot_store::Db) -> anyhow::Result<()> { + for row in db.list_sources()? { + let Some(subject) = row.raw_subject.clone() else { + continue; + }; + let Ok(mut spec) = serde_json::from_value::(row.spec.clone()) else { + tracing::warn!(source = %row.id, "raw output not moved to a NATS topic step: its spec does not read"); + continue; + }; + if spec.pipeline.add_output_topic(&subject).is_none() { + tracing::warn!(source = %row.id, "raw output not moved to a NATS topic step: it has no Output"); + continue; + } + let normalised = serde_json::to_value(&spec)?; + db.put_source( + &ot_store::SourceWrite { + id: &row.id, + name: &row.name, + transport: &row.transport, + codec: &row.codec, + priority: row.priority, + spec: &normalised, + problems: &row.problems, + }, + "op:migration", + )?; + db.set_raw_output(&row.id, None, "op:migration")?; + tracing::info!(source = %row.id, %subject, "raw output moved to a NATS topic step"); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn a_raw_output_moves_to_a_nats_topic_step_once() { + let mut db = ot_store::Db::open_in_memory().unwrap(); + let spec: serde_json::Value = + serde_json::from_str(include_str!("../../../docs/examples/gps-udp.json")).unwrap(); + let none = json!([]); + let write = ot_store::SourceWrite { + id: spec["id"].as_str().unwrap(), + name: "GPS", + transport: "udp", + codec: "json", + priority: 100, + spec: &spec, + problems: &none, + }; + db.put_source(&write, "op:test").unwrap(); + db.set_raw_output(write.id, Some("raw.gps"), "op:test") + .unwrap(); + migrate_raw_outputs(&mut db).unwrap(); + let row = db.get_source(write.id).unwrap().unwrap(); + assert!(row.raw_subject.is_none()); + let moved: SourceSpec = serde_json::from_value(row.spec.clone()).unwrap(); + let topic = moved.pipeline.node("raw").expect("a NATS topic step"); + assert!(matches!(&topic.step, ot_source::flow::Step::Nats(t) if t.subject == "raw.gps")); + // Beside the Output: fed by the same step. + let feeds = |id: &str| { + moved + .pipeline + .edges + .iter() + .find(|e| e.target == id) + .unwrap() + .source + .clone() + }; + assert_eq!(feeds("raw"), feeds("output")); + assert!(moved.pipeline.shape_problems().is_empty()); + // Again: nothing more to move. + migrate_raw_outputs(&mut db).unwrap(); + assert_eq!( + db.get_source(write.id).unwrap().unwrap().revision, + row.revision + ); + } + /// A UDP listener saved before 0.4.5, without the risk acceptance, is /// not started, and its status says why; with the acceptance it runs. #[tokio::test] @@ -586,7 +657,7 @@ mod tests { redis_namespace: ns.clone(), site: ot_core::SiteCode::new("TST").unwrap(), nats: crate::config::NatsArgs { - // Nothing listens here; only raw output would use it. + // Nothing listens here; only NATS topic steps would use it. url: "nats://127.0.0.1:9".into(), creds: None, token: None, diff --git a/crates/ot-source/src/flow.rs b/crates/ot-source/src/flow.rs index 92244f2..561c7c7 100644 --- a/crates/ot-source/src/flow.rs +++ b/crates/ot-source/src/flow.rs @@ -283,14 +283,14 @@ impl PipelineSpec { if !ok { out.push(invalid( Some(&n.id), - format!( - "step id {:?}: 1-64 of letters, digits, - and _", - n.id - ), + format!("step id {:?}: 1-64 of letters, digits, - and _", n.id), )); } if !ids.insert(n.id.as_str()) { - out.push(invalid(Some(&n.id), format!("two steps are called {:?}", n.id))); + out.push(invalid( + Some(&n.id), + format!("two steps are called {:?}", n.id), + )); } if (n.id == INPUT) != matches!(n.step, Step::Input) { out.push(invalid( @@ -317,27 +317,45 @@ impl PipelineSpec { for e in &self.edges { let (from, to) = (self.node(&e.source), self.node(&e.target)); if from.is_none() && e.source != INPUT { - out.push(invalid(None, format!("a connection from {:?}, which is not a step", e.source))); + out.push(invalid( + None, + format!("a connection from {:?}, which is not a step", e.source), + )); continue; } let Some(to) = to else { - out.push(invalid(None, format!("a connection to {:?}, which is not a step", e.target))); + out.push(invalid( + None, + format!("a connection to {:?}, which is not a step", e.target), + )); continue; }; if !pairs.insert((e.source.as_str(), e.target.as_str())) { - out.push(invalid(Some(&to.id), "it is connected twice from one step".into())); + out.push(invalid( + Some(&to.id), + "it is connected twice from one step".into(), + )); continue; } *incoming.entry(e.target.as_str()).or_default() += 1; match (gives(&e.source), to.step.takes()) { - (_, None) => out.push(invalid(Some(&to.id), "nothing connects into the input".into())), + (_, None) => out.push(invalid( + Some(&to.id), + "nothing connects into the input".into(), + )), (None, _) => out.push(invalid( Some(&e.source), - format!("a {} passes nothing on", from.map_or("step", |f| f.step.name())), + format!( + "a {} passes nothing on", + from.map_or("step", |f| f.step.name()) + ), )), (Some(Data::Records), Some(Data::Observations)) => out.push(invalid( Some(&to.id), - format!("a {} takes observations: put a Map before it", to.step.name()), + format!( + "a {} takes observations: put a Map before it", + to.step.name() + ), )), (Some(Data::Observations), Some(Data::Records)) => out.push(invalid( Some(&to.id), @@ -563,6 +581,34 @@ impl PipelineSpec { } } +impl PipelineSpec { + /// Add a NATS topic step publishing on `subject` what the Output takes + /// (fed by the Output's step before it), as the raw output of a source + /// from before flows did. Returns its id; `None` if there is no Output + /// fed by a step. + pub fn add_output_topic(&mut self, subject: &str) -> Option { + let out = self.output()?.id.clone(); + let parent = self.edges.iter().find(|e| e.target == out)?.source.clone(); + let mut id = "raw".to_owned(); + let mut n = 1; + while self.node(&id).is_some() { + n += 1; + id = format!("raw-{n}"); + } + self.nodes.push(node( + &id, + Step::Nats(NatsStep { + subject: subject.to_owned(), + }), + )); + self.edges.push(FlowEdge { + source: parent, + target: id.clone(), + }); + Some(id) + } +} + #[cfg(test)] mod tests { use super::*; @@ -586,13 +632,24 @@ mod tests { "throttle": {"min_interval_secs": 5} })); let ids: Vec<&str> = p.nodes.iter().map(|n| n.id.as_str()).collect(); - assert_eq!(ids, ["input", "reject", "map", "registry", "filter", "throttle", "output"]); + assert_eq!( + ids, + [ + "input", "reject", "map", "registry", "filter", "throttle", "output" + ] + ); assert_eq!(p.edges.len(), 6); assert!(p.shape_problems().is_empty(), "{:?}", p.shape_problems()); - let Step::Map(m) = &p.node("map").unwrap().step else { panic!() }; - assert!(m.mapping.reject.is_empty(), "the rejects moved to their own step"); + let Step::Map(m) = &p.node("map").unwrap().step else { + panic!() + }; + assert!( + m.mapping.reject.is_empty(), + "the rejects moved to their own step" + ); // Saved, it reads back the same. - let again: PipelineSpec = serde_json::from_value(serde_json::to_value(&p).unwrap()).unwrap(); + let again: PipelineSpec = + serde_json::from_value(serde_json::to_value(&p).unwrap()).unwrap(); assert_eq!(again, p); assert!(p.output_registry() == RegistryStage::default() && !p.output_tracked()); } @@ -622,36 +679,66 @@ mod tests { #[test] fn shape_problems_name_their_step() { - let base = || json!({"codec": {"type": "json"}, "nodes": [ + let base = || { + json!({"codec": {"type": "json"}, "nodes": [ {"id": "m", "type": "map", "config": {"mapping": map()}}, {"id": "f", "type": "filter", "config": {}}, {"id": "out", "type": "output"} - ], "edges": [{"source": "input", "target": "m"}, {"source": "m", "target": "f"}, {"source": "f", "target": "out"}]}); + ], "edges": [{"source": "input", "target": "m"}, {"source": "m", "target": "f"}, {"source": "f", "target": "out"}]}) + }; let problems = |v: Value| -> Vec<(bool, Option)> { - spec(v).shape_problems().into_iter().map(|p| (p.invalid, p.node)).collect() + spec(v) + .shape_problems() + .into_iter() + .map(|p| (p.invalid, p.node)) + .collect() }; // A filter before the map. let mut v = base(); v["edges"] = json!([{"source": "input", "target": "f"}, {"source": "f", "target": "m"}, {"source": "m", "target": "out"}]); - assert_eq!(problems(v), [(true, Some("f".into())), (true, Some("m".into()))]); + assert_eq!( + problems(v), + [(true, Some("f".into())), (true, Some("m".into()))] + ); // Two inputs into one step. let mut v = base(); - v["edges"].as_array_mut().unwrap().push(json!({"source": "input", "target": "f"})); + v["edges"] + .as_array_mut() + .unwrap() + .push(json!({"source": "input", "target": "f"})); assert!(problems(v).contains(&(true, Some("f".into())))); // A loop. let mut v = base(); - v["nodes"].as_array_mut().unwrap().push(json!({"id": "g", "type": "filter", "config": {}})); - v["nodes"].as_array_mut().unwrap().push(json!({"id": "h", "type": "filter", "config": {}})); - v["edges"].as_array_mut().unwrap().extend([json!({"source": "g", "target": "h"}), json!({"source": "h", "target": "g"})]); + v["nodes"] + .as_array_mut() + .unwrap() + .push(json!({"id": "g", "type": "filter", "config": {}})); + v["nodes"] + .as_array_mut() + .unwrap() + .push(json!({"id": "h", "type": "filter", "config": {}})); + v["edges"].as_array_mut().unwrap().extend([ + json!({"source": "g", "target": "h"}), + json!({"source": "h", "target": "g"}), + ]); assert!(problems(v).contains(&(true, Some("g".into())))); // Two outputs. let mut v = base(); - v["nodes"].as_array_mut().unwrap().push(json!({"id": "out2", "type": "output"})); + v["nodes"] + .as_array_mut() + .unwrap() + .push(json!({"id": "out2", "type": "output"})); assert!(problems(v).contains(&(true, None))); // Loose ends are only incomplete: a draft. let mut v = base(); - v["nodes"].as_array_mut().unwrap().push(json!({"id": "t", "type": "throttle", "config": {}})); - v["edges"].as_array_mut().unwrap().push(json!({"source": "f", "target": "t"})); + v["nodes"] + .as_array_mut() + .unwrap() + .push(json!({"id": "t", "type": "throttle", "config": {}})); + v["edges"] + .as_array_mut() + .unwrap() + .push(json!({"source": "f", "target": "t"})); assert_eq!(problems(v), [(false, Some("t".into()))]); let mut v = base(); v["edges"] = json!([{"source": "input", "target": "m"}]); diff --git a/crates/ot-source/src/pipeline.rs b/crates/ot-source/src/pipeline.rs index 7964718..a1e9a0b 100644 --- a/crates/ot-source/src/pipeline.rs +++ b/crates/ot-source/src/pipeline.rs @@ -421,7 +421,9 @@ impl Pipeline { Step::Filter(f) => Run::Filter(f.clone()), Step::Registry(r) => Run::Registry(r.clone()), Step::Affiliation(a) => Run::Affiliation(a.clone()), - Step::Tracker(t) => Run::Tracker(Stage::new(t.clone()).map_err(PipelineError::Tracker)?), + Step::Tracker(t) => { + Run::Tracker(Stage::new(t.clone()).map_err(PipelineError::Tracker)?) + } Step::Throttle(t) => Run::Throttle { spec: t.clone(), last: HashMap::new(), @@ -433,7 +435,9 @@ impl Pipeline { } let mut children = vec![Vec::new(); spec.nodes.len()]; for e in &spec.edges { - if let (Some(&from), Some(&to)) = (index.get(e.source.as_str()), index.get(e.target.as_str())) { + if let (Some(&from), Some(&to)) = + (index.get(e.source.as_str()), index.get(e.target.as_str())) + { children[from].push(to); } } @@ -462,7 +466,8 @@ impl Pipeline { .or_else(|| steps.iter().position(|s| matches!(s, Run::Map { .. }))); Ok(Self { source_id: source_id.into(), - codec: Codec::new(spec.codec.clone()).map_err(|e| PipelineError::Codec(e.to_string()))?, + codec: Codec::new(spec.codec.clone()) + .map_err(|e| PipelineError::Codec(e.to_string()))?, spec, steps, children, @@ -551,13 +556,16 @@ impl Pipeline { pub fn prune(&mut self, now: DateTime) { for s in &mut self.steps { match s { - Run::Map { ttl_secs, statics, .. } => { + Run::Map { + ttl_secs, statics, .. + } => { let ttl = chrono::Duration::from_std(Duration::from_secs(*ttl_secs)) .unwrap_or(chrono::Duration::MAX); statics.retain(|_, e| now - e.updated_at < ttl); } Run::Throttle { spec, last, .. } => { - let keep = chrono::Duration::milliseconds((spec.heartbeat_secs * 1000.0) as i64 * 2); + let keep = + chrono::Duration::milliseconds((spec.heartbeat_secs * 1000.0) as i64 * 2); last.retain(|_, w| now - w.at < keep); } _ => {} @@ -615,7 +623,9 @@ impl Pipeline { } for record in records { self.counts.records += 1; - let sample = trace.as_deref_mut().and_then(|t| t.begin(frame, &self.order)); + let sample = trace + .as_deref_mut() + .and_then(|t| t.begin(frame, &self.order)); if let (Some(t), Some(at)) = (trace.as_deref_mut(), sample) { t.samples[at].item(INPUT, || record.clone()); } @@ -633,10 +643,17 @@ impl Pipeline { // Plots still waiting for the rest of their scan. if let Some(tr) = trace { for (i, s) in self.steps.iter().enumerate() { - let Run::Tracker(Stage::Builtin(t)) = s else { continue }; + let Run::Tracker(Stage::Builtin(t)) = s else { + continue; + }; let id = &self.spec.nodes[i].id; for f in &mut tr.samples { - let mine: Vec<&Observation> = f.plots.iter().filter(|(n, _)| n == id).map(|(_, o)| o).collect(); + let mine: Vec<&Observation> = f + .plots + .iter() + .filter(|(n, _)| n == id) + .map(|(_, o)| o) + .collect(); if !mine.is_empty() && t.waiting().any(|w| mine.contains(&w)) && let Some(st) = f.stage(id) @@ -730,7 +747,9 @@ impl Pipeline { } } Run::Tracker(_) => { - let Some(obs) = self.typed(&id, &item, out, sample.as_deref_mut()) else { return }; + let Some(obs) = self.typed(&id, &item, out, sample.as_deref_mut()) else { + return; + }; if item.kind == RuleKind::Observation { self.counts.plots += 1; if self.keep_plots { @@ -750,8 +769,17 @@ impl Pipeline { } } Run::Throttle { .. } => { - let Some(obs) = self.typed(&id, &item, out, sample.as_deref_mut()) else { return }; - let Run::Throttle { spec, last, pending } = &mut self.steps[at] else { unreachable!() }; + let Some(obs) = self.typed(&id, &item, out, sample.as_deref_mut()) else { + return; + }; + let Run::Throttle { + spec, + last, + pending, + } = &mut self.steps[at] + else { + unreachable!() + }; let key = obs.source_track_key.clone(); if obs.state == Some(ot_core::TrackState::Dropped) { // A source track's end always passes. @@ -789,7 +817,9 @@ impl Pipeline { let stage = self.default_registry.clone(); self.link(&stage, &mut item, registry, out); } - let Some(obs) = self.typed(&id, &item, out, sample.as_deref_mut()) else { return }; + let Some(obs) = self.typed(&id, &item, out, sample.as_deref_mut()) else { + return; + }; self.counts.emitted += 1; if let Some(f) = sample { f.obs(&id, &obs); @@ -833,7 +863,13 @@ impl Pipeline { /// Look up the item's entity with these Entity links: grades it, passes /// fields between entity and track, and queues the entity's updates. - fn link(&mut self, stage: &RegistryStage, item: &mut Item, registry: &dyn RegistryLookup, out: &mut Output) { + fn link( + &mut self, + stage: &RegistryStage, + item: &mut Item, + registry: &dyn RegistryLookup, + out: &mut Output, + ) { let matched = stage.run(&mut item.value, registry); // Values mapped to the entity go no further than here. if let Some(ext) = item.value.get_mut("ext").and_then(Value::as_object_mut) { @@ -849,14 +885,17 @@ impl Pipeline { let fresh: Vec<(String, Value)> = m .updates .into_iter() - .filter(|(field, v)| self.entity_sent.get(&(m.entity_id.clone(), field.clone())) != Some(v)) + .filter(|(field, v)| { + self.entity_sent.get(&(m.entity_id.clone(), field.clone())) != Some(v) + }) .collect(); if !fresh.is_empty() { if self.entity_sent.len() > 100_000 { self.entity_sent.clear(); } for (field, v) in &fresh { - self.entity_sent.insert((m.entity_id.clone(), field.clone()), v.clone()); + self.entity_sent + .insert((m.entity_id.clone(), field.clone()), v.clone()); } out.entity_updates.push((m.entity_id, fresh)); } @@ -875,7 +914,9 @@ impl Pipeline { out: &mut Output, trace: &mut Option<&mut Trace>, ) { - let Run::Map { spec, .. } = &self.steps[at] else { return }; + let Run::Map { spec, .. } = &self.steps[at] else { + return; + }; let schema_version = spec.schema_version; let outcome = spec.apply(&item.value); let mut sample = match (trace.as_deref_mut(), item.sample) { @@ -892,7 +933,10 @@ impl Pipeline { } MapOutcome::Unmatched => { if let Some(f) = sample { - f.dropped(id, "no mapping rule matched it (or the rule's key was empty)"); + f.dropped( + id, + "no mapping rule matched it (or the rule's key was empty)", + ); } self.counts.unmatched += 1; return; @@ -905,10 +949,21 @@ impl Pipeline { self.counts.statics += 1; if let Some(f) = sample.as_deref_mut() { f.item(id, || mapped_json(m)); - f.note(id, format!("kept the static identity of {}, to fill its later reports", m.key)); + f.note( + id, + format!( + "kept the static identity of {}, to fill its later reports", + m.key + ), + ); } let stored = self.static_key(at, &m.key); - let Run::Map { statics, changed, .. } = &mut self.steps[at] else { return }; + let Run::Map { + statics, changed, .. + } = &mut self.steps[at] + else { + return; + }; let entry = statics.entry(m.key.clone()).or_insert_with(|| StaticEntry { fields: Value::Object(Default::default()), identifiers: Vec::new(), @@ -929,7 +984,12 @@ impl Pipeline { out.statics.push((stored, entry.clone())); } for mut m in mapped.into_iter().filter(|m| m.kind.reports()) { - let Run::Map { statics, changed, .. } = &mut self.steps[at] else { return }; + let Run::Map { + statics, changed, .. + } = &mut self.steps[at] + else { + return; + }; if let Some(s) = statics.get(&m.key) { mapping::fill_missing(&mut m.fields, &s.fields); for i in &s.identifiers { @@ -958,9 +1018,11 @@ impl Pipeline { } } // Later steps work on the observation's JSON form. - let obs = match mapping::finalize(&self.source_id, schema_version, &m, item.received_at).and_then(|o| { - serde_json::to_value(&o).map_err(|e| mapping::FinalizeError::Invalid(e.to_string())) - }) { + let obs = match mapping::finalize(&self.source_id, schema_version, &m, item.received_at) + .and_then(|o| { + serde_json::to_value(&o) + .map_err(|e| mapping::FinalizeError::Invalid(e.to_string())) + }) { Ok(v) => v, Err(e) => { self.counts.invalid += 1; @@ -1003,7 +1065,12 @@ impl Pipeline { /// wait `scan_hold_secs` for more; `force` runs them regardless). The /// worker calls this on a timer; a dry run calls it with `force` at the /// end. - pub fn flush(&mut self, now: DateTime, force: bool, registry: &dyn RegistryLookup) -> Output { + pub fn flush( + &mut self, + now: DateTime, + force: bool, + registry: &dyn RegistryLookup, + ) -> Output { let mut out = Output::default(); self.run_trackers(now, force, registry, &mut out, &mut None); out @@ -1026,7 +1093,9 @@ impl Pipeline { for (i, s) in self.steps.iter().enumerate() { let Run::Tracker(stage) = s else { continue }; let why = match stage { - Stage::Plugin { .. } => "a tracker plugin's reports are not traced back to the records behind them", + Stage::Plugin { .. } => { + "a tracker plugin's reports are not traced back to the records behind them" + } Stage::Builtin(_) => { "no track report: the tracker has not confirmed a track from these plots (or merged them into a nearby plot, or they came too late for their scan)" } @@ -1092,9 +1161,11 @@ impl Pipeline { for (i, obs) in reports.into_iter().enumerate() { let mut sample = None; if let Some(t) = trace.as_deref_mut() { - sample = dets - .get(i) - .and_then(|d| t.samples.iter().position(|f| f.plots.iter().any(|(n, p)| n == &id && p == d))); + sample = dets.get(i).and_then(|d| { + t.samples + .iter() + .position(|f| f.plots.iter().any(|(n, p)| n == &id && p == d)) + }); if let Some(f) = sample.and_then(|s| t.samples.get_mut(s)) { f.obs(&id, &obs); } @@ -1199,7 +1270,8 @@ mod tests { /// A flow from steps and connections, after the input. fn flow(nodes: Value, edges: Value) -> PipelineSpec { - serde_json::from_value(json!({"codec": {"type": "json"}, "nodes": nodes, "edges": edges})).unwrap() + serde_json::from_value(json!({"codec": {"type": "json"}, "nodes": nodes, "edges": edges})) + .unwrap() } fn pos_map() -> Value { @@ -1241,15 +1313,33 @@ mod tests { }; let a = feed(&mut p, ®, rec(1, 32.0, Some("ALPHA"))); assert_eq!(a.observations.len(), 1, "named and south: to correlation"); - assert_eq!(a.published.iter().map(|(s, _)| s.as_str()).collect::>(), ["feed.all"]); + assert_eq!( + a.published + .iter() + .map(|(s, _)| s.as_str()) + .collect::>(), + ["feed.all"] + ); // The Output looks the entity up, with no Entity links step before it. assert_eq!(a.observations[0].ext["registry"]["entity_id"], "e1"); - assert!(a.published[0].1["ext"].get("registry").is_none(), "a topic gets the mapped data"); + assert!( + a.published[0].1["ext"].get("registry").is_none(), + "a topic gets the mapped data" + ); let b = feed(&mut p, ®, rec(2, 34.0, None)); assert!(b.observations.is_empty(), "unnamed, and north"); - assert_eq!(b.published.iter().map(|(s, _)| s.as_str()).collect::>(), ["feed.all", "feed.north"]); + assert_eq!( + b.published + .iter() + .map(|(s, _)| s.as_str()) + .collect::>(), + ["feed.all", "feed.north"] + ); assert_eq!(b.published[1].1["position"]["latitude"], 34.0); - assert_eq!((p.counts.emitted, p.counts.published, p.counts.filtered), (1, 3, 2)); + assert_eq!( + (p.counts.emitted, p.counts.published, p.counts.filtered), + (1, 3, 2) + ); } #[test] @@ -1270,14 +1360,29 @@ mod tests { let mut trace = Trace::new(5); let body = json!([{"id": 1, "lat": 32.0, "lon": 1.0, "ts": "2026-09-24T12:00:00Z"}, {"id": 2, "lat": 40.0, "lon": 1.0, "ts": "2026-09-24T12:00:00Z"}]); - p.process_traced(&Frame::new(body.to_string().into_bytes()), &Reg::new(), &mut trace); - let ids: Vec<&str> = trace.samples[0].stages.iter().map(|s| s.id.as_str()).collect(); + p.process_traced( + &Frame::new(body.to_string().into_bytes()), + &Reg::new(), + &mut trace, + ); + let ids: Vec<&str> = trace.samples[0] + .stages + .iter() + .map(|s| s.id.as_str()) + .collect(); assert_eq!(ids, ["input", "m", "f", "raw", "out"]); let (a, b) = (&trace.samples[0], &trace.samples[1]); - assert_eq!((stage(a, "out").items.len(), stage(a, "raw").items.len()), (1, 1)); + assert_eq!( + (stage(a, "out").items.len(), stage(a, "raw").items.len()), + (1, 1) + ); assert_eq!(stage(b, "f").dropped, ["drop_if matched"]); assert!(stage(b, "out").items.is_empty()); - assert_eq!(stage(b, "raw").items.len(), 1, "the other branch still has it"); + assert_eq!( + stage(b, "raw").items.len(), + 1, + "the other branch still has it" + ); } #[test] @@ -1286,7 +1391,9 @@ mod tests { json!([{"id": "f", "type": "filter", "config": {}}, {"id": "out", "type": "output"}]), json!([{"source": "input", "target": "f"}, {"source": "f", "target": "out"}]), ); - assert!(matches!(Pipeline::new("x", spec), Err(PipelineError::Flow(m)) if m.contains("put a Map before it"))); + assert!( + matches!(Pipeline::new("x", spec), Err(PipelineError::Flow(m)) if m.contains("put a Map before it")) + ); } #[test] @@ -1526,7 +1633,11 @@ mod tests { assert_eq!(p.counts.plots, 8); assert_eq!(p.counts.emitted, 6); // Nothing waits: frames are scans. - assert!(p.flush(Utc::now(), true, &Reg::new()).observations.is_empty()); + assert!( + p.flush(Utc::now(), true, &Reg::new()) + .observations + .is_empty() + ); } fn stage<'a>(f: &'a crate::trace::SampleTrace, id: &str) -> &'a crate::trace::StageTrace { @@ -1574,14 +1685,24 @@ mod tests { let ids: Vec<&str> = f[0].stages.iter().map(|s| s.id.as_str()).collect(); assert_eq!( ids, - ["input", "map", "registry", "affiliation", "filter", "output"], + [ + "input", + "map", + "registry", + "affiliation", + "filter", + "output" + ], "the flow's steps, in the order they are met" ); assert_eq!(trace.frames[0].format, "json"); // A static report is kept by the map, to join later reports. let map = stage(&f[0], "map"); assert_eq!(map.items[0]["kind"], "static"); - assert!(map.dropped.is_empty(), "keeping a static identity is not a drop"); + assert!( + map.dropped.is_empty(), + "keeping a static identity is not a drop" + ); assert!(map.notes[0].starts_with("kept the static identity of 1")); // A position report: joined, graded, published. assert_eq!(stage(&f[1], "map").items[0]["name"], "EXAMPLE"); diff --git a/crates/ot-source/src/source.rs b/crates/ot-source/src/source.rs index d234ab9..73ac94a 100644 --- a/crates/ot-source/src/source.rs +++ b/crates/ot-source/src/source.rs @@ -264,7 +264,10 @@ impl SourceSpec { fn checks(&self, schema: Option>) -> Vec { let mut out = Vec::new(); let input = || Some(crate::flow::INPUT.to_owned()); - let push = |out: &mut Vec, node: Option, field: Option<&'static str>, error: SourceError| { + let push = |out: &mut Vec, + node: Option, + field: Option<&'static str>, + error: SourceError| { let invalid = matches!(error, SourceError::BadId(_) | SourceError::NoName); out.push(Check { node, @@ -281,7 +284,12 @@ impl SourceSpec { if let Some(m) = &self.emitter_motion && let Err(e) = m.check() { - push(&mut out, None, Some("emitter_motion"), SourceError::Tracker(e)); + push( + &mut out, + None, + Some("emitter_motion"), + SourceError::Tracker(e), + ); } let id_ok = !self.id.is_empty() && self.id.len() <= 64 @@ -290,7 +298,12 @@ impl SourceSpec { .bytes() .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-' || b == b'_'); if !id_ok { - push(&mut out, None, Some("id"), SourceError::BadId(self.id.clone())); + push( + &mut out, + None, + Some("id"), + SourceError::BadId(self.id.clone()), + ); } if self.name.trim().is_empty() { push(&mut out, None, Some("name"), SourceError::NoName); @@ -329,17 +342,33 @@ impl SourceSpec { self.transport, TransportConfig::GrpcClient { .. } | TransportConfig::GrpcServer { .. } ); - let protobuf = matches!(self.pipeline.codec, crate::codec::CodecConfig::Protobuf { .. }); - let plugin = matches!(self.pipeline.codec, crate::codec::CodecConfig::Plugin { .. }); + let protobuf = matches!( + self.pipeline.codec, + crate::codec::CodecConfig::Protobuf { .. } + ); + let plugin = matches!( + self.pipeline.codec, + crate::codec::CodecConfig::Plugin { .. } + ); // A gRPC source's protobuf is compiled (once) by the check below. if (plugin || (protobuf && !grpc)) && let Err(e) = crate::codec::Codec::new(self.pipeline.codec.clone()) { - push(&mut out, input(), Some("codec"), SourceError::Tracker(e.to_string())); + push( + &mut out, + input(), + Some("codec"), + SourceError::Tracker(e.to_string()), + ); } let missing = self.transport.missing(); for (field, what) in &missing { - push(&mut out, input(), Some(*field), SourceError::Transport(format!("{what} is not set"))); + push( + &mut out, + input(), + Some(*field), + SourceError::Transport(format!("{what} is not set")), + ); } if let Err(e) = self.transport.check() { push(&mut out, input(), None, SourceError::Transport(e)); @@ -357,10 +386,12 @@ impl SourceSpec { Ok(Some(_)) if !missing.is_empty() => {} Ok(Some(proto)) => { let fit = match &self.transport { - TransportConfig::GrpcClient { method, request, .. } => { - proto.client_request(method, request).map(drop) + TransportConfig::GrpcClient { + method, request, .. + } => proto.client_request(method, request).map(drop), + TransportConfig::GrpcServer { methods, .. } => { + proto.server_methods(methods).map(drop) } - TransportConfig::GrpcServer { methods, .. } => proto.server_methods(methods).map(drop), _ => Ok(()), }; if let Err(e) = fit { @@ -369,15 +400,20 @@ impl SourceSpec { } } } - if let TransportConfig::TcpClient { framing, .. } | TransportConfig::TcpServer { framing, .. } = - &self.transport + if let TransportConfig::TcpClient { framing, .. } + | TransportConfig::TcpServer { framing, .. } = &self.transport && let Err(e) = crate::frame::Framer::new(framing.clone()) { push(&mut out, input(), Some("framing"), e.into()); } // Last, so that a broken setting is reported before its security. if let Some(e) = self.authentication_problem() { - push(&mut out, input(), Some("unauthenticated"), SourceError::Unauthenticated(e)); + push( + &mut out, + input(), + Some("unauthenticated"), + SourceError::Unauthenticated(e), + ); } if let Some(schema) = schema { let wanted = self.pipeline.schema_version(); @@ -563,7 +599,11 @@ mod tests { (incomplete, Some("map".into()), None), (incomplete, Some("input".into()), Some("host".into())), (incomplete, Some("input".into()), Some("port".into())), - (incomplete, Some("map".into()), Some("schema_version".into())), + ( + incomplete, + Some("map".into()), + Some("schema_version".into()) + ), ] ); assert!(matches!(draft.validate(), Err(SourceError::Mapping(_)))); @@ -590,7 +630,10 @@ mod tests { ) ); assert_eq!( - found.iter().filter(|p| p.1.as_deref() == Some("map")).count(), + found + .iter() + .filter(|p| p.1.as_deref() == Some("map")) + .count(), 5, "unknown field, no position (twice), duplicate rule, schema: {found:?}" ); diff --git a/crates/ot-source/tests/sapient_acoustic_example.rs b/crates/ot-source/tests/sapient_acoustic_example.rs index 17532b3..e15909a 100644 --- a/crates/ot-source/tests/sapient_acoustic_example.rs +++ b/crates/ot-source/tests/sapient_acoustic_example.rs @@ -28,7 +28,8 @@ fn acoustic_example_maps_a_sapient_bearing() { } } }); - let MapOutcome::Mapped(mapped) = source.pipeline.mappings().next().unwrap().apply(&record) else { + let MapOutcome::Mapped(mapped) = source.pipeline.mappings().next().unwrap().apply(&record) + else { panic!("acoustic record did not match the source mapping"); }; let observation = finalize(&source.id, 1, &mapped[0], Utc::now()).expect("valid bearing"); diff --git a/crates/ot-source/tests/stanag4607.rs b/crates/ot-source/tests/stanag4607.rs index 4c17cb4..fdb4c9d 100644 --- a/crates/ot-source/tests/stanag4607.rs +++ b/crates/ot-source/tests/stanag4607.rs @@ -276,7 +276,9 @@ fn gmti_plots_carry_ground_error_ellipses() { continue; } located += usize::from(r.get("lat").is_some()); - let ot_source::mapping::MapOutcome::Mapped(mapped) = spec.pipeline.mappings().next().unwrap().apply(r) else { + let ot_source::mapping::MapOutcome::Mapped(mapped) = + spec.pipeline.mappings().next().unwrap().apply(r) + else { continue; }; for m in mapped.iter().filter(|m| m.rule == "target") { From 634c88c1064664fcccc92890ea22acdb630f9a9d Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Wed, 21 Oct 2026 17:29:41 -0500 Subject: [PATCH 13/16] Sources: a workspace of steps the user places and connects (#104) The editor is a full workspace: the step list on the left (Records: Reject, Map; Observations: Filter, Entity links, Affiliation, Tracker, Throttle; Outputs: Output, NATS topic), steps dragged anywhere and connected by hand, one input each, any number of each step, branching to the source's one Output and any NATS topics. Clicking a step opens its settings and samples over the workspace; the input's are the transport and codec. The feed's own settings, status and history open from the top row. Connections that would feed a step the wrong data, a second input or a loop are refused as they are drawn. The topology on the Sources list draws each source's flow. Co-Authored-By: Claude Opus 5.5 --- ui/src/api/client.ts | 30 +- ui/src/index.css | 74 +-- ui/src/lib/pipeline.test.ts | 26 +- ui/src/lib/pipeline.ts | 196 +------ ui/src/lib/trace.test.ts | 4 +- ui/src/lib/trace.ts | 47 +- ui/src/pages/sources/SourceEditor.tsx | 338 +++++++----- ui/src/pages/sources/SourceStatus.tsx | 2 +- ui/src/pages/sources/Topology.tsx | 73 +-- .../pages/sources/designer/StageSamples.tsx | 4 +- ui/src/pages/sources/flow/FeedPanel.tsx | 114 ++++ ui/src/pages/sources/flow/NodePanel.tsx | 262 +++++++++ ui/src/pages/sources/flow/StepNode.tsx | 19 +- ui/src/pages/sources/flow/StepPanel.tsx | 269 ---------- ui/src/pages/sources/flow/sourceFlow.test.ts | 197 +++---- ui/src/pages/sources/flow/sourceFlow.ts | 499 ++++++++++-------- ui/src/pages/sources/flow/stepIcons.tsx | 40 +- 17 files changed, 1093 insertions(+), 1101 deletions(-) create mode 100644 ui/src/pages/sources/flow/FeedPanel.tsx create mode 100644 ui/src/pages/sources/flow/NodePanel.tsx delete mode 100644 ui/src/pages/sources/flow/StepPanel.tsx diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index 8b90808..18e1f64 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -832,14 +832,23 @@ const enc = encodeURIComponent // --- Sources ------------------------------------------------------------------------------- /** A source spec as stored: transport + pipeline, all JSON. */ +/** A Map step's mapping. */ +export interface MappingSpec { + schema_version?: number + rules: unknown[] + reject?: unknown[] +} + export interface SourceSpec { id: string name: string description?: string transport: { type: string; [k: string]: unknown } + /** The codec, and the flow of steps after the input (crates/ot-source/src/flow.rs). */ pipeline: { codec: { type: string; [k: string]: unknown } - mapping: { schema_version?: number; rules: unknown[]; reject?: unknown[] } + nodes?: unknown[] + edges?: unknown[] [k: string]: unknown } priority?: number @@ -913,25 +922,12 @@ export interface SourceRow { status: SourceStatus | null } -/** A pipeline stage, in the order a frame meets them. */ -export type SourceStage = - | 'source' - | 'decode' - | 'map' - | 'static_join' - | 'registry' - | 'affiliation' - | 'filter' - | 'tracker' - | 'throttle' - | 'output' - /** One reason a source may not run. */ export interface SourceProblem { /** `invalid`: the source cannot be saved. `incomplete`: it saves as a draft. */ kind: 'invalid' | 'incomplete' - /** The stage it belongs to; null for the source as a whole (its id and name). */ - stage: SourceStage | null + /** The step of the flow it belongs to (`input` for the transport and codec); null for the source as a whole. */ + node: string | null /** The setting, where there is one (for example `host`). */ field?: string message: string @@ -980,7 +976,7 @@ export interface ProbeResult { codec: SourceSpec['pipeline']['codec'] suggested_records_path: string | null fields: FieldStat[] - suggestion: { mapping: SourceSpec['pipeline']['mapping']; proposals: Proposal[]; identifiers: unknown[]; missing: string[] } + suggestion: { mapping: MappingSpec; proposals: Proposal[]; identifiers: unknown[]; missing: string[] } samples_saved: number | null sample_frames: { received_at: string diff --git a/ui/src/index.css b/ui/src/index.css index f90431c..5bd5be8 100644 --- a/ui/src/index.css +++ b/ui/src/index.css @@ -905,8 +905,8 @@ body { gap: var(--space-md); } -/* Source editor (#104): the steps on the left, the selected step's settings and samples (or the - source's own settings, status and history) on the right. It fills the page below the tabs. */ +/* Source editor (#104): a workspace of the source's steps, the step list on its left; a step's + settings (or the feed's) open over its right. It fills the page below the tabs. */ .source-editor { display: flex; flex-direction: column; @@ -928,21 +928,27 @@ body { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; - max-width: 50vw; + max-width: 40vw; } -.source-editor-body { +.source-workspace { + position: relative; flex: 1; min-height: 0; - display: grid; - grid-template-columns: minmax(380px, 440px) minmax(0, 1fr); - gap: var(--space-md); } -.source-editor-aside { - min-height: 0; - overflow-y: auto; - padding-right: var(--space-xs); +.source-workspace-panel { + position: absolute; + top: 0; + right: 0; + bottom: 0; + width: min(760px, 58%); + min-width: 360px; + z-index: 5; + display: flex; + background: var(--color-bg-primary); + border-left: 1px solid var(--color-glass-border); + box-shadow: var(--shadow-deep); } .source-editor-json { @@ -951,19 +957,29 @@ body { overflow: hidden; } -/* Settings beside their samples; stacked on narrower screens. */ -.step-panel { - display: grid; - grid-template-columns: minmax(0, 1fr) minmax(300px, 420px); - gap: var(--space-lg); - align-items: start; +.node-panel { + display: flex; + flex-direction: column; + flex: 1; + min-width: 0; } -.step-samples { - position: sticky; - top: 0; - max-height: calc(100vh - 180px); +.node-panel-head { + display: flex; + align-items: center; + gap: var(--space-sm); + padding: var(--space-sm) var(--space-md); + border-bottom: 1px solid var(--color-glass-border); +} + +.node-panel-body { + flex: 1; min-height: 0; + overflow-y: auto; + padding: var(--space-md); + display: flex; + flex-direction: column; + gap: var(--space-sm); } .step-settings { @@ -973,19 +989,9 @@ body { min-width: 0; } -@media (max-width: 1400px) { - .step-panel { - grid-template-columns: minmax(0, 1fr); - } - .step-samples { - position: static; - max-height: 60vh; - } -} - -/* A step on the canvas. */ +/* A step on the workspace. */ .flow-step { - width: 240px; + width: 220px; padding: 8px 10px; border-radius: var(--radius-md); background: var(--color-bg-secondary); @@ -1023,7 +1029,7 @@ body { letter-spacing: 0.08em; } -.step-panel .flow-step-kind { +.node-panel-head .flow-step-kind { margin-left: 0; } diff --git a/ui/src/lib/pipeline.test.ts b/ui/src/lib/pipeline.test.ts index b596add..72b366b 100644 --- a/ui/src/lib/pipeline.test.ts +++ b/ui/src/lib/pipeline.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import type { SchemaOverview, SourceSpec } from '../api/client' -import { describeCondition, describeValue, destinations, fieldMap, pipelineStages, valueSources } from './pipeline' +import { describeCondition, describeValue, destinations, fieldMap, valueSources } from './pipeline' const schema = { published_core: [], @@ -81,27 +81,3 @@ describe('destinations', () => { }) }) -describe('pipelineStages', () => { - it('shows a tracker stage before publishing, with the classification it may give', () => { - const spec = { - id: 'radar', - name: 'Radar', - transport: { type: 'udp' }, - reports: 'detections', - pipeline: { - codec: { type: 'json' }, - mapping: { rules: [{ name: 'plot', key: 'id', fields: {} }] }, - tracker: { algorithm: 'mht', domain: 'surface' }, - }, - } as unknown as SourceSpec - const stages = pipelineStages(spec) - const ids = stages.map((s) => s.id) - expect(ids.indexOf('tracker')).toBe(ids.length - 2) - const tracker = stages.find((s) => s.id === 'tracker')! - expect(tracker.title).toBe('tracker · MHT') - expect(tracker.facts.find((f) => f.label === 'Classification')!.value).toBe('unknown affiliation, surface; no identity') - // With a tracker the feed's plots arrive as tracks. - expect(stages.at(-1)!.summary).toBe('system track, published to NATS') - }) -}) - diff --git a/ui/src/lib/pipeline.ts b/ui/src/lib/pipeline.ts index 2aee9dc..1eb4916 100644 --- a/ui/src/lib/pipeline.ts +++ b/ui/src/lib/pipeline.ts @@ -1,5 +1,5 @@ /** - * Plain-language views of a source pipeline, for the Pipeline flowchart and field map. + * Plain-language views of a source's settings, for its steps and the field map. * * Mirrors the server's evaluation (ot-source `expr.rs`, `mapping.rs`, `pipeline.rs`): a value spec * is a path string or an object with one source (`path`, `const`, `first`, `template`, `cases`, @@ -100,190 +100,6 @@ export function describeValue(spec: ValueSpec | undefined): string { return [bits.join(' '), ...after].filter(Boolean).join(' · ') } -// --- Stages -------------------------------------------------------------------------------- - -export interface Fact { - label: string - value: string -} - -export interface Stage { - id: string - title: string - /** One line under the title in the flowchart. */ - summary: string - facts: Fact[] -} - -const secs = (s: number) => (s >= 86400 ? `${+(s / 86400).toFixed(1)} d` : s >= 3600 ? `${+(s / 3600).toFixed(1)} h` : s >= 60 ? `${+(s / 60).toFixed(1)} min` : `${s} s`) - -function transportStage(spec: SourceSpec): Stage { - const t = spec.transport - const where = String(t.url ?? t.bind ?? (t.host ? `${t.host}:${t.port ?? ''}` : '')) - const facts: Fact[] = Object.entries(t) - .filter(([k]) => k !== 'type') - .map(([k, v]) => ({ label: k, value: typeof v === 'string' ? v : JSON.stringify(v) })) - return { id: 'transport', title: t.type.replace('_', ' '), summary: where, facts } -} - -/** The stages a record passes through, in the order the source worker runs them. */ -export function pipelineStages(spec: SourceSpec): Stage[] { - const p = spec.pipeline as SourceSpec['pipeline'] & Obj - const codec = p.codec - const rules = (p.mapping.rules as Obj[]) ?? [] - const reject = (p.mapping.reject as Obj[]) ?? [] - const fieldCount = rules.reduce((n, r) => n + Object.keys((r.fields as Obj) ?? {}).length, 0) - const out: Stage[] = [ - transportStage(spec), - { - id: 'decode', - title: `decode ${codec.type}`, - summary: typeof codec.records === 'string' ? `records at "${codec.records}"` : 'one record per frame', - facts: Object.entries(codec) - .filter(([k]) => k !== 'type') - .map(([k, v]) => ({ label: k, value: typeof v === 'string' ? v : JSON.stringify(v) })), - }, - ] - if (reject.length) { - out.push({ - id: 'reject', - title: 'reject', - summary: plural(reject.length, 'rule'), - facts: reject.map((r) => ({ label: String(r.reason), value: describeCondition(r.when) })), - }) - } - out.push({ - id: 'map', - title: 'map', - summary: `${plural(rules.length, 'rule')} · ${plural(fieldCount, 'field')}`, - facts: [], - }) - if (rules.some((r) => r.kind === 'static')) { - const ttl = Number((p.static_join as Obj | undefined)?.ttl_secs ?? 7 * 86400) - out.push({ - id: 'join', - title: 'identity join', - summary: `keeps identity fields ${secs(ttl)}`, - facts: [ - { label: 'What it does', value: 'Fields from static rules are cached per track key and fill gaps in later position reports.' }, - { label: 'Kept for', value: secs(ttl) }, - ], - }) - } - const reg = p.registry as Obj | undefined - const links = entityLinks(reg) - const toTrack = links.filter((l) => l.direction !== 'to_entity').length - out.push({ - id: 'registry', - title: 'entity', - summary: `${plural(toTrack, 'field')} from the entity${links.length > toTrack ? `, ${links.length - toTrack} back to it` : ''}${reg ? '' : ' (defaults)'}`, - facts: [ - { label: 'Schemes', value: ((reg?.schemes as string[]) ?? []).join(', ') || 'every identifier' }, - { label: 'Name to grade', value: String(reg?.broadcast_name ?? 'name') }, - { label: 'Used at grades', value: ((reg?.apply_grades as string[]) ?? DEFAULT_GRADES).join(', ') }, - ...links.map((l) => - l.direction === 'to_entity' - ? { label: `updates entity ${l.entity}`, value: `from the track's ${l.track}` } - : { label: `sets ${l.track}`, value: `from the entity's ${l.entity}` }, - ), - ], - }) - const aff = p.affiliation as Obj | undefined - if (aff) { - const n = (k: string) => ((aff[k] as string[]) ?? []).length - out.push({ - id: 'affiliation', - title: 'affiliation', - summary: `by country · ${n('friendly')} friendly, ${n('hostile')} hostile`, - facts: [ - { label: 'Country from', value: describeValue(aff.country as ValueSpec) }, - { label: 'Friend', value: ((aff.friendly as string[]) ?? []).join(' ') || '—' }, - { label: 'Hostile', value: ((aff.hostile as string[]) ?? []).join(' ') || '—' }, - { label: 'Neutral', value: ((aff.neutral as string[]) ?? []).join(' ') || '—' }, - { label: 'Other countries', value: String(aff.otherwise ?? 'left unset') }, - { label: 'No country', value: String(aff.unknown_country ?? 'left unset') }, - ], - }) - } - const filter = p.filter as Obj | undefined - if (filter) { - const facts: Fact[] = [] - if (filter.keep_if) facts.push({ label: 'Keep only if', value: describeCondition(filter.keep_if) }) - if (filter.drop_if) facts.push({ label: 'Drop if', value: describeCondition(filter.drop_if) }) - out.push({ id: 'filter', title: 'filter', summary: filter.keep_if ? 'keep only matching' : 'drop matching', facts }) - } - const tr = p.tracker as Obj | undefined - if (tr) { - const mht = (tr.mht as Obj | undefined) ?? {} - const alg = tr.algorithm === 'mht' ? 'MHT' : tr.algorithm === 'plugin' ? `plugin ${String(tr.plugin ?? '')}` : 'GNN' - const profile: Fact[] = tr.profile ? [{ label: 'Profile', value: String(tr.profile) }] : [] - const facts: Fact[] = - tr.algorithm === 'plugin' - ? [ - ...profile, - { label: 'Tracker', value: `the ${String(tr.plugin ?? '?')} plugin (Settings → General → Plugins)` }, - { label: 'Options', value: Object.keys((tr.options as Obj | undefined) ?? {}).length ? JSON.stringify(tr.options) : 'its defaults' }, - { label: 'Track keys', value: 'as the plugin names them' }, - ] - : [ - ...profile, - { label: 'Association', value: alg === 'MHT' ? `multiple hypotheses, final after ${Number(mht.n_scan ?? 3)} scans` : 'global nearest neighbour' }, - { label: 'Classification', value: `unknown affiliation${tr.domain ? `, ${String(tr.domain)}` : ', domain from the plots if they report one'}; no identity` }, - { - label: 'Confirmed at', - value: `existence ${Number(tr.confirm_probability ?? 0.95)}, after at least ${Number(tr.confirm_hits ?? 3)} plots`, - }, - { label: 'Clutter', value: `${Number(tr.clutter_density ?? 1e-6)} false plots per m², detection ${Number(tr.detection_probability ?? 0.9)}` }, - { - label: 'Dropped after', - value: tr.auto_timing != null ? 'a few missed revisits (auto timing)' : `${secs(Number(tr.drop_confirmed_secs ?? 8))} without a plot`, - }, - { label: 'Track keys', value: `${String(tr.key_prefix ?? 'T')}-1, -2…` }, - ] - out.push({ - id: 'tracker', - title: `tracker · ${alg}`, - summary: `plots → tracks · ${tr.domain ? `${String(tr.domain)}, ` : ''}${tr.profile ? `${String(tr.profile)} · ` : ''}unknown affiliation`, - facts, - }) - } - const th = p.throttle as Obj | undefined - if (th) { - const min = Number(th.min_interval_secs ?? 0) - const beat = Number(th.heartbeat_secs ?? 600) - const move = Number(th.min_move_m ?? 0) - out.push({ - id: 'throttle', - title: 'throttle', - summary: `≥ ${secs(min)} apart · ${move} m move`, - facts: [ - { label: 'At most every', value: secs(min) }, - { label: 'Between that and the heartbeat', value: move > 0 ? `only after moving ${move} m` : 'every report' }, - { label: 'Heartbeat', value: `always after ${secs(beat)}` }, - ], - }) - } - const plots = spec.reports === 'detections' && !tr - out.push({ - id: 'publish', - title: 'correlation → TRACKS', - summary: plots ? 'plots associated with system tracks' : 'system track, published to NATS', - facts: [ - { - label: 'Correlation', - value: plots - ? 'Each plot updates the nearest system track inside the gate; plots with none are dropped.' - : 'Each source track reports for one system track, paired with other sources’ tracks on a shared identifier or kinematic agreement.', - }, - { - label: 'Published', - value: 'Confirmed tracks that a source allowed to stand alone reports for: the GOLD fields plus the output schema’s attributes.', - }, - ], - }) - return out -} - // --- Entity links -------------------------------------------------------------------------- export interface EntityLink { @@ -371,7 +187,13 @@ export function destinations(target: string, schema: SchemaOverview): Destinatio export function fieldMap(spec: SourceSpec, schema: SchemaOverview): FieldRow[] { const p = spec.pipeline as SourceSpec['pipeline'] & Obj const rows: FieldRow[] = [] - for (const [i, r] of ((p.mapping.rules as Obj[]) ?? []).entries()) { + // Every Map's rules, and the first Entity links step's (a source from before flows has one of each). + const nodes = (p.nodes as { type?: string; config?: Obj }[] | undefined) ?? [] + const rules = nodes.length + ? nodes.filter((n) => n.type === 'map').flatMap((n) => ((n.config?.mapping as Obj | undefined)?.rules as Obj[]) ?? []) + : (((p.mapping as Obj | undefined)?.rules as Obj[]) ?? []) + const registry = nodes.length ? nodes.find((n) => n.type === 'registry')?.config : (p.registry as Obj | undefined) + for (const [i, r] of rules.entries()) { const stage = `${String(r.name ?? `rule ${i + 1}`)}${r.kind === 'static' ? ' (identity)' : ''}` rows.push({ id: `${i}:key`, @@ -404,7 +226,7 @@ export function fieldMap(spec: SourceSpec, schema: SchemaOverview): FieldRow[] { }) } } - for (const [n, l] of entityLinks(p.registry as Obj | undefined).entries()) { + for (const [n, l] of entityLinks(registry).entries()) { rows.push( l.direction === 'to_entity' ? { diff --git a/ui/src/lib/trace.test.ts b/ui/src/lib/trace.test.ts index 335e444..7442845 100644 --- a/ui/src/lib/trace.test.ts +++ b/ui/src/lib/trace.test.ts @@ -74,7 +74,7 @@ describe('samplesAt', () => { const filter = samplesAt(trace, 'filter') expect(filter[2].notes).toEqual(['dropped by Filter: drop_if matched']) // Not a stage of this sample's (older) trace. - expect(filter[3].notes[0]).toMatch(/not in the pipeline/) + expect(filter[3].notes[0]).toMatch(/not in the flow/) }) it('says when a stage holds a sample, and when nothing came out', () => { @@ -84,7 +84,7 @@ describe('samplesAt', () => { const publish = samplesAt(trace, 'publish') expect(publish[0].notes.some((n) => n.startsWith('held by Tracker'))).toBe(true) const one: PreviewTrace = { frames: trace.frames, samples: [{ frame: 0, stages: [{ id: 'map', items: [] }] }] } - expect(samplesAt(one, 'map')[0].notes).toEqual(['Nothing came out of this stage.']) + expect(samplesAt(one, 'map')[0].notes).toEqual(['Nothing came out of this step.']) }) it('shows at most five samples, and none without a trace', () => { diff --git a/ui/src/lib/trace.ts b/ui/src/lib/trace.ts index 027330a..32d8dee 100644 --- a/ui/src/lib/trace.ts +++ b/ui/src/lib/trace.ts @@ -1,14 +1,15 @@ /** - * The pipeline designer's live preview: the first stored records (samples) as they are after one - * stage, from the dry run's trace (`POST /sources/validate` with `trace`). + * A source step's samples: the first stored records as they are after that step, from the dry + * run's trace (`POST /sources/validate` with `trace`). */ import type { PreviewResult, PreviewTrace, TraceFrameView } from '../api/client' /** Records the designer follows through the pipeline. */ export const TRACED_SAMPLES = 5 -/** Stage names for "dropped by …", by the stage ids of `pipelineStages`. */ +/** Step names for "dropped by …", by step type (a step's id starts with its type). */ export const STAGE_LABEL: Record = { + input: 'Input', transport: 'Transport', decode: 'Decode', reject: 'Reject', @@ -51,9 +52,16 @@ function frameView(f: TraceFrameView | undefined, title: string): SampleView { return { title, blocks: [block], notes } } +/** A step's name for a note: its type's, numbered like its id (`Filter 2`). */ +function label(stageId: string): string { + const m = /^(.*?)(?:-(\d+))?$/.exec(stageId) + const name = STAGE_LABEL[m?.[1] ?? stageId] + return name ? `${name}${m?.[2] ? ` ${m[2]}` : ''}` : stageId +} + function note(stageId: string, reason: string): string { if (reason.startsWith('rejected:')) return reason - return `dropped by ${STAGE_LABEL[stageId] ?? stageId}: ${reason}` + return `dropped by ${label(stageId)}: ${reason}` } /** @@ -62,9 +70,9 @@ function note(stageId: string, reason: string): string { * dropped at or before the stage says where and why, and one a stage held back (a tracker * waiting for the rest of a scan) says so. */ -export function samplesAt(trace: PreviewTrace | undefined, stageId: string): SampleView[] { +export function samplesAt(trace: PreviewTrace | undefined, stageId: string, path?: string[]): SampleView[] { const samples = (trace?.samples ?? []).slice(0, TRACED_SAMPLES) - if (stageId === 'transport') { + if (stageId === 'transport' || stageId === 'input') { const byFrame = new Map() samples.forEach((s, i) => byFrame.set(s.frame, [...(byFrame.get(s.frame) ?? []), i + 1])) return [...byFrame].map(([frame, ns]) => @@ -74,20 +82,22 @@ export function samplesAt(trace: PreviewTrace | undefined, stageId: string): Sam return samples.map((s, i) => { const title = `#${i + 1}` const at = s.stages.findIndex((x) => x.id === stageId) - if (at < 0) return { title, blocks: [], notes: ['This stage was not in the pipeline the preview ran.'] } + if (at < 0) return { title, blocks: [], notes: ['This step was not in the flow the preview ran.'] } const notes: string[] = [] - for (const x of s.stages.slice(0, at + 1)) { + // Only the steps on the way here: a drop on another branch is not this step's. + const before = s.stages.slice(0, at + 1).filter((x) => !path || path.includes(x.id) || x === s.stages[at]) + for (const x of before) { for (const r of x.dropped ?? []) notes.push(note(x.id, r)) // Not a drop: said only at the stage that did it. - if (x === s.stages[at]) for (const n of x.notes ?? []) notes.push(`${STAGE_LABEL[x.id] ?? x.id}: ${n}`) + if (x === s.stages[at]) for (const n of x.notes ?? []) notes.push(`${label(x.id)}: ${n}`) if (x.held) { notes.push( - `held by ${STAGE_LABEL[x.id] ?? x.id}: waiting for the rest of its scan (the preview ran it at the end of the samples, or when a later frame completed it)`, + `held by ${label(x.id)}: waiting for the rest of its scan (the preview ran it at the end of the samples, or when a later frame completed it)`, ) } } const blocks = s.stages[at].items.map(pretty) - if (!blocks.length && !notes.length) notes.push('Nothing came out of this stage.') + if (!blocks.length && !notes.length) notes.push('Nothing came out of this step.') return { title, blocks, notes } }) } @@ -106,12 +116,21 @@ export const MAX_TRACED_SAMPLES = 10 * on), and those of them the filter dropped. With no filter in the pipeline yet, what the last * stage before where it goes passed on, and nothing dropped. */ -export function filterInput(trace: PreviewTrace | undefined): { observations: unknown[]; dropped: unknown[] } { +export function filterInput( + trace: PreviewTrace | undefined, + id = 'filter', + parent?: string, +): { observations: unknown[]; dropped: unknown[] } { const observations: unknown[] = [] const dropped: unknown[] = [] for (const s of trace?.samples ?? []) { - const at = s.stages.findIndex((x) => x.id === 'filter') - const before = at > 0 ? s.stages[at - 1] : [...s.stages].reverse().find((x) => ['affiliation', 'registry', 'join', 'map'].includes(x.id)) + const at = s.stages.findIndex((x) => x.id === id) + // What reaches it: the step that feeds it. + const before = parent + ? s.stages.find((x) => x.id === parent) + : at > 0 + ? s.stages[at - 1] + : [...s.stages].reverse().find((x) => ['affiliation', 'registry', 'join', 'map'].includes(x.id)) if (!before) continue observations.push(...before.items) if (at > 0 && s.stages[at].dropped?.length) { diff --git a/ui/src/pages/sources/SourceEditor.tsx b/ui/src/pages/sources/SourceEditor.tsx index 7e49c08..7de4fb6 100644 --- a/ui/src/pages/sources/SourceEditor.tsx +++ b/ui/src/pages/sources/SourceEditor.tsx @@ -1,8 +1,8 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' -import { TbTrash } from 'react-icons/tb' -import type { NodeChange } from '@xyflow/react' +import { TbSettings, TbTrash } from 'react-icons/tb' +import type { Connection, Edge, EdgeChange, NodeChange } from '@xyflow/react' import '@xyflow/react/dist/style.css' -import { Badge, Breadcrumbs, Button, Input, Label, SaveButton, SegmentedControl, Toggle, useToast } from '@kineticlogic/staresdk' +import { Badge, Breadcrumbs, Button, SaveButton, SegmentedControl, Toggle, useToast } from '@kineticlogic/staresdk' import { CodeEditor } from '@kineticlogic/staresdk/code-editor' import { FlowCanvas, type FlowCanvasEdge, type FlowCanvasNode, type FlowPaletteItem } from '@kineticlogic/staresdk/flow-canvas' import { @@ -19,78 +19,43 @@ import { import { useCan } from '../../auth/context' import { InfoTip } from '../../components/InfoTip' import { errorMessage } from '../../lib/format' -import { pipelineStages } from '../../lib/pipeline' import { SOURCE_STATES_INFO, sourceState } from '../../lib/sourceState' import { MAX_TRACED_SAMPLES, TRACED_SAMPLES } from '../../lib/trace' -import { SourceHistory, SourceStatus } from './SourceStatus' +import { FeedPanel } from './flow/FeedPanel' +import { NodePanel } from './flow/NodePanel' import { StepNode, type StepView } from './flow/StepNode' -import { StepPanel } from './flow/StepPanel' -import { entryIcon, stepIcon } from './flow/stepIcons' +import { stepIcon } from './flow/stepIcons' import { - addEntry, - kindOf, - paletteEntries, - positionOf, - problemsByStep, - removable, - removeStep, - replaces, - stepOfEntry, - stepsOf, - stepTitle, - traceStage, - type StepId, + canConnect, + defaultConfig, + flowOf, + INPUT, + laidOut, + nextNodeId, + nodeSummary, + nodeTitle, + problemsByNode, + STEPS, + withFlow, + withoutPositions, + type Flow, + type FlowNode, + type StepType, } from './flow/sourceFlow' const REFRESH_MS = 5000 +const PREVIEW_DELAY_MS = 600 +const NODE_TYPES = { step: StepNode } /** Whether parsed JSON has a source's shape, enough to show as steps (the server checks the rest). */ function isSpecShaped(v: unknown): v is SourceSpec { const o = (x: unknown): x is Record => typeof x === 'object' && x !== null && !Array.isArray(x) if (!o(v) || typeof v.id !== 'string' || typeof v.name !== 'string' || !o(v.transport) || typeof v.transport.type !== 'string') return false const p = v.pipeline - if (!o(p) || !o(p.codec) || typeof p.codec.type !== 'string' || !o(p.mapping) || !Array.isArray(p.mapping.rules)) return false - return p.mapping.reject === undefined || Array.isArray(p.mapping.reject) -} -const PREVIEW_DELAY_MS = 600 -const NODE_TYPES = { step: StepNode } - -/** A source's own settings: its name and description. */ -function SourcePanel({ spec, row, readOnly, onChange }: { spec: SourceSpec; row: SourceRow; readOnly: boolean; onChange: (s: SourceSpec) => void }) { + if (!o(p) || !o(p.codec) || typeof p.codec.type !== 'string' || !Array.isArray(p.nodes) || !Array.isArray(p.edges)) return false return ( -
-
-
- - onChange({ ...spec, name: e.target.value })} /> -
-
- - -
-
- - { - const { description: _, ...rest } = spec - onChange(e.target.value ? { ...rest, description: e.target.value } : rest) - }} - /> -
-
-

Status

- -

History

- -
+ p.nodes.every((n) => o(n) && typeof n.id === 'string' && typeof n.type === 'string') && + p.edges.every((e) => o(e) && typeof e.source === 'string' && typeof e.target === 'string') ) } @@ -114,20 +79,21 @@ function useBoxHeight(): [(el: HTMLElement | null) => void, number] { } /** - * A dry run of the source as edited over its stored samples, re-run shortly after it stops changing: - * what stops it running (each problem on its stage) and its samples stage by stage. A draft runs as - * far as it can. Nothing is saved or published. + * A dry run of the source as edited over its stored samples, re-run shortly after it stops changing + * (moving a step does not count): what stops it running, each problem on its step, and its samples + * step by step. A draft runs as far as it can. Nothing is saved or published. */ function useLiveCheck(spec: SourceSpec | null, id: string, nonce: number, trace: number, enabled: boolean) { const [preview, setPreview] = useState(null) const [problems, setProblems] = useState(null) const [error, setError] = useState(null) const seq = useRef(0) + const key = spec ? JSON.stringify(withoutPositions(spec)) : '' useEffect(() => { - if (!enabled || !spec) return + if (!enabled || !key) return const n = ++seq.current const t = setTimeout(() => { - api.preview(spec, id, [], trace).then( + api.preview(JSON.parse(key) as SourceSpec, id, [], trace).then( (r) => { if (n !== seq.current) return setPreview(r) @@ -144,14 +110,27 @@ function useLiveCheck(spec: SourceSpec | null, id: string, nonce: number, trace: ) }, PREVIEW_DELAY_MS) return () => clearTimeout(t) - }, [spec, id, nonce, trace, enabled]) + }, [key, id, nonce, trace, enabled]) return { preview, problems, error } } +/** The steps from the input to `id`. */ +function pathTo(flow: Flow, id: string): string[] { + const parent = new Map(flow.edges.map((e) => [e.target, e.source])) + const out = [id] + let at = id + while (parent.has(at) && out.length <= flow.nodes.length) { + at = parent.get(at)! + out.unshift(at) + } + return out +} + /** - * One source as a flow of steps (#104): the palette, the source's steps in pipeline order, and the - * settings and samples of the step selected, or with none selected the source's own settings, status - * and history. Nothing changes on the running source until Save. + * One source as a workspace of steps (#104): the step list on the left; the input, and the steps + * the user places and connects, to the source's Output and NATS topics. Clicking a step opens its + * settings and samples; the feed's own settings, status and history open from the top row. Nothing + * changes on the running source until Save. */ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; onClose: () => void; onDeleted: () => void }) { const { confirm, toast } = useToast() @@ -161,7 +140,9 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o const [loadError, setLoadError] = useState(null) const [spec, setSpec] = useState(null) const [baseline, setBaseline] = useState('') - const [selected, setSelected] = useState(null) + const [selected, setSelected] = useState(null) + const [selectedEdges, setSelectedEdges] = useState>(new Set()) + const [feedOpen, setFeedOpen] = useState(false) const [view, setView] = useState<'flow' | 'json'>('flow') const [jsonText, setJsonText] = useState('') const [jsonBox, jsonHeight] = useBoxHeight() @@ -174,10 +155,12 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o const [capturing, setCapturing] = useState(false) const [nonce, setNonce] = useState(0) + // A source from before flows has no places for its steps: they are laid out once, as loaded. const adopt = useCallback((r: SourceRow) => { + const s = withFlow(r.spec, laidOut(flowOf(r.spec))) setRow(r) - setSpec(r.spec) - setBaseline(JSON.stringify(r.spec)) + setSpec(s) + setBaseline(JSON.stringify(s)) }, []) // The row (its status, enabled, stored problems) refreshes; the spec being edited does not. @@ -203,80 +186,135 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o api.plugins().then(setPlugins, () => setPlugins([])) }, []) + const flow = useMemo(() => (spec ? flowOf(spec) : { nodes: [], edges: [] }), [spec]) const dirty = spec !== null && JSON.stringify(spec) !== baseline - const live = useLiveCheck(spec, id, nonce, selected === 'filter' ? MAX_TRACED_SAMPLES : TRACED_SAMPLES, canAdmin) + const current = selected && flow.nodes.some((n) => n.id === selected) ? selected : null + const currentType = flow.nodes.find((n) => n.id === current)?.type + const live = useLiveCheck(spec, id, nonce, currentType === 'filter' ? MAX_TRACED_SAMPLES : TRACED_SAMPLES, canAdmin) // What stops it running: the live check's, else what was found when it was saved. const problems = useMemo(() => live.problems ?? row?.problems ?? [], [live.problems, row?.problems]) - const steps = useMemo(() => (spec ? stepsOf(spec) : []), [spec]) - const byStep = useMemo(() => problemsByStep(problems, steps), [problems, steps]) - // A step taken out another way (the JSON view, emptying the Reject rules) is no longer selected. - const current = selected && steps.includes(selected) ? selected : null + const byNode = useMemo(() => problemsByNode(problems, flow), [problems, flow]) const edit = (next: SourceSpec) => { setSpec(next) setSaved(false) setNotice(null) } + const editFlow = (f: Flow) => { + if (spec) edit(withFlow(spec, f)) + } const nodes: FlowCanvasNode[] = useMemo(() => { if (!spec) return [] - const summaries = new Map(pipelineStages(spec).map((s) => [s.id, s.summary])) - return steps.map((step, i) => ({ - id: step, + return flow.nodes.map((n) => ({ + id: n.id, type: 'step', - position: positionOf(i), - selected: step === current, - draggable: false, - deletable: removable(step), + position: n.position ?? { x: 0, y: 0 }, + selected: n.id === current, + deletable: n.type !== 'input', data: { - kind: kindOf(step), - icon: stepIcon(spec, step), - title: stepTitle(spec, step), - summary: summaries.get(traceStage(step)) ?? '', - problems: byStep.byStep[step] ?? [], + type: n.type, + icon: stepIcon(spec, n.type), + title: nodeTitle(spec, n), + summary: nodeSummary(spec, n), + problems: byNode.byNode[n.id] ?? [], } satisfies StepView, })) - }, [spec, steps, current, byStep]) + }, [spec, flow, current, byNode]) const edges: FlowCanvasEdge[] = useMemo( - () => steps.slice(1).map((step, i) => ({ id: `${steps[i]}->${step}`, source: steps[i], target: step, deletable: false })), - [steps], + () => + flow.edges.map((e) => { + const eid = `${e.source}->${e.target}` + return { id: eid, source: e.source, target: e.target, selected: selectedEdges.has(eid) } + }), + [flow, selectedEdges], ) + const transportType = spec?.transport.type const palette: FlowPaletteItem[] = useMemo( - () => paletteEntries(plugins).map((e) => ({ type: e.type, label: e.label, group: e.group, icon: entryIcon(e.type) })), - [plugins], + () => + (Object.keys(STEPS) as Exclude[]).map((t) => ({ + type: t, + label: STEPS[t].label, + group: STEPS[t].group, + icon: stepIcon({ transport: { type: transportType ?? '' } } as SourceSpec, t), + })), + [transportType], ) - // Steps keep their places and order. Clicking one selects it; deleting an added stage removes it - // from the source. + // Steps move where they are dragged; clicking one opens it; deleting one takes its connections. const onNodesChange = (changes: NodeChange[]) => { const picks = changes.flatMap((c) => (c.type === 'select' ? [c] : [])) if (picks.length) { const on = picks.find((c) => c.selected) - if (on) setSelected(on.id as StepId) - else if (picks.some((c) => c.id === selected)) setSelected(null) + if (on) { + setSelected(on.id) + setFeedOpen(false) + } else if (picks.some((c) => c.id === selected)) setSelected(null) } - const removed = changes.flatMap((c) => (c.type === 'remove' ? [c.id as StepId] : [])) - if (removed.length && spec) { - edit(removed.reduce(removeStep, spec)) - if (selected && removed.includes(selected)) setSelected(null) + const moved = new Map(changes.flatMap((c) => (c.type === 'position' && c.position ? [[c.id, c.position] as const] : []))) + const removed = new Set(changes.flatMap((c) => (c.type === 'remove' && c.id !== INPUT ? [c.id] : []))) + if (!moved.size && !removed.size) return + editFlow({ + nodes: flow.nodes + .filter((n) => !removed.has(n.id)) + .map((n) => { + const p = moved.get(n.id) + return p ? { ...n, position: { x: Math.round(p.x), y: Math.round(p.y) } } : n + }), + edges: flow.edges.filter((e) => !removed.has(e.source) && !removed.has(e.target)), + }) + if (selected && removed.has(selected)) setSelected(null) + } + + const onEdgesChange = (changes: EdgeChange[]) => { + const picks = changes.flatMap((c) => (c.type === 'select' ? [c] : [])) + if (picks.length) { + setSelectedEdges((prev) => { + const next = new Set(prev) + for (const c of picks) { + if (c.selected) next.add(c.id) + else next.delete(c.id) + } + return next + }) } + const removed = new Set(changes.flatMap((c) => (c.type === 'remove' ? [c.id] : []))) + if (removed.size) editFlow({ ...flow, edges: flow.edges.filter((e) => !removed.has(`${e.source}->${e.target}`)) }) + } + + const isValidConnection = (c: Connection | Edge) => !!c.source && !!c.target && canConnect(flow, c.source, c.target) + + const onConnect = (c: Connection) => { + if (!c.source || !c.target || !canConnect(flow, c.source, c.target)) return + editFlow({ ...flow, edges: [...flow.edges, { source: c.source, target: c.target }] }) } - const onDropNode = async (type: string) => { - if (!spec || readOnly) return - const step = stepOfEntry(type) - if (replaces(spec, type)) { - const what = step === 'source' ? 'transport' : 'decoding' - const label = palette.find((p) => p.type === type)?.label ?? '' - const ok = await confirm(`Replace this source's ${what} settings with a new ${label} step?`, { title: 'Replace step', confirmLabel: 'Replace' }) - if (!ok) return + const onDropNode = (type: string, position: { x: number; y: number }) => { + if (!spec || readOnly || !(type in STEPS)) return + const t = type as Exclude + if (t === 'output' && flow.nodes.some((n) => n.type === 'output')) { + setNotice('A source has one Output.') + return + } + const node: FlowNode = { + id: nextNodeId( + t, + flow.nodes.map((n) => n.id), + ), + type: t, + position: { x: Math.round(position.x), y: Math.round(position.y) }, } - edit(addEntry(spec, type, plugins)) - if (step) setSelected(step) + const config = defaultConfig(t) + if (config !== undefined) node.config = config + editFlow({ ...flow, nodes: [...flow.nodes, node] }) + setSelected(node.id) + setFeedOpen(false) } + const setConfig = (nodeId: string, config: unknown) => editFlow({ ...flow, nodes: flow.nodes.map((n) => (n.id === nodeId ? { ...n, config } : n)) }) + const save = async () => { if (!spec) return setSaving(true) @@ -351,7 +389,10 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o } const state = sourceState(row) - const general = [...(notice ? [notice] : []), ...byStep.general] + const general = [...(notice ? [notice] : []), ...byNode.general] + const node = flow.nodes.find((n) => n.id === current) + const output = flow.nodes.find((n) => n.type === 'output') + const tracked = !!output && pathTo(flow, output.id).some((s) => flow.nodes.find((n) => n.id === s)?.type === 'tracker') return (
@@ -367,7 +408,7 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o {SOURCE_STATES_INFO} {general.length > 0 && ( - + {general.join(' · ')} )} @@ -382,6 +423,17 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o { value: 'json', label: 'JSON' }, ]} /> +
{view === 'flow' ? ( -
+
false} + onEdgesChange={onEdgesChange} + onConnect={onConnect} + isValidConnection={isValidConnection} palette={palette} - onDropNode={(type) => void onDropNode(type)} + onDropNode={onDropNode} readOnly={readOnly} height="100%" - style={{ minHeight: 0 }} /> - + {(node || feedOpen) && ( + + )}
) : (
@@ -435,7 +495,7 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o onChange={(t) => { setJsonText(t) try { - const next = JSON.parse(t) as SourceSpec + const next = JSON.parse(t) as unknown if (isSpecShaped(next) && next.id === id) edit(next) } catch { // Keep the last valid settings; the editor marks the parse error. diff --git a/ui/src/pages/sources/SourceStatus.tsx b/ui/src/pages/sources/SourceStatus.tsx index 7657ceb..d994053 100644 --- a/ui/src/pages/sources/SourceStatus.tsx +++ b/ui/src/pages/sources/SourceStatus.tsx @@ -42,7 +42,7 @@ export function SourceStatus({ source }: { source: SourceRow }) {
{source.revision} · saved {fmtTime(source.updated_at_ms)}
- {(source.spec.pipeline as { tracker?: { auto_timing?: unknown } }).tracker?.auto_timing != null && ( + {((source.spec.pipeline.nodes as { type?: string; config?: { auto_timing?: unknown } }[] | undefined) ?? []).some((n) => n.type === 'tracker' && n.config?.auto_timing != null) && ( <>
Tracker timing
diff --git a/ui/src/pages/sources/Topology.tsx b/ui/src/pages/sources/Topology.tsx index 1e137c7..767f899 100644 --- a/ui/src/pages/sources/Topology.tsx +++ b/ui/src/pages/sources/Topology.tsx @@ -3,6 +3,7 @@ import '@xyflow/react/dist/base.css' import { GraphView, type GraphViewEdge, type GraphViewNode } from '@kineticlogic/staresdk/graph-view' import { api, type ServerStatus, type SourceRow, type SystemMetrics } from '../../api/client' import { sourceState } from '../../lib/sourceState' +import { flowOf, INPUT, nodeTitle } from './flow/sourceFlow' const REFRESH_MS = 10_000 const STAGE_W = 136 @@ -10,35 +11,6 @@ const END_W = 176 type Tone = GraphViewNode['tone'] -/** Pipeline stages in the order the source worker runs them (ot-source `Pipeline::process`). */ -function stages(s: SourceRow, recent: Record, per: (n: number) => string) { - const p = s.spec.pipeline - const drop = (n: number, what: string) => (n > 0 ? `${per(n)} ${what}` : undefined) - const out: { key: string; label: string; sublabel?: string; tone?: Tone }[] = [ - { - key: 'decode', - label: `decode ${p.codec.type}`, - sublabel: drop(recent.decode_error ?? 0, 'errors') ?? `${per(recent.frames ?? 0)} in`, - tone: (recent.decode_error ?? 0) > 0 ? 'warning' : undefined, - }, - { - key: 'map', - label: 'map', - sublabel: drop((recent.rejected ?? 0) + (recent.unmatched ?? 0), 'dropped') ?? `${p.mapping.rules.length} rules`, - }, - ] - if (p.static_join) out.push({ key: 'join', label: 'identity join' }) - // The registry stage runs on every source (with defaults when not configured). - out.push({ key: 'registry', label: 'registry' }) - if (p.affiliation) out.push({ key: 'affiliation', label: 'affiliation' }) - if (p.filter) out.push({ key: 'filter', label: 'filter', sublabel: drop(recent.filtered ?? 0, 'dropped') }) - if (p.throttle) out.push({ key: 'throttle', label: 'throttle', sublabel: drop(recent.throttled ?? 0, 'held') }) - // What leaves the chain, on the last stage unless it reports its own drops. - const last = out[out.length - 1] - last.sublabel ??= `${per(recent.emitted ?? 0)} out` - return out -} - const SOURCE_TONE: Record = { disabled: 'muted', starting: 'warning', @@ -48,8 +20,8 @@ const SOURCE_TONE: Record = { } /** - * Every source's processing chain, left to right, into correlation and out to NATS: the tracks - * stream and any per-source raw feed. Moving dashes mark links that carried data in the last few + * Every source's flow, left to right, into correlation and out to NATS: the tracks stream and + * each source's NATS topics. Moving dashes mark links that carried data in the last few * minutes; dashed grey links belong to disabled sources. */ export default function Topology({ sources, onSelect }: { sources: SourceRow[]; onSelect: (id: string) => void }) { @@ -94,26 +66,25 @@ export default function Topology({ sources, onSelect }: { sources: SourceRow[]; tone: SOURCE_TONE[st.label], width: END_W, }) - let prev = `src:${s.id}` - let flowing = recent.frames ?? 0 - for (const stage of stages(s, recent, per)) { - const id = `${s.id}:${stage.key}` - nodes.push({ id, label: stage.label, sublabel: stage.sublabel, tone: off ? 'muted' : stage.tone, width: STAGE_W }) - link(prev, id, flowing, off) - prev = id - flowing = recent.emitted ?? 0 - } - link(prev, 'engine', recent.emitted ?? 0, off) - if (s.raw_subject) { - const raw = `raw:${s.id}` + // The source's flow: its steps after the input, the Output into correlation, each NATS + // topic out on its own. + const flow = flowOf(s.spec) + const at = (id: string) => (id === INPUT ? `src:${s.id}` : `${s.id}:${id}`) + for (const n of flow.nodes) { + if (n.type === 'input' || n.type === 'output') continue + const nats = n.type === 'nats' nodes.push({ - id: raw, - label: s.raw_subject, - sublabel: `raw feed · ${per(recent.raw_published ?? 0)}`, - width: END_W, - tone: off ? 'muted' : (recent.raw_error ?? 0) > 0 ? 'warning' : undefined, + id: at(n.id), + label: nodeTitle(s.spec, n), + sublabel: nats ? 'NATS topic' : undefined, + tone: off ? 'muted' : nats && (recent.publish_error ?? 0) > 0 ? 'warning' : undefined, + width: nats ? END_W : STAGE_W, }) - link(prev, raw, recent.raw_published ?? 0, off) + } + for (const e of flow.edges) { + const to = flow.nodes.find((n) => n.id === e.target) + if (to?.type === 'output') link(at(e.source), 'engine', recent.emitted ?? 0, off) + else link(at(e.source), at(e.target), to?.type === 'nats' ? (recent.published ?? 0) : (recent.records ?? 0), off) } } @@ -161,8 +132,8 @@ export default function Topology({ sources, onSelect }: { sources: SourceRow[]; height={height} layerGap={36} onNodeClick={(id) => { - // Source, raw-feed and stage nodes open their source; shared nodes do nothing. - const source = /^(src|raw):/.test(id) ? id.slice(4) : id.includes(':') ? id.slice(0, id.lastIndexOf(':')) : null + // Source and step nodes open their source; shared nodes do nothing. + const source = id.startsWith('src:') ? id.slice(4) : id.includes(':') ? id.slice(0, id.indexOf(':')) : null if (source) onSelect(source) }} /> diff --git a/ui/src/pages/sources/designer/StageSamples.tsx b/ui/src/pages/sources/designer/StageSamples.tsx index bc4db07..d6179b6 100644 --- a/ui/src/pages/sources/designer/StageSamples.tsx +++ b/ui/src/pages/sources/designer/StageSamples.tsx @@ -7,8 +7,8 @@ import { countsLine, samplesAt } from '../../../lib/trace' * The live preview's samples after one stage: a line of counts and any errors, then each traced * sample as pretty-printed JSON (or what became of it). The list scrolls, not the modal. */ -export function StageSamples({ result, stageId }: { result: PreviewResult; stageId: string }) { - const samples = samplesAt(result.trace, stageId) +export function StageSamples({ result, stageId, path }: { result: PreviewResult; stageId: string; path?: string[] }) { + const samples = samplesAt(result.trace, stageId, path) const errors = [...new Set(result.errors ?? [])] const counts = countsLine(result) return ( diff --git a/ui/src/pages/sources/flow/FeedPanel.tsx b/ui/src/pages/sources/flow/FeedPanel.tsx new file mode 100644 index 0000000..24af2fa --- /dev/null +++ b/ui/src/pages/sources/flow/FeedPanel.tsx @@ -0,0 +1,114 @@ +import { TbX } from 'react-icons/tb' +import { Button, Input, Label } from '@kineticlogic/staresdk' +import type { SourceRow, SourceSpec } from '../../../api/client' +import { InfoTip } from '../../../components/InfoTip' +import { EmitterMotionForm, PublishForm, SecurityForm } from '../designer/StageForms' +import { SourceHistory, SourceStatus } from '../SourceStatus' + +/** + * The feed's own settings, for everything it reports: its name and description, what it reports + * and how correlation takes it, its priority and security label; then its status and history. + */ +export function FeedPanel({ + spec, + row, + tracked, + problems, + readOnly, + onChange, + onClose, +}: { + spec: SourceSpec + row: SourceRow + /** Whether a Tracker comes before the Output. */ + tracked: boolean + problems: string[] + readOnly: boolean + onChange: (s: SourceSpec) => void + onClose: () => void +}) { + const without = (key: K): SourceSpec => { + const next = { ...spec } + delete next[key] + return next + } + return ( +
+
+ Feed +

{spec.name || spec.id}

+ +
+
+ {problems.map((m) => ( +
+ {m} +
+ ))} +
+
+
+ + onChange({ ...spec, name: e.target.value })} /> +
+
+ + +
+
+ + onChange(e.target.value ? { ...spec, description: e.target.value } : without('description'))} + /> +
+
+ onChange(reports ? { ...spec, reports } : without('reports'))} + onPublishAlone={(publish_alone) => onChange({ ...spec, publish_alone })} + onConfirmAfter={(confirm_after) => onChange({ ...spec, confirm_after })} + /> +
+
+ + + Where several sources report one track, priority breaks ties: it picks the position among equally certain reports, the + identity and classification when no report is corroborated by the registry, and each extension field. Higher wins. Default 100. + +
+ + onChange(e.target.value.trim() === '' ? without('priority') : { ...spec, priority: Math.round(Number(e.target.value)) }) + } + /> +
+ onChange({ ...spec, security })} /> + onChange({ ...spec, emitter_motion })} /> +
+

Status

+ +

History

+ +
+
+ ) +} diff --git a/ui/src/pages/sources/flow/NodePanel.tsx b/ui/src/pages/sources/flow/NodePanel.tsx new file mode 100644 index 0000000..0b9c62c --- /dev/null +++ b/ui/src/pages/sources/flow/NodePanel.tsx @@ -0,0 +1,262 @@ +import { TbDownload, TbWand, TbX } from 'react-icons/tb' +import { Button, Input, Label, Select } from '@kineticlogic/staresdk' +import type { PluginInfo, PreviewResult, ProbeResult, SchemaOverview, SourceSpec } from '../../../api/client' +import { InfoTip } from '../../../components/InfoTip' +import { sampleFields } from '../../../lib/conditions' +import { filterInput } from '../../../lib/trace' +import { MapEditor } from '../designer/MapEditor' +import { StageSamples } from '../designer/StageSamples' +import { AffiliationForm, FilterForm, JoinForm, RegistryForm, RejectForm, ThrottleForm, TrackerForm } from '../designer/StageForms' +import { SenderAuth } from '../SenderAuth' +import { TransportForm } from '../TransportForm' +import { codecChoice, codecChoices, KIND_WORD, nodeTitle, TRANSPORTS, withCodec, withTransport, type FlowNode, type StepType } from './sourceFlow' + +type Obj = Record + +/** What each step does, for the ⓘ beside its heading. */ +const STEP_HELP: Record = { + input: 'Where frames come from, and how each becomes records: the transport, how it connects or listens, how a stream is cut into frames, and the codec.', + reject: 'Drops records before mapping, each rule with a reason counted in the source’s metrics.', + map: 'Turns records into observations: the track key, identifiers and OpenTrack fields each rule takes from the record. Identity fields from static rules fill the same track’s later reports.', + filter: 'Keeps or drops observations; dropped ones are counted as filtered.', + registry: 'Finds each track’s entity by its identifiers, grades the match by broadcast name, and at a corroborated grade passes fields between entity and track. Without one before the Output, the defaults run there.', + affiliation: 'Sets affiliation from a country code, by the country lists.', + tracker: 'Forms tracks from detections (plots).', + throttle: 'Limits how often each source track is passed on, by time and distance moved.', + output: 'The source’s system tracks, to correlation. A source has one.', + nats: 'Publishes what reaches it, as JSON, on its NATS subject, without correlation.', +} + +/** + * The settings of one step, and its samples as they leave it. The input's are the source's + * transport and codec; the others' are the step's own. + */ +export function NodePanel({ + spec, + node, + path, + problems, + schema, + plugins, + preview, + previewError, + probe, + capturing, + readOnly, + onSpec, + onConfig, + onCapture, + onClose, +}: { + spec: SourceSpec + node: FlowNode + /** The steps from the input to this one. */ + path: string[] + problems: string[] + schema: SchemaOverview | null + plugins: PluginInfo[] + preview: PreviewResult | null + previewError: string | null + probe: ProbeResult | null + capturing: boolean + readOnly: boolean + onSpec: (spec: SourceSpec) => void + onConfig: (config: unknown) => void + onCapture: () => void + onClose: () => void +}) { + const config = (node.config ?? {}) as Obj + const set = (patch: Obj) => onConfig({ ...config, ...patch }) + + let settings: React.ReactNode = null + switch (node.type) { + case 'input': { + const detected = probe && JSON.stringify(probe.codec) !== JSON.stringify(spec.pipeline.codec) ? probe.codec : null + settings = ( +
+
+
+ + v && onSpec(withCodec(spec, v, plugins))} + /> +
+
+ {detected && ( +
+ + + The captured samples decode with these settings: {JSON.stringify(detected)}. Using them replaces the codec's settings. + +
+ )} + onSpec({ ...spec, transport })} + onCodec={(codec) => onSpec({ ...spec, pipeline: { ...spec.pipeline, codec } })} + /> +
+ ) + break + } + case 'reject': + settings = onConfig(rules)} /> + break + case 'map': { + const mapping = (config.mapping as Obj | undefined) ?? { rules: [] } + const suggested = probe?.suggestion.mapping + const offer = suggested && suggested.rules.length > 0 && JSON.stringify(suggested.rules) !== JSON.stringify(mapping.rules) + const statics = ((mapping.rules as Obj[] | undefined) ?? []).some((r) => r.kind === 'static') + settings = ( + <> + {offer && ( +
+ + + A mapping inferred from the captured samples: {suggested.rules.length} rule(s) + {probe.suggestion.missing.length ? `; not found in the samples: ${probe.suggestion.missing.join(', ')}` : ''}. Using it replaces + this step's rules. + +
+ )} + {schema ? set({ mapping: m })} /> : LOADING…} + {statics && set({ static_join })} />} + + ) + break + } + case 'filter': { + const input = filterInput(preview?.trace, node.id, path[path.length - 2]) + const fields = sampleFields([...input.observations, ...(preview?.observations ?? [])]) + const key = (o: unknown) => String((o as { source_track_key?: unknown }).source_track_key ?? '') + const examples = [...new Set(input.dropped.map(key).filter(Boolean))].slice(0, 3) + settings = + break + } + case 'registry': + settings = + break + case 'affiliation': + settings = + break + case 'tracker': + settings = + break + case 'throttle': + settings = + break + case 'nats': + settings = ( +
+
+
+ + + The NATS subject to publish on: dot-separated tokens, no wildcards or spaces, outside the system tracks' subjects. + +
+ set({ subject: e.target.value })} + /> +
+
+ ) + break + } + + const title = nodeTitle(spec, node) + return ( +
+
+ {KIND_WORD[node.type]} +

+ {title} + {STEP_HELP[node.type]} +

+ +
+
+ {problems.map((m) => ( +
+ {m} +
+ ))} + {settings && ( +
+ {settings} +
+ )} +
+

+ Samples + + A dry run of the source as edited (not yet saved) over its stored samples, as they leave this step. Nothing is published. + +

+ + {node.type === 'input' && ( + <> + + + Connect with these settings and take up to 20 frames or 15 s of the live feed, whichever comes first. They replace the + source's stored samples. + + + )} +
+ {previewError &&
{previewError}
} + {preview && } +
+
+ ) +} diff --git a/ui/src/pages/sources/flow/StepNode.tsx b/ui/src/pages/sources/flow/StepNode.tsx index 7683cd1..cc09fc2 100644 --- a/ui/src/pages/sources/flow/StepNode.tsx +++ b/ui/src/pages/sources/flow/StepNode.tsx @@ -1,10 +1,10 @@ import type { ReactNode } from 'react' import { Handle, Position, type NodeProps } from '@xyflow/react' -import { KIND_WORD, type StepKind } from './sourceFlow' +import { gives, KIND_WORD, takes, type StepType } from './sourceFlow' /** What the editor gives each step to draw (the canvas node's `data`). */ export interface StepView extends Record { - kind: StepKind + type: StepType icon: ReactNode title: string summary: string @@ -12,29 +12,28 @@ export interface StepView extends Record { } /** - * One step of a source on the canvas: its icon, heading and one line of its settings, joined to the - * step before and after it, with the first thing that stops it running in red (all of them are in - * its settings). + * One step of a source on the workspace: its icon, heading and one line of its settings, a + * connection in unless it is the input and out unless it is an output, and the first thing that + * stops it running in red (its settings list them all). */ -export function StepNode({ data, selected }: NodeProps) { +export function StepNode({ data, selected, isConnectable }: NodeProps) { const step = data as StepView const failing = step.problems.length > 0 return (
- {step.kind !== 'source' && } + {takes(step.type) && }
{step.icon} {step.title} - {KIND_WORD[step.kind]} + {KIND_WORD[step.type]}
{step.summary && {step.summary}} {failing && ( @@ -43,7 +42,7 @@ export function StepNode({ data, selected }: NodeProps) { )} {step.problems.length > 1 && +{step.problems.length - 1} more} - {step.kind !== 'output' && } + {gives(step.type) && }
) } diff --git a/ui/src/pages/sources/flow/StepPanel.tsx b/ui/src/pages/sources/flow/StepPanel.tsx deleted file mode 100644 index dad725a..0000000 --- a/ui/src/pages/sources/flow/StepPanel.tsx +++ /dev/null @@ -1,269 +0,0 @@ -import { TbDownload, TbWand } from 'react-icons/tb' -import { Button, Input, Label } from '@kineticlogic/staresdk' -import type { PreviewResult, ProbeResult, SchemaOverview, SourceSpec } from '../../../api/client' -import { InfoTip } from '../../../components/InfoTip' -import { sampleFields } from '../../../lib/conditions' -import { filterInput } from '../../../lib/trace' -import { MapEditor } from '../designer/MapEditor' -import { StageSamples } from '../designer/StageSamples' -import { - AffiliationForm, - EmitterMotionForm, - FilterForm, - JoinForm, - PublishForm, - RegistryForm, - RejectForm, - SecurityForm, - ThrottleForm, - TrackerForm, -} from '../designer/StageForms' -import { SenderAuth } from '../SenderAuth' -import { TransportForm } from '../TransportForm' -import { KIND_WORD, kindOf, stepTitle, traceStage, type StepId } from './sourceFlow' - -type Obj = Record -type Pipeline = SourceSpec['pipeline'] & Obj - -/** What each step does, for the ⓘ beside its heading. */ -const STEP_HELP: Record = { - source: 'Where frames come from: the transport, how it connects or listens, and how a stream is cut into frames.', - decode: 'Turns each frame into records.', - reject: 'Drops raw records before mapping, each rule with a reason counted in the source’s metrics.', - map: 'Turns records into observations: the track key, identifiers and OpenTrack fields each rule takes from the record.', - join: 'Keeps the identity fields that static rules report per track key, and fills them into later observations with the same key.', - registry: 'Finds each track’s entity by its identifiers, grades the match by broadcast name, and at a corroborated grade passes fields between entity and track.', - affiliation: 'Sets affiliation from a country code, by the country lists.', - filter: 'Keeps or drops observations after mapping and the entity stage; dropped ones are counted as filtered.', - tracker: 'Forms tracks from detections (plots) before they leave the source.', - throttle: 'Limits how often each source track is passed on, by time and distance moved.', - output: 'What the source’s observations are, which decides how correlation takes them, the label they carry, and its priority.', -} - -/** Drop pipeline keys set to undefined, so the saved spec carries only what is configured. */ -const clean = (p: Pipeline): Pipeline => JSON.parse(JSON.stringify(p)) - -/** The settings of one step of a source, then its samples as they leave that step. */ -export function StepPanel({ - spec, - step, - problems, - schema, - preview, - previewError, - probe, - capturing, - readOnly, - onChange, - onCapture, -}: { - spec: SourceSpec - step: StepId - problems: string[] - schema: SchemaOverview | null - preview: PreviewResult | null - previewError: string | null - probe: ProbeResult | null - capturing: boolean - readOnly: boolean - onChange: (spec: SourceSpec) => void - onCapture: () => void -}) { - const p = spec.pipeline as Pipeline - const setPipeline = (next: Pipeline) => onChange({ ...spec, pipeline: clean(next) }) - - let settings: React.ReactNode = null - switch (step) { - case 'source': - settings = ( - <> - onChange({ ...spec, transport })} - onCodec={(codec) => setPipeline({ ...p, codec })} - /> - - - ) - break - case 'decode': { - const detected = probe && JSON.stringify(probe.codec) !== JSON.stringify(spec.pipeline.codec) ? probe.codec : null - settings = ( - <> - {detected && ( -
- - - The captured samples decode with these settings: {JSON.stringify(detected)}. Using them replaces this step's settings. - -
- )} - onChange({ ...spec, transport })} - onCodec={(codec) => setPipeline({ ...p, codec })} - /> - - ) - break - } - case 'reject': - settings = ( - setPipeline({ ...p, mapping: { ...p.mapping, reject: reject.length ? reject : undefined } })} - /> - ) - break - case 'map': { - const suggested = probe?.suggestion.mapping - const offer = suggested && suggested.rules.length > 0 && JSON.stringify(suggested.rules) !== JSON.stringify(p.mapping.rules) - settings = ( - <> - {offer && ( -
- - - A mapping inferred from the captured samples: {suggested.rules.length} rule(s) - {probe.suggestion.missing.length ? `; not found in the samples: ${probe.suggestion.missing.join(', ')}` : ''}. Using it replaces - this step's rules. - -
- )} - {schema ? ( - setPipeline({ ...p, mapping: mapping as Pipeline['mapping'] })} /> - ) : ( - LOADING… - )} - - ) - break - } - case 'join': - settings = setPipeline({ ...p, static_join })} /> - break - case 'registry': - settings = setPipeline({ ...p, registry })} /> - break - case 'affiliation': - settings = setPipeline({ ...p, affiliation })} /> - break - case 'filter': { - const input = filterInput(preview?.trace) - const fields = sampleFields([...input.observations, ...(preview?.observations ?? [])]) - const key = (o: unknown) => String((o as { source_track_key?: unknown }).source_track_key ?? '') - const examples = [...new Set(input.dropped.map(key).filter(Boolean))].slice(0, 3) - const effect = preview ? { dropped: preview.counts?.filtered ?? 0, examples } : null - settings = setPipeline({ ...p, filter })} fields={fields} effect={effect} /> - break - } - case 'tracker': - settings = setPipeline({ ...p, tracker })} /> - break - case 'throttle': - settings = setPipeline({ ...p, throttle })} /> - break - case 'output': - settings = ( -
- { - const { reports: _, ...rest } = spec - onChange(reports ? { ...rest, reports } : rest) - }} - onPublishAlone={(publish_alone) => onChange({ ...spec, publish_alone })} - onConfirmAfter={(confirm_after) => onChange({ ...spec, confirm_after })} - /> -
-
- - - Where several sources report one track, priority breaks ties: it picks the position among equally certain reports, the - identity and classification when no report is corroborated by the registry, and each extension field. Higher wins. Default 100. - -
- { - const { priority: _, ...rest } = spec - onChange(e.target.value.trim() === '' ? rest : { ...rest, priority: Math.round(Number(e.target.value)) }) - }} - /> -
- onChange({ ...spec, security })} /> - onChange({ ...spec, emitter_motion })} /> -
- ) - break - } - - return ( -
-
-
- {KIND_WORD[kindOf(step)]} -

- {stepTitle(spec, step)} - {STEP_HELP[step]} -

-
- {problems.map((m) => ( -
- {m} -
- ))} -
- {settings} -
-
-
-
-

- Samples - - A dry run of the source as edited (not yet saved) over its stored samples, as they leave this step. Nothing is published. - -

- - {step === 'source' && ( - <> - - - Connect with this step's settings and take up to 20 frames or 15 s of the live feed, whichever comes first. They replace the - source's stored samples. - - - )} -
- {previewError &&
{previewError}
} - {preview && } -
-
- ) -} diff --git a/ui/src/pages/sources/flow/sourceFlow.test.ts b/ui/src/pages/sources/flow/sourceFlow.test.ts index 63210b0..e2d5f5c 100644 --- a/ui/src/pages/sources/flow/sourceFlow.test.ts +++ b/ui/src/pages/sources/flow/sourceFlow.test.ts @@ -1,141 +1,150 @@ import { describe, expect, it } from 'vitest' import type { PluginInfo, SourceSpec } from '../../../api/client' import { - addEntry, + canConnect, + COLUMN, + defaultConfig, + flowOf, idFromName, + laidOut, newSourceSpec, - paletteEntries, - problemsByStep, - REQUIRED, - removeStep, - replaces, - STEP_ORDER, - stepOfEntry, - stepsOf, - stepTitle, + nextNodeId, + nodeTitle, + problemsByNode, validId, + withCodec, + withFlow, + withoutPositions, + withTransport, + type Flow, } from './sourceFlow' -/** Every source spec the docs ship (docs/examples). */ -const files = import.meta.glob('../../../../../docs/examples/**/*.json', { import: 'default', eager: true }) -const examples = Object.entries(files).filter(([, v]) => typeof v === 'object' && v !== null && 'transport' in v && 'pipeline' in v) - const plugin = (name: string, framing: Record | null = null): PluginInfo => ({ name, kinds: ['codec'], enabled: true, default_options: { a: 1 }, framing }) as unknown as PluginInfo -describe('stepsOf', () => { - it('finds examples to check', () => { - expect(examples.length).toBeGreaterThan(5) - }) - - it.each(examples)('%s: every required step, in pipeline order', (_, spec) => { - const steps = stepsOf(spec) - for (const r of REQUIRED) expect(steps).toContain(r) - expect(steps).toEqual(STEP_ORDER.filter((s) => steps.includes(s))) - }) - - it.each(examples)('%s: each added stage removes back to the source as it was', (_, spec) => { - for (const entry of paletteEntries([]).filter((e) => e.group === 'Stages')) { - const step = stepOfEntry(entry.type)! - const had = stepsOf(spec).includes(step) - const added = addEntry(spec, entry.type, []) - expect(stepsOf(added)).toContain(step) - if (had) expect(added).toBe(spec) - else expect(removeStep(added, step)).toEqual(spec) - } - }) +/** input → map → { filter → output, nats }, and a throttle on its own. */ +const branched = (): Flow => ({ + nodes: [ + { id: 'input', type: 'input' }, + { id: 'map', type: 'map', config: defaultConfig('map') }, + { id: 'filter', type: 'filter', config: {} }, + { id: 'output', type: 'output' }, + { id: 'nats', type: 'nats', config: { subject: 'raw' } }, + { id: 'throttle', type: 'throttle', config: defaultConfig('throttle') }, + ], + edges: [ + { source: 'input', target: 'map' }, + { source: 'map', target: 'filter' }, + { source: 'filter', target: 'output' }, + { source: 'map', target: 'nats' }, + ], }) describe('a new source', () => { - it('has its Source, Decode, Map and Output steps', () => { + it('has its input, an empty Map and the Output, connected and placed', () => { const s = newSourceSpec('ais', 'AIS', 'websocket') - expect(stepsOf(s)).toEqual(['source', 'decode', 'map', 'output']) + const f = flowOf(s) + expect(f.nodes.map((n) => n.type)).toEqual(['input', 'map', 'output']) + expect(f.edges).toEqual([ + { source: 'input', target: 'map' }, + { source: 'map', target: 'output' }, + ]) + expect(f.nodes.every((n) => n.position)).toBe(true) expect(s.transport).toEqual({ type: 'websocket', url: '' }) - expect(stepTitle(s, 'source')).toBe('WebSocket') - expect(stepTitle(s, 'decode')).toBe('JSON') - expect(stepTitle(s, 'output')).toBe('Tracks') + expect(nodeTitle(s, f.nodes[0])).toBe('WebSocket') }) it('takes its id from its name', () => { expect(idFromName('adsb.lol feed (EU)')).toBe('adsb-lol-feed-eu') expect(validId(idFromName('adsb.lol feed (EU)'))).toBe(true) expect(validId('Bad Id')).toBe(false) - expect(validId('')).toBe(false) }) }) -describe('palette entries', () => { +describe('canConnect', () => { + const f = branched() + + it('lets a step that passes observations on feed one that takes them', () => { + expect(canConnect(f, 'filter', 'throttle')).toBe(true) + expect(canConnect(f, 'map', 'throttle')).toBe(true) + }) + + it('refuses the wrong data, a second input, a loop, and anything out of an output', () => { + expect(canConnect(f, 'input', 'throttle')).toBe(false) // records into an observation step + expect(canConnect(f, 'filter', 'map')).toBe(false) // observations into a Map + expect(canConnect(f, 'map', 'output')).toBe(false) // the Output has its input + expect(canConnect(f, 'output', 'throttle')).toBe(false) // an output passes nothing on + expect(canConnect(f, 'throttle', 'throttle')).toBe(false) + const chain: Flow = { ...f, edges: f.edges.filter((e) => e.target !== 'filter').concat({ source: 'throttle', target: 'filter' }) } + // filter feeds output; throttle feeds filter: output → … → throttle would loop, filter → throttle too. + expect(canConnect(chain, 'filter', 'throttle')).toBe(false) + }) +}) + +describe('steps', () => { + it('are numbered by type when taken', () => { + expect(nextNodeId('filter', ['input', 'map'])).toBe('filter') + expect(nextNodeId('filter', ['filter', 'filter-2'])).toBe('filter-3') + }) + + it('without places are laid out in columns from the input', () => { + const f = laidOut(branched()) + const at = (id: string) => f.nodes.find((n) => n.id === id)!.position! + expect(at('input')).toEqual({ x: 0, y: 0 }) + expect(at('map').x).toBe(COLUMN) + expect(at('filter').x).toBe(2 * COLUMN) + expect(at('nats').x).toBe(2 * COLUMN) + expect(at('filter').y).not.toBe(at('nats').y) + expect(at('throttle').x).toBe(4 * COLUMN) // not connected: after the rest + }) + + it('are dry-run without their places', () => { + const s = withFlow(newSourceSpec('x', 'X', 'udp'), laidOut(branched())) + expect(flowOf(withoutPositions(s)).nodes.some((n) => n.position)).toBe(false) + }) +}) + +describe('the input', () => { const base = newSourceSpec('x', 'X', 'tcp_client') - it('swap the transport, and the acceptance that belonged to the old one goes', () => { - const udp = addEntry(base, 'source:udp', []) - expect(udp.transport.type).toBe('udp') - const accepted: SourceSpec = { ...udp, unauthenticated: 'accepted' } - expect(replaces(accepted, 'source:mqtt')).toBe(true) - expect(replaces(accepted, 'source:udp')).toBe(false) - const mqtt = addEntry(accepted, 'source:mqtt', []) + it('swaps its transport, and the acceptance that belonged to the old one goes', () => { + const accepted: SourceSpec = { ...withTransport(base, 'udp'), unauthenticated: 'accepted' } + const mqtt = withTransport(accepted, 'mqtt') expect(mqtt.transport).toEqual({ type: 'mqtt', url: '', topics: [] }) expect(mqtt.unauthenticated).toBeUndefined() - expect(addEntry(accepted, 'source:udp', [])).toBe(accepted) + expect(withTransport(accepted, 'udp')).toBe(accepted) }) - it('frame CoT on a stream by its end tag', () => { - const cot = addEntry(base, 'decode:cot_xml', []) + it('frames CoT on a stream by its end tag, and takes it away again', () => { + const cot = withCodec(base, 'cot_xml', []) expect(cot.pipeline.codec).toEqual({ type: 'cot_xml' }) expect(cot.transport.framing).toEqual({ type: 'end_tag', tag: '' }) - const server = addEntry(cot, 'source:tcp_server', []) - expect(server.transport.framing).toEqual({ type: 'end_tag', tag: '' }) - // Off CoT, the end tag goes; framing set by hand stays. - expect(addEntry(server, 'decode:json', []).transport.framing).toEqual({ type: 'lines' }) - const custom = { ...server, transport: { ...server.transport, framing: { type: 'delimiter', delimiter: '\\x03' } } } - expect(addEntry(custom, 'decode:json', []).transport.framing).toEqual(custom.transport.framing) + expect(withCodec(cot, 'json', []).transport.framing).toEqual({ type: 'lines' }) + const custom = { ...cot, transport: { ...cot.transport, framing: { type: 'delimiter', delimiter: '\\x03' } } } + expect(withCodec(custom, 'json', []).transport.framing).toEqual(custom.transport.framing) }) - it('take a codec plugin with its options and framing', () => { + it('takes a codec plugin with its options and framing', () => { const p = plugin('stanag4607', { type: 'length_field', offset: 2, width: 'u32' }) - expect(paletteEntries([p]).map((e) => e.type)).toContain('decode:plugin:stanag4607') - const s = addEntry(base, 'decode:plugin:stanag4607', [p]) + const s = withCodec(base, 'plugin:stanag4607', [p]) expect(s.pipeline.codec).toEqual({ type: 'plugin', plugin: 'stanag4607', options: { a: 1 } }) expect(s.transport.framing).toEqual({ type: 'length_field', offset: 2, width: 'u32' }) - expect(stepTitle(s, 'decode')).toBe('stanag4607') - expect(replaces(s, 'decode:plugin:stanag4607')).toBe(false) - expect(replaces(s, 'decode:json')).toBe(true) - expect(addEntry(s, 'decode:json', [p]).transport.framing).toEqual({ type: 'lines' }) - }) - - it('set what the output reports', () => { - const plots = addEntry(base, 'output:detections', []) - expect(plots.reports).toBe('detections') - expect(stepTitle(plots, 'output')).toBe('Detections') - expect('reports' in addEntry(plots, 'output:tracks', [])).toBe(false) - }) - - it('never remove a required step', () => { - for (const r of REQUIRED) expect(removeStep(base, r)).toBe(base) - }) - - it('show Static join while the Map has a static rule, and never remove it', () => { - expect(stepsOf(base)).not.toContain('join') - const statics: SourceSpec = { ...base, pipeline: { ...base.pipeline, mapping: { rules: [{ name: 'ids', kind: 'static', key: 'mmsi' }] } } } - expect(stepsOf(statics)).toEqual(['source', 'decode', 'map', 'join', 'output']) - expect(removeStep(statics, 'join')).toBe(statics) - expect(paletteEntries([]).map((e) => e.type)).not.toContain('stage:join') + expect(withCodec(s, 'json', [p]).transport.framing).toEqual({ type: 'lines' }) }) }) -describe('problemsByStep', () => { - it('puts each problem on its step, the rest with the source', () => { - const steps = stepsOf(newSourceSpec('x', 'X', 'tcp_client')) - const p = problemsByStep( +describe('problemsByNode', () => { + it('puts each problem on its step, the rest with the feed', () => { + const p = problemsByNode( [ - { kind: 'incomplete', stage: 'source', field: 'host', message: 'the host is not set' }, - { kind: 'incomplete', stage: 'map', message: 'mapping has no rules' }, - { kind: 'incomplete', stage: 'static_join', message: 'not shown' }, - { kind: 'invalid', stage: null, field: 'name', message: 'source name must not be empty' }, + { kind: 'incomplete', node: 'input', field: 'host', message: 'the host is not set' }, + { kind: 'incomplete', node: 'map', message: 'mapping has no rules' }, + { kind: 'incomplete', node: 'gone', message: 'not shown' }, + { kind: 'invalid', node: null, field: 'name', message: 'source name must not be empty' }, ], - steps, + branched(), ) - expect(p.byStep).toEqual({ source: ['the host is not set'], map: ['mapping has no rules'] }) + expect(p.byNode).toEqual({ input: ['the host is not set'], map: ['mapping has no rules'] }) expect(p.general).toEqual(['not shown', 'source name must not be empty']) }) }) diff --git a/ui/src/pages/sources/flow/sourceFlow.ts b/ui/src/pages/sources/flow/sourceFlow.ts index 96bed3e..3fbb156 100644 --- a/ui/src/pages/sources/flow/sourceFlow.ts +++ b/ui/src/pages/sources/flow/sourceFlow.ts @@ -1,32 +1,113 @@ /** - * A source as a flow of steps (#104): the canvas is a view of the source's spec. Its steps follow the - * pipeline's fixed order (crates/ot-source/src/pipeline.rs), so they connect themselves: adding a - * stage puts it in its place, and removing one closes the gap. Source, Decode, Map and Output are - * always there; Static join is there while the Map has a static rule; the other stages are added - * from the palette and may be removed. + * A source as a flow of steps (#104), as the server keeps it (crates/ot-source/src/flow.rs): one + * input (the transport and its codec), then steps the user places and connects, each taking one + * input and feeding any number of others, ending at the source's one Output (system tracks, to + * correlation) and any NATS topics (its mapped data, without correlation). + * + * Records flow from the input through Reject steps to a Map, which makes observations; Filter, + * Entity links, Affiliation, Tracker and Throttle steps take observations; Output and NATS topics + * take them and pass nothing on. */ -import type { PluginInfo, SourceProblem, SourceSpec, SourceStage } from '../../../api/client' +import type { PluginInfo, SourceProblem, SourceSpec } from '../../../api/client' import { DEFAULT_LINKS } from '../../../lib/pipeline' type Obj = Record type Transport = SourceSpec['transport'] type Codec = SourceSpec['pipeline']['codec'] -/** The steps a source can have, in the order a frame meets them. */ -export const STEP_ORDER = ['source', 'decode', 'reject', 'map', 'join', 'registry', 'affiliation', 'filter', 'tracker', 'throttle', 'output'] as const -export type StepId = (typeof STEP_ORDER)[number] +export const INPUT = 'input' -/** Steps every source has: they cannot be removed. */ -export const REQUIRED: ReadonlySet = new Set(['source', 'decode', 'map', 'output']) +export type StepType = 'input' | 'reject' | 'map' | 'filter' | 'registry' | 'affiliation' | 'tracker' | 'throttle' | 'output' | 'nats' -/** Whether a step can be taken out on the canvas: an added stage. Static join follows the Map's static rules. */ -export const removable = (step: StepId) => !REQUIRED.has(step) && step !== 'join' +export interface FlowNode { + id: string + type: StepType + config?: unknown + position?: { x: number; y: number } +} + +export interface FlowEdge { + source: string + target: string +} -export type StepKind = 'source' | 'decode' | 'stage' | 'output' +/** A flow's steps and connections, as the spec's `pipeline` holds them. */ +export interface Flow { + nodes: FlowNode[] + edges: FlowEdge[] +} -export const kindOf = (step: StepId): StepKind => (step === 'source' || step === 'decode' || step === 'output' ? step : 'stage') +type Data = 'records' | 'observations' + +/** What each step takes and passes on (`null`: nothing). */ +const DATA: Record = { + input: { takes: null, gives: 'records' }, + reject: { takes: 'records', gives: 'records' }, + map: { takes: 'records', gives: 'observations' }, + filter: { takes: 'observations', gives: 'observations' }, + registry: { takes: 'observations', gives: 'observations' }, + affiliation: { takes: 'observations', gives: 'observations' }, + tracker: { takes: 'observations', gives: 'observations' }, + throttle: { takes: 'observations', gives: 'observations' }, + output: { takes: 'observations', gives: null }, + nats: { takes: 'observations', gives: null }, +} -/** Transports, as the palette and the step heading name them, with what a new one starts as. */ +export const takes = (t: StepType) => DATA[t].takes +export const gives = (t: StepType) => DATA[t].gives + +/** Each step's name, and the group the palette lists it under. */ +export const STEPS: Record, { label: string; group: 'Records' | 'Observations' | 'Outputs' }> = { + reject: { label: 'Reject', group: 'Records' }, + map: { label: 'Map', group: 'Records' }, + filter: { label: 'Filter', group: 'Observations' }, + registry: { label: 'Entity links', group: 'Observations' }, + affiliation: { label: 'Affiliation', group: 'Observations' }, + tracker: { label: 'Tracker', group: 'Observations' }, + throttle: { label: 'Throttle', group: 'Observations' }, + output: { label: 'Output', group: 'Outputs' }, + nats: { label: 'NATS topic', group: 'Outputs' }, +} + +/** The word under a step's heading. */ +export const KIND_WORD: Record = { + input: 'Input', + reject: 'Records', + map: 'Map', + filter: 'Observations', + registry: 'Observations', + affiliation: 'Observations', + tracker: 'Observations', + throttle: 'Observations', + output: 'Correlation', + nats: 'NATS', +} + +/** What a new step starts as. */ +export function defaultConfig(type: StepType): unknown { + switch (type) { + case 'reject': + return [{ reason: 'no_position', when: { path: 'lat', exists: false } }] + case 'map': + return { mapping: { rules: [] } } + case 'filter': + return {} + case 'registry': + return { links: DEFAULT_LINKS } + case 'affiliation': + return { country: 'platform.flag', friendly: [], hostile: [], neutral: [] } + case 'tracker': + return { algorithm: 'gnn' } + case 'throttle': + return { min_interval_secs: 5, min_move_m: 0, heartbeat_secs: 600 } + case 'nats': + return { subject: '' } + default: + return undefined + } +} + +/** Transports, as the input's settings name them, with what a new one starts as. */ export const TRANSPORTS: Record = { tcp_client: { label: 'TCP client', initial: { type: 'tcp_client', host: '', port: 0, framing: { type: 'lines' } } }, tcp_server: { label: 'TCP server', initial: { type: 'tcp_server', bind: '0.0.0.0:8087', framing: { type: 'lines' } } }, @@ -39,7 +120,7 @@ export const TRANSPORTS: Record = file: { label: 'File', initial: { type: 'file', path: '' } }, } -/** Codecs, likewise. A plugin codec is named by its plugin. */ +/** Codecs, likewise. A plugin codec is offered by its plugin's name. */ export const CODECS: Record = { json: { label: 'JSON', initial: { type: 'json' } }, cot_xml: { label: 'CoT XML', initial: { type: 'cot_xml' } }, @@ -47,157 +128,17 @@ export const CODECS: Record = { protobuf: { label: 'Protobuf', initial: { type: 'protobuf', files: {}, message: '' } }, } -/** The stages added from the palette, with what a new one starts as. */ -const STAGES: Record, { label: string; add: (s: SourceSpec) => SourceSpec; remove: (s: SourceSpec) => SourceSpec }> = { - reject: { - label: 'Reject', - add: (s) => withPipeline(s, { mapping: { ...s.pipeline.mapping, reject: [{ reason: 'no_position', when: { path: 'lat', exists: false } }] } }), - remove: (s) => withPipeline(s, { mapping: { ...s.pipeline.mapping, reject: undefined } }), - }, - registry: { - label: 'Entity links', - add: (s) => withPipeline(s, { registry: { links: DEFAULT_LINKS } }), - remove: (s) => withPipeline(s, { registry: undefined }), - }, - affiliation: { - label: 'Affiliation', - add: (s) => withPipeline(s, { affiliation: { country: 'platform.flag', friendly: [], hostile: [], neutral: [] } }), - remove: (s) => withPipeline(s, { affiliation: undefined }), - }, - filter: { - label: 'Filter', - add: (s) => withPipeline(s, { filter: {} }), - remove: (s) => withPipeline(s, { filter: undefined }), - }, - tracker: { - label: 'Tracker', - add: (s) => withPipeline(s, { tracker: { algorithm: 'gnn' } }), - remove: (s) => withPipeline(s, { tracker: undefined }), - }, - throttle: { - label: 'Throttle', - add: (s) => withPipeline(s, { throttle: { min_interval_secs: 5, min_move_m: 0, heartbeat_secs: 600 } }), - remove: (s) => withPipeline(s, { throttle: undefined }), - }, -} - -const OUTPUTS: Record<'tracks' | 'detections', string> = { tracks: 'Tracks', detections: 'Detections' } - -/** Set pipeline keys; one set to undefined is dropped, so the spec carries only what is configured. */ -function withPipeline(spec: SourceSpec, patch: Obj): SourceSpec { - const pipeline: Obj = { ...spec.pipeline, ...patch } - for (const [k, v] of Object.entries(patch)) if (v === undefined) delete pipeline[k] - const mapping = pipeline.mapping as Obj - if (mapping && mapping.reject === undefined) delete mapping.reject - return { ...spec, pipeline: pipeline as SourceSpec['pipeline'] } -} - -const rejects = (spec: SourceSpec) => ((spec.pipeline.mapping.reject as unknown[] | undefined) ?? []).length > 0 - -/** The steps this source has, in order. */ -export function stepsOf(spec: SourceSpec): StepId[] { - const p = spec.pipeline as Obj - const has: Record = { - source: true, - decode: true, - reject: rejects(spec), - map: true, - join: (spec.pipeline.mapping.rules as { kind?: string }[]).some((r) => r?.kind === 'static'), - registry: p.registry !== undefined, - affiliation: p.affiliation !== undefined, - filter: p.filter !== undefined, - tracker: p.tracker !== undefined, - throttle: p.throttle !== undefined, - output: true, - } - return STEP_ORDER.filter((s) => has[s]) -} - -/** A step's heading: what it is for this source (its transport, codec, or output). */ -export function stepTitle(spec: SourceSpec, step: StepId): string { - switch (step) { - case 'source': - return TRANSPORTS[spec.transport.type]?.label ?? spec.transport.type - case 'decode': { - const c = spec.pipeline.codec - if (c.type === 'plugin') return String(c.plugin || 'Plugin') - return CODECS[c.type]?.label ?? c.type - } - case 'map': - return 'Map' - case 'output': - return OUTPUTS[spec.reports ?? 'tracks'] - case 'join': - return 'Static join' - default: - return STAGES[step].label - } -} - -/** The word under a step's heading. */ -export const KIND_WORD: Record = { source: 'Source', decode: 'Decode', stage: 'Stage', output: 'Output' } - -/** The trace's name for a step (crates/ot-source/src/trace.rs; `samplesAt`). */ -export const traceStage = (step: StepId): string => (step === 'source' ? 'transport' : step === 'output' ? 'publish' : step) - -/** The step a problem belongs to, from the server's pipeline stage; `null`: the source as a whole. */ -const STEP_OF_STAGE: Record = { - source: 'source', - decode: 'decode', - map: 'map', - static_join: 'join', - registry: 'registry', - affiliation: 'affiliation', - filter: 'filter', - tracker: 'tracker', - throttle: 'throttle', - output: 'output', -} - -export interface StepProblems { - byStep: Partial> - /** Problems with the source as a whole, or with a stage it does not show. */ - general: string[] -} - -/** The server's problems, each on its step. */ -export function problemsByStep(problems: SourceProblem[], steps: StepId[]): StepProblems { - const out: StepProblems = { byStep: {}, general: [] } - for (const p of problems) { - const step = p.stage ? STEP_OF_STAGE[p.stage] : null - if (step && steps.includes(step)) (out.byStep[step] ??= []).push(p.message) - else out.general.push(p.message) - } - return out -} - -/** A palette item's type: `source:`, `decode:` (`decode:plugin:`), `stage:` or `output:`. */ -export interface PaletteEntry { - type: string - label: string - group: 'Source' | 'Decode' | 'Stages' | 'Output' -} - -export function paletteEntries(plugins: PluginInfo[]): PaletteEntry[] { +/** The codecs the input can use: the built-in ones and each enabled codec plugin, as `plugin:`. */ +export function codecChoices(plugins: PluginInfo[]): { value: string; label: string }[] { return [ - ...Object.entries(TRANSPORTS).map(([t, d]) => ({ type: `source:${t}`, label: d.label, group: 'Source' as const })), - ...Object.entries(CODECS).map(([c, d]) => ({ type: `decode:${c}`, label: d.label, group: 'Decode' as const })), - ...plugins.filter((p) => p.kinds.includes('codec') && p.enabled).map((p) => ({ type: `decode:plugin:${p.name}`, label: p.name, group: 'Decode' as const })), - ...(Object.keys(STAGES) as (keyof typeof STAGES)[]).map((s) => ({ type: `stage:${s}`, label: STAGES[s].label, group: 'Stages' as const })), - ...(Object.keys(OUTPUTS) as (keyof typeof OUTPUTS)[]).map((r) => ({ type: `output:${r}`, label: OUTPUTS[r], group: 'Output' as const })), + ...Object.entries(CODECS).map(([value, c]) => ({ value, label: c.label })), + ...plugins.filter((p) => p.kinds.includes('codec') && p.enabled).map((p) => ({ value: `plugin:${p.name}`, label: p.name })), ] } -/** The step a palette item makes or replaces. */ -export function stepOfEntry(type: string): StepId | null { - const [kind, sub] = type.split(':') - if (kind === 'source' || kind === 'decode' || kind === 'output') return kind - if (kind === 'stage' && sub in STAGES) return sub as StepId - return null -} +export const codecChoice = (c: Codec) => (c.type === 'plugin' ? `plugin:${String(c.plugin ?? '')}` : c.type) const streamed = (t: Transport) => t.type === 'tcp_client' || t.type === 'tcp_server' - const LINES = { type: 'lines' } const COT_FRAMING = { type: 'end_tag', tag: '' } const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b) @@ -215,84 +156,181 @@ function framingOf(codec: Codec, plugins: PluginInfo[]): unknown { return null } -/** - * Whether adding this palette item would replace settings the source has: a different transport, - * codec or output in place of the one it has. A stage it already has is not added again. - */ -export function replaces(spec: SourceSpec, type: string): boolean { - const [kind, sub, name] = type.split(':') - if (kind === 'source') return sub !== spec.transport.type - if (kind === 'decode') { - const c = spec.pipeline.codec - return sub === 'plugin' ? c.type !== 'plugin' || c.plugin !== name : sub !== c.type - } - return false +/** The source with another transport: its defaults, framed for the codec; the risk acceptance goes with the old one. */ +export function withTransport(spec: SourceSpec, type: string): SourceSpec { + const d = TRANSPORTS[type] + if (!d || type === spec.transport.type) return spec + const next: SourceSpec = { ...spec, transport: cotFraming(structuredClone(d.initial), spec.pipeline.codec) } + delete next.unauthenticated + return next } -/** The source with this palette item added: a stage put in its place, or a transport, codec or output swapped in. */ -export function addEntry(spec: SourceSpec, type: string, plugins: PluginInfo[]): SourceSpec { - const [kind, sub, name] = type.split(':') - switch (kind) { - case 'source': { - const d = TRANSPORTS[sub] - if (!d || sub === spec.transport.type) return spec - const next: SourceSpec = { ...spec, transport: cotFraming(structuredClone(d.initial), spec.pipeline.codec) } - // The acceptance belongs to the listener it was given for. - delete next.unauthenticated - return next +/** The source with another codec (`plugin:` for a plugin's): the framing the old one set goes, the new one's comes. */ +export function withCodec(spec: SourceSpec, choice: string, plugins: PluginInfo[]): SourceSpec { + if (choice === codecChoice(spec.pipeline.codec)) return spec + let codec: Codec | null = null + let transport = spec.transport + const before = framingOf(spec.pipeline.codec, plugins) + if (before && streamed(transport) && same(transport.framing, before)) transport = { ...transport, framing: LINES } + if (choice.startsWith('plugin:')) { + const p = plugins.find((x) => x.name === choice.slice('plugin:'.length)) + if (p) { + codec = { type: 'plugin', plugin: p.name, options: structuredClone(p.default_options) } + if (p.framing && streamed(transport)) transport = { ...transport, framing: structuredClone(p.framing) } } - case 'decode': { - // The framing the old codec set goes with it. - const before = framingOf(spec.pipeline.codec, plugins) - if (before && streamed(spec.transport) && same(spec.transport.framing, before)) { - spec = { ...spec, transport: { ...spec.transport, framing: LINES } } - } - let codec: Codec | null = null - if (sub === 'plugin') { - const p = plugins.find((x) => x.name === name) - if (p && !(spec.pipeline.codec.type === 'plugin' && spec.pipeline.codec.plugin === p.name)) { - codec = { type: 'plugin', plugin: p.name, options: structuredClone(p.default_options) } - // A stream of the plugin's format is framed its way. - if (p.framing && streamed(spec.transport)) spec = { ...spec, transport: { ...spec.transport, framing: structuredClone(p.framing) } } - } - } else if (CODECS[sub] && sub !== spec.pipeline.codec.type) { - codec = structuredClone(CODECS[sub].initial) + } else if (CODECS[choice]) { + codec = structuredClone(CODECS[choice].initial) + } + if (!codec) return spec + return { ...spec, transport: cotFraming(transport, codec), pipeline: { ...spec.pipeline, codec } } +} + +/** The source's flow. */ +export const flowOf = (spec: SourceSpec): Flow => ({ + nodes: (spec.pipeline.nodes as FlowNode[] | undefined) ?? [], + edges: (spec.pipeline.edges as FlowEdge[] | undefined) ?? [], +}) + +/** The source with this flow. */ +export const withFlow = (spec: SourceSpec, flow: Flow): SourceSpec => ({ ...spec, pipeline: { ...spec.pipeline, nodes: flow.nodes, edges: flow.edges } }) + +/** A new step's id: its type, numbered when taken (`filter`, `filter-2`…). */ +export function nextNodeId(type: StepType, ids: string[]): string { + if (!ids.includes(type)) return type + let n = 2 + while (ids.includes(`${type}-${n}`)) n++ + return `${type}-${n}` +} + +/** Whether `source` may feed `target`: one input each, no loops, and the right data. */ +export function canConnect(flow: Flow, source: string, target: string): boolean { + if (source === target) return false + const from = flow.nodes.find((n) => n.id === source) + const to = flow.nodes.find((n) => n.id === target) + if (!from || !to) return false + const out = gives(from.type) + if (!out || takes(to.type) !== out) return false + if (flow.edges.some((e) => e.target === target)) return false + // No loop: `target` must not already lead to `source`. + const children = new Map() + for (const e of flow.edges) children.set(e.source, [...(children.get(e.source) ?? []), e.target]) + const stack = [target] + const seen = new Set() + while (stack.length) { + const at = stack.pop()! + if (at === source) return false + if (seen.has(at)) continue + seen.add(at) + stack.push(...(children.get(at) ?? [])) + } + return true +} + +/** Where steps go that have no place yet (a source from before flows): columns by distance from the input. */ +export const COLUMN = 300 +export const ROW = 130 + +export function laidOut(flow: Flow): Flow { + if (flow.nodes.every((n) => n.position)) return flow + const children = new Map() + for (const e of flow.edges) children.set(e.source, [...(children.get(e.source) ?? []), e.target]) + const depth = new Map([[INPUT, 0]]) + const queue = [INPUT] + while (queue.length) { + const at = queue.shift()! + for (const c of children.get(at) ?? []) { + if (!depth.has(c)) { + depth.set(c, depth.get(at)! + 1) + queue.push(c) } - if (!codec) return spec - return { ...withPipeline(spec, { codec }), transport: cotFraming(spec.transport, codec) } - } - case 'stage': { - const s = STAGES[sub as keyof typeof STAGES] - if (!s || stepsOf(spec).includes(sub as StepId)) return spec - return s.add(spec) - } - case 'output': { - if (sub !== 'tracks' && sub !== 'detections') return spec - const next: SourceSpec = { ...spec, reports: sub } - if (sub === 'tracks') delete next.reports - return next } } - return spec + const rows = new Map() + const loose = Math.max(0, ...depth.values()) + 1 + return { + ...flow, + nodes: flow.nodes.map((n) => { + if (n.position) return n + const col = depth.get(n.id) ?? loose + const row = rows.get(col) ?? 0 + rows.set(col, row + 1) + return { ...n, position: { x: col * COLUMN, y: row * ROW } } + }), + } } -/** The source without this stage. Source, Decode, Map, Static join and Output stay. */ -export function removeStep(spec: SourceSpec, step: StepId): SourceSpec { - if (!removable(step)) return spec - return STAGES[step as keyof typeof STAGES].remove(spec) +export interface NodeProblems { + byNode: Record + /** Problems with the source as a whole (its feed settings), or with a step it no longer has. */ + general: string[] } -/** Where each step sits: one column, top to bottom. */ -export const STEP_GAP = 112 -export const positionOf = (index: number) => ({ x: 0, y: index * STEP_GAP }) +/** The server's problems, each on its step. */ +export function problemsByNode(problems: SourceProblem[], flow: Flow): NodeProblems { + const ids = new Set(flow.nodes.map((n) => n.id)) + const out: NodeProblems = { byNode: {}, general: [] } + for (const p of problems) { + if (p.node && ids.has(p.node)) (out.byNode[p.node] ??= []).push(p.message) + else out.general.push(p.message) + } + return out +} -/** A new source: its transport, an empty JSON decode and mapping, reporting tracks. */ +/** A step's heading. */ +export function nodeTitle(spec: SourceSpec, n: FlowNode): string { + if (n.type === 'input') return TRANSPORTS[spec.transport.type]?.label ?? spec.transport.type + if (n.type === 'nats') return String((n.config as Obj | undefined)?.subject || 'NATS topic') + return STEPS[n.type].label +} + +/** One line of a step's settings. */ +export function nodeSummary(spec: SourceSpec, n: FlowNode): string { + const c = (n.config ?? {}) as Obj + switch (n.type) { + case 'input': { + const codec = spec.pipeline.codec + const t = spec.transport + const where = String(t.url ?? t.bind ?? t.path ?? (t.host ? `${t.host}:${t.port}` : '')) + const decode = codec.type === 'plugin' ? String(codec.plugin ?? 'plugin') : (CODECS[codec.type]?.label ?? codec.type) + return [where, decode].filter(Boolean).join(' · ') + } + case 'reject': + return plural(((n.config as unknown[]) ?? []).length, 'rule') + case 'map': + return plural((((c.mapping as Obj | undefined)?.rules as unknown[]) ?? []).length, 'rule') + case 'filter': + return c.keep_if ? 'keep only matching' : c.drop_if ? 'drop matching' : 'keeps everything' + case 'tracker': + return String(c.algorithm ?? 'gnn').toUpperCase() + case 'throttle': + return `at most every ${String(c.min_interval_secs ?? 0)} s` + case 'output': + return spec.reports === 'detections' ? 'detections' : 'tracks' + default: + return '' + } +} + +const plural = (n: number, one: string) => `${n} ${n === 1 ? one : `${one}s`}` + +/** A new source: its transport, a JSON input, an empty Map and the Output, connected. */ export function newSourceSpec(id: string, name: string, transport: string): SourceSpec { + const flow: Flow = { + nodes: [ + { id: INPUT, type: 'input', position: { x: 0, y: 0 } }, + { id: 'map', type: 'map', config: { mapping: { rules: [] } }, position: { x: COLUMN, y: 0 } }, + { id: 'output', type: 'output', position: { x: 2 * COLUMN, y: 0 } }, + ], + edges: [ + { source: INPUT, target: 'map' }, + { source: 'map', target: 'output' }, + ], + } return { id, name, transport: structuredClone(TRANSPORTS[transport]?.initial ?? TRANSPORTS.http_poll.initial), - pipeline: { codec: { type: 'json' }, mapping: { rules: [] } }, + pipeline: { codec: { type: 'json' }, nodes: flow.nodes, edges: flow.edges }, } } @@ -307,3 +345,16 @@ export function idFromName(name: string): string { } export const validId = (id: string) => /^[a-z0-9_-]{1,64}$/.test(id) + +/** The spec without where its steps are drawn: what a dry run depends on. */ +export function withoutPositions(spec: SourceSpec): SourceSpec { + const flow = flowOf(spec) + return withFlow(spec, { + ...flow, + nodes: flow.nodes.map((n) => { + const copy = { ...n } + delete copy.position + return copy + }), + }) +} diff --git a/ui/src/pages/sources/flow/stepIcons.tsx b/ui/src/pages/sources/flow/stepIcons.tsx index 1198754..2ccf174 100644 --- a/ui/src/pages/sources/flow/stepIcons.tsx +++ b/ui/src/pages/sources/flow/stepIcons.tsx @@ -4,28 +4,23 @@ import { TbApi, TbArrowsExchange, TbBan, - TbBinary, - TbBraces, TbBroadcast, TbClockPause, - TbCode, TbFile, TbFilter, TbFlag, - TbLink, TbMessages, TbNetwork, TbPlugConnected, - TbPuzzle, TbRadar, TbRoute, TbSend, TbServer, - TbTargetArrow, + TbTopologyStar, TbWorldDownload, } from 'react-icons/tb' import type { SourceSpec } from '../../../api/client' -import type { StepId } from './sourceFlow' +import type { StepType } from './sourceFlow' const TRANSPORT_ICONS: Record = { tcp_client: , @@ -39,38 +34,19 @@ const TRANSPORT_ICONS: Record = { file: , } -const CODEC_ICONS: Record = { - json: , - cot_xml: , - xml: , - protobuf: , - plugin: , -} - -const STAGE_ICONS: Partial> = { +const STEP_ICONS: Record, ReactNode> = { reject: , map: , - join: , + filter: , registry: , affiliation: , - filter: , tracker: , throttle: , output: , + nats: , } -/** A step's icon, for the source as it is. */ -export function stepIcon(spec: SourceSpec, step: StepId): ReactNode { - if (step === 'source') return TRANSPORT_ICONS[spec.transport.type] - if (step === 'decode') return CODEC_ICONS[spec.pipeline.codec.type] - return STAGE_ICONS[step] -} - -/** A palette item's icon, from its type (`source:udp`, `decode:plugin:x`, `stage:filter`, `output:tracks`). */ -export function entryIcon(type: string): ReactNode { - const [kind, sub] = type.split(':') - if (kind === 'source') return TRANSPORT_ICONS[sub] - if (kind === 'decode') return CODEC_ICONS[sub] - if (kind === 'stage') return STAGE_ICONS[sub as StepId] - return STAGE_ICONS.output +/** A step's icon: the input's is its transport's. */ +export function stepIcon(spec: SourceSpec, type: StepType): ReactNode { + return type === 'input' ? TRANSPORT_ICONS[spec.transport.type] : STEP_ICONS[type] } From 87f2b19cdf9b3321dab695382894e8eb39a2e92d Mon Sep 17 00:00:00 2001 From: Parker Hornstein Date: Wed, 21 Oct 2026 17:35:27 -0500 Subject: [PATCH 14/16] Sources: a step's settings open beside the workspace (#104) The workspace narrows rather than hides steps under the panel; the status no longer lists a raw output. Co-Authored-By: Claude Opus 5.5 --- ui/src/index.css | 21 ++++++++++--------- ui/src/pages/sources/SourceEditor.tsx | 30 ++++++++++++++------------- ui/src/pages/sources/SourceStatus.tsx | 4 +--- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/ui/src/index.css b/ui/src/index.css index 5bd5be8..d55e991 100644 --- a/ui/src/index.css +++ b/ui/src/index.css @@ -906,7 +906,7 @@ body { } /* Source editor (#104): a workspace of the source's steps, the step list on its left; a step's - settings (or the feed's) open over its right. It fills the page below the tabs. */ + settings (or the feed's) open beside it, on the right. It fills the page below the tabs. */ .source-editor { display: flex; flex-direction: column; @@ -932,23 +932,24 @@ body { } .source-workspace { - position: relative; flex: 1; min-height: 0; + display: flex; + gap: var(--space-md); +} + +.source-workspace-canvas { + flex: 1; + min-width: 0; } .source-workspace-panel { - position: absolute; - top: 0; - right: 0; - bottom: 0; - width: min(760px, 58%); + width: min(680px, 46%); min-width: 360px; - z-index: 5; display: flex; + border: 1px solid var(--color-glass-border); + border-radius: var(--radius-md); background: var(--color-bg-primary); - border-left: 1px solid var(--color-glass-border); - box-shadow: var(--shadow-deep); } .source-editor-json { diff --git a/ui/src/pages/sources/SourceEditor.tsx b/ui/src/pages/sources/SourceEditor.tsx index 7de4fb6..8db3bcc 100644 --- a/ui/src/pages/sources/SourceEditor.tsx +++ b/ui/src/pages/sources/SourceEditor.tsx @@ -443,20 +443,22 @@ export default function SourceEditor({ id, onClose, onDeleted }: { id: string; o
{view === 'flow' ? (
- +
+ +
{(node || feedOpen) && (
- {(node || feedOpen) && ( - - )} + ) : null} +
+
) : (