diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 537437e..5e04b7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -413,12 +413,13 @@ jobs: run: | .venv/bin/maturin develop --features rocksdb - # The tests ask the build which index backends it has, so no environment - # variable is needed — and cannot drift from what was actually compiled. - - name: RocksDB-backed index store tests + # Durable reactions need a persistent index, so the hermetic suite runs + # here as well as in the default build. The tests ask the build which + # index backends it has, so no environment variable is needed. + - name: Hermetic tests with RocksDB enabled run: | .venv/bin/python -c "import drasi; assert 'rocksdb' in drasi.host_info()['index_backends']" - .venv/bin/pytest tests/e2e/test_stores.py -v + .venv/bin/pytest -m "not plugins and not oci and not docker" -v coverage: name: Conversion and error coverage diff --git a/CHANGELOG.md b/CHANGELOG.md index f40c22d..b91717c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,35 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ## [Unreleased] +### Changed + +- Built against drasi-lib 0.9.3, drasi-core 0.5.10, drasi-host-sdk 0.11.3 and + drasi-plugin-sdk 0.11.3 (previously 0.8.9, 0.5.7, 0.10.0 and 0.10.0). The + store crates moved with them: drasi-state-store-redb 0.2.8 and + drasi-index-rocksdb 0.6.4. Published plugins must match on `major.minor`; + the current `ghcr.io/drasi-project` releases do. `host_info()` now reports + FFI ABI 0.14.0. + +- Reactions registered before `start()` now start after their queries. drasi-lib + 0.9 will not subscribe to a query that is still stopped, and the query + auto-start workaround was starting reactions first. + +- A durable reaction now requires a persistent index store as well as a state + store. drasi-lib 0.9 rejects a durable subscription to an in-memory query, + because the outbox the checkpoint points at would not survive a restart. + Pass `index_store={"kind": "rocksdb", "path": ...}` to `Drasi.create`. + Missing it raises `DURABLE_REQUIRES_INDEX_STORE`. + +- A failing reaction callback follows the recovery policy. `strict` and + `auto_reset` stop the reaction and leave the checkpoint where it was, so the + event is replayed the next time the reaction starts. `skip_gap` records the + event as handled and continues. Previously the loop logged the failure and + carried on without advancing the checkpoint. + +- Plugin reactions report their own recovery policy and bootstrap + implementation. The wrapper had been answering with the trait defaults, so a + plugin's policy never reached startup recovery. + ### Documentation - How to declare a synthetic join. The concepts page said a query can span diff --git a/Cargo.lock b/Cargo.lock index 90d6010..1b62c99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -191,17 +191,16 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.65.1" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.13.1", "cexpr", "clang-sys", + "itertools 0.12.1", "lazy_static", "lazycell", - "peeking_take_while", - "prettyplease", "proc-macro2", "quote", "regex", @@ -673,9 +672,9 @@ dependencies = [ [[package]] name = "drasi-core" -version = "0.5.7" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fae0da1b998ef09ab1115cc5b8c09b4d82305d8d432961c869db700ef69fbc19" +checksum = "9e06737256dbabee9ff58a5a74ed267bf41f20437d0e9397755a3a8c73de5d63" dependencies = [ "approx", "async-recursion", @@ -715,33 +714,33 @@ dependencies = [ [[package]] name = "drasi-ffi-primitives" -version = "0.10.0" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34fcfb2ad8c4d0140eda6dfaabaf5fad996923f9bdb7cd6e6a4d44e3a1b28088" +checksum = "5104f2bd052d320f9f062ab24cc10e4bcabbf6ceb444e111fc07cdf62babc577" [[package]] name = "drasi-functions-cypher" -version = "0.5.7" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "839da21be5fc2115fa11599f72df6e37cec89391b35d2a0e6defc42098e205a3" +checksum = "d59d003512a6cf59fe510a7991dd05bf6edd114e0a5aafba9d085bc82f8d2415" dependencies = [ "drasi-core", ] [[package]] name = "drasi-functions-gql" -version = "0.5.7" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33fa471da6c513979dabc7ddb7c2b69df75d07fa7e6856c0afb9ce57a87dee76" +checksum = "397da52359c1f97cc002c812f259c53a4699e41d093e1865192ecbcbc7cd3d7d" dependencies = [ "drasi-core", ] [[package]] name = "drasi-host-sdk" -version = "0.10.0" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4a628bf170705d0e8b3f6aa769e20ec82e3a2f223f4d5dc3f24dcbe5eacb85" +checksum = "f8b3640eab33216fa1f78c709951126dd81e2c96df2b29c71a324ef201d11090" dependencies = [ "anyhow", "async-trait", @@ -768,14 +767,15 @@ dependencies = [ "tokio", "tokio-stream", "toml", + "tracing", "x509-parser", ] [[package]] name = "drasi-index-rocksdb" -version = "0.5.8" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac63be55f0699830a14ef87b06196d7dd67ec608f84bccafb5fc965a90858c35" +checksum = "a9d02d07abeb268d6d45fb10550a27c0c846a1e290dc204ef869479250a6b047" dependencies = [ "anyhow", "async-stream", @@ -803,9 +803,9 @@ dependencies = [ [[package]] name = "drasi-lib" -version = "0.8.9" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "863f443ed29d748b3ddf8a65f6cdbcd752e59bf74f9db467cd817bee35896621" +checksum = "90757ef1daf6f96da9d350d263b32942ccf0ee197f289f4673f558485ec88d4d" dependencies = [ "anyhow", "async-trait", @@ -841,9 +841,9 @@ dependencies = [ [[package]] name = "drasi-middleware" -version = "0.5.8" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2f0f50afaee2d7a562f7cc736832a48e93c2daf3d22db59ebd793eec5bfb257" +checksum = "3fcb1c840aac9d83a5bf3d8c3dea5de283ed5fadb3a830d49862193f3b4d9c7a" dependencies = [ "async-trait", "base64", @@ -859,9 +859,9 @@ dependencies = [ [[package]] name = "drasi-plugin-sdk" -version = "0.10.0" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3026bcee8fbf28618480509625183ddaa71c13ad5411d64836e37a15213b49e" +checksum = "4f540dfeea3d9443c27f9a56da3627772bc16b52021d968d46223438de27cda5" dependencies = [ "anyhow", "async-stream", @@ -944,9 +944,9 @@ dependencies = [ [[package]] name = "drasi-state-store-redb" -version = "0.2.4" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40014f64c97cd42edd5260deea70caff8dbc87798b0f5d32348e8bc4fe9237e2" +checksum = "8d8570f07e3076d5c04880e21ca2b1374db6b4407562188b985ad49410f9bda4" dependencies = [ "async-trait", "drasi-lib", @@ -1802,11 +1802,11 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "0.11.0+8.1.1" +version = "0.16.0+8.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e" +checksum = "ce3d60bc059831dc1c83903fb45c103f75db65c5a7bf22272764d9cc683e348c" dependencies = [ - "bindgen 0.65.1", + "bindgen 0.69.5", "bzip2-sys", "cc", "glob", @@ -2194,12 +2194,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "peg" version = "0.8.6" @@ -2374,16 +2368,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.119", -] - [[package]] name = "primeorder" version = "0.13.6" @@ -2914,9 +2898,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe" +checksum = "6bd13e55d6d7b8cd0ea569161127567cd587676c99f4472f779a0279aa60a7a7" dependencies = [ "libc", "librocksdb-sys", diff --git a/Cargo.toml b/Cargo.toml index 80f5a0e..e2bef57 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] } # ghcr.io/drasi-project are annotated with the drasi-core / drasi-lib / # drasi-plugin-sdk versions they were built against, and the host rejects any # plugin whose versions do not match ours on major.minor. See docs/plugins.md. -drasi-lib = { version = "=0.8.9", features = [ +drasi-lib = { version = "=0.9.3", features = [ # Query middleware. `jq` is deliberately excluded: it pulls in `jq-rs`, # which links against libjq, and a C dependency would put the # cross-compiled aarch64 and Windows wheels at risk. The rest have no @@ -37,11 +37,11 @@ drasi-lib = { version = "=0.8.9", features = [ "middleware-relabel", "middleware-unwind", ] } -drasi-core = "=0.5.7" -drasi-host-sdk = { version = "=0.10.0", features = ["watcher", "registry"] } -drasi-plugin-sdk = "=0.10.0" -drasi-state-store-redb = "=0.2.4" -drasi-index-rocksdb = { version = "=0.5.8", optional = true, features = ["plugin-descriptor"] } +drasi-core = "=0.5.10" +drasi-host-sdk = { version = "=0.11.3", features = ["watcher", "registry"] } +drasi-plugin-sdk = "=0.11.3" +drasi-state-store-redb = "=0.2.8" +drasi-index-rocksdb = { version = "=0.6.4", optional = true, features = ["plugin-descriptor"] } anyhow = "1" async-trait = "0.1" diff --git a/README.md b/README.md index 34f5728..9045066 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,11 @@ A durable reaction only advances its checkpoint once your callback succeeds, so an unhandled event is replayed after a restart: ```python -drasi = await Drasi.create("app", state_store={"kind": "redb", "path": "state.redb"}) +drasi = await Drasi.create( + "app", + state_store={"kind": "redb", "path": "state.redb"}, + index_store={"kind": "rocksdb", "path": "index"}, +) async def handle(event): diff --git a/docs/plugins.md b/docs/plugins.md index 33be271..a87881e 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -11,9 +11,9 @@ Plugins are published as OCI artifacts to `ghcr.io/drasi-project`: ``` ghcr.io/drasi-project/{type}/{kind}:{version}-{arch} -ghcr.io/drasi-project/source/postgres:0.1.13-linux-amd64 -ghcr.io/drasi-project/source/mock:0.2.7-darwin-arm64 -ghcr.io/drasi-project/reaction/log:0.2.5-linux-arm64 +ghcr.io/drasi-project/source/postgres:0.2.12-linux-amd64 +ghcr.io/drasi-project/source/mock:0.2.12-darwin-arm64 +ghcr.io/drasi-project/reaction/log:0.2.9-linux-arm64 ``` Short references are expanded against the default registry, so `source/postgres` @@ -49,10 +49,10 @@ Inspect what this host offers: >>> import drasi >>> drasi.host_info() {'arch_suffix': 'darwin-arm64', - 'core_version': '0.5.7', - 'ffi_sdk_version': '0.11.0', - 'lib_version': '0.8.9', - 'sdk_version': '0.10.0', + 'core_version': '0.5.10', + 'ffi_sdk_version': '0.14.0', + 'lib_version': '0.9.3', + 'sdk_version': '0.11.3', 'target_triple': 'aarch64-apple-darwin'} ``` diff --git a/examples/README.md b/examples/README.md index 32674e4..8ca4daf 100644 --- a/examples/README.md +++ b/examples/README.md @@ -48,9 +48,9 @@ Check it worked: ``` ``` -{'arch_suffix': 'darwin-arm64', 'core_version': '0.5.7', - 'ffi_sdk_version': '0.11.0', 'lib_version': '0.8.9', - 'sdk_version': '0.10.0', 'target_triple': 'aarch64-apple-darwin'} +{'arch_suffix': 'darwin-arm64', 'core_version': '0.5.10', + 'ffi_sdk_version': '0.14.0', 'lib_version': '0.9.3', + 'sdk_version': '0.11.3', 'target_triple': 'aarch64-apple-darwin'} ``` (Your `target_triple` and `arch_suffix` will reflect your own machine.) @@ -146,13 +146,13 @@ matches your machine, so you never deal with architecture tags. ``` host: aarch64-apple-darwin - drasi-core 0.5.7, drasi-lib 0.8.9 - plugin sdk 0.10.0, ffi abi 0.11.0 + drasi-core 0.5.10, drasi-lib 0.9.3 + plugin sdk 0.11.3, ffi abi 0.14.0 56 plugins published; sources include: cloudflare-radar, dataverse, grpc, gtfs-rt, here-traffic, http, ... -source/mock resolves to 0.2.7 for aarch64-apple-darwin +source/mock resolves to 0.2.12 for aarch64-apple-darwin installed to /tmp/drasi-python-plugins/plugin-demo/libdrasi_source_mock.dylib signature: unsigned diff --git a/scripts/build_plugins.py b/scripts/build_plugins.py index 26912f5..4eb2b87 100644 --- a/scripts/build_plugins.py +++ b/scripts/build_plugins.py @@ -44,8 +44,8 @@ # (crate, version, cargo lib name) PLUGINS = [ - ("drasi-source-mock", "0.2.7", "drasi_source_mock"), - ("drasi-reaction-log", "0.2.5", "drasi_reaction_log"), + ("drasi-source-mock", "0.2.12", "drasi_source_mock"), + ("drasi-reaction-log", "0.2.9", "drasi_reaction_log"), ] diff --git a/src/components.rs b/src/components.rs index b534f4f..69e4809 100644 --- a/src/components.rs +++ b/src/components.rs @@ -284,6 +284,14 @@ impl Reaction for BoxedReaction { self.0.needs_snapshot_on_fresh_start() } + fn default_recovery_policy(&self) -> ReactionRecoveryPolicy { + self.0.default_recovery_policy() + } + + async fn bootstrap(&self, ctx: drasi_lib::reactions::BootstrapContext) -> Result<()> { + self.0.bootstrap(ctx).await + } + async fn initialize(&self, context: ReactionRuntimeContext) { self.0.initialize(context).await; } @@ -329,6 +337,11 @@ impl StreamingReaction { sender: std::sync::Mutex::new(Some(sender)), } } + + pub fn with_auto_start(mut self, auto_start: bool) -> Self { + self.base.auto_start = auto_start; + self + } } #[async_trait] @@ -367,9 +380,12 @@ impl Reaction for StreamingReaction { return Ok(()); }; + let policy = self + .base + .resolved_recovery_policy(ReactionRecoveryPolicy::Strict); let task = tokio::spawn(async move { let result = base - .run_standard_loop(shutdown_rx, checkpoints, move |event| { + .run_standard_loop(shutdown_rx, checkpoints, policy, move |event| { let sender = sender.clone(); async move { let value = serde_json::to_value(&*event)?; @@ -443,6 +459,11 @@ impl PythonReaction { locals: Some(locals), } } + + pub fn with_auto_start(mut self, auto_start: bool) -> Self { + self.base.auto_start = auto_start; + self + } } /// Invokes the Python callable with the result rendered as a plain dict. @@ -516,6 +537,17 @@ impl Reaction for PythonReaction { self.durable } + fn default_recovery_policy(&self) -> ReactionRecoveryPolicy { + self.base + .resolved_recovery_policy(ReactionRecoveryPolicy::Strict) + } + + /// `auto_reset` re-bootstraps from a snapshot. The other policies do not, + /// and drasi-lib rejects the contradictory pairings. + fn needs_snapshot_on_fresh_start(&self) -> bool { + self.default_recovery_policy() == ReactionRecoveryPolicy::AutoReset + } + async fn initialize(&self, context: ReactionRuntimeContext) { self.base.initialize(context).await; } @@ -527,9 +559,10 @@ impl Reaction for PythonReaction { let callback = Arc::clone(&self.callback); let locals = self.locals.clone(); + let policy = self.default_recovery_policy(); let task = tokio::spawn(async move { let result = base - .run_standard_loop(shutdown_rx, checkpoints, move |event| { + .run_standard_loop(shutdown_rx, checkpoints, policy, move |event| { let callback = Arc::clone(&callback); let locals = locals.clone(); async move { diff --git a/src/engine.rs b/src/engine.rs index 9a60ef9..35ab94b 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -54,6 +54,9 @@ pub struct Inner { /// Whether a durable state store was configured. Durable reactions need /// one to persist their checkpoints across restarts. durable_capable: bool, + /// Whether a persistent index store was configured. Without one every + /// query is volatile, and a durable reaction cannot subscribe to it. + persistent_indexes: bool, /// Set by `close`. The engine refuses `start` once shut down, but would /// otherwise still accept components that could never run. closed: AtomicBool, @@ -62,11 +65,16 @@ pub struct Inner { #[allow(dead_code)] identity_host: Option>, /// Queries registered while the engine was stopped, whose auto-start was - /// suppressed. `drasi-lib` 0.8.9 starts an auto-start query the moment it - /// is added, without the `is_running()` guard that `add_source` and + /// suppressed. `drasi-lib` starts an auto-start query the moment it is + /// added, without the `is_running()` guard that `add_source` and /// `add_reaction` both apply, so `start()` would then start it a second /// time. See `add_query`. deferred_queries: Mutex>, + /// Reactions registered while the engine was stopped, whose auto-start + /// was suppressed. `DrasiLib::start` starts reactions before this binding + /// starts deferred queries, and drasi-lib 0.9 will not subscribe to a + /// query that is still stopped. See `start`. + deferred_reactions: Mutex>, } impl Inner { @@ -84,7 +92,7 @@ impl Inner { Ok(()) } - /// Registers a query, suppressing the premature start in `drasi-lib` 0.8.9. + /// Registers a query, suppressing the premature start in `drasi-lib`. /// /// `add_query` there starts an auto-start query immediately even when the /// engine is stopped, unlike `add_source` and `add_reaction`, which both @@ -186,6 +194,52 @@ impl Inner { self.deferred_queries.lock().await.retain(|held| held != id); } + /// Whether an auto-start reaction has to wait until deferred queries run. + /// + /// `DrasiLib::start` starts reactions before this binding starts the + /// queries whose auto-start `register_query` suppressed. drasi-lib 0.9 + /// refuses to subscribe to a query that is still stopped, so those + /// reactions fail at startup. Register them with auto-start off and start + /// them from `start()` once the queries are running. + async fn hold_reaction_until_queries(&self, auto_start: bool) -> bool { + auto_start && !self.core.is_running().await + } + + async fn note_deferred_reaction(&self, id: &str, defer: bool) { + let mut deferred = self.deferred_reactions.lock().await; + match (defer, deferred.iter().any(|held| held == id)) { + (true, false) => deferred.push(id.to_string()), + (false, true) => deferred.retain(|held| held != id), + _ => {} + } + } + + async fn forget_deferred_reaction(&self, id: &str) { + self.deferred_reactions + .lock() + .await + .retain(|held| held != id); + } + + /// Starts the reactions whose auto-start was suppressed so they would not + /// subscribe before deferred queries were running. + /// + /// Kept across restarts: their stored config says auto-start is off, so + /// `start_all` would skip them on every later `start()`. + async fn start_deferred_reactions(&self) -> PyResult<()> { + let ids = self.deferred_reactions.lock().await.clone(); + for id in ids { + let already_running = matches!( + self.core.get_reaction_status(&id).await, + Ok(ComponentStatus::Running) + ); + if !already_running { + self.core.start_reaction(&id).await.map_err(engine_error)?; + } + } + Ok(()) + } + fn next_stream_id(&self) -> u64 { self.stream_counter.fetch_add(1, Ordering::Relaxed) } @@ -258,6 +312,7 @@ impl Drasi { // Parse eagerly so a malformed option raises before the caller awaits. let options = CreateOptions::parse(secrets, state_store, index_store, identity)?; let durable_capable = options.has_state_store(); + let persistent_indexes = options.has_index_store(); let wanted_identity = options.identity_plugin(); // The identity plugin's own configuration may reference a secret, so the // headless host gets the same mapping the engine will use. @@ -310,8 +365,10 @@ impl Drasi { default_plugin_dir: Mutex::new(None), stream_counter: AtomicU64::new(0), durable_capable, + persistent_indexes, closed: AtomicBool::new(false), deferred_queries: Mutex::new(Vec::new()), + deferred_reactions: Mutex::new(Vec::new()), identity_host, }), }; @@ -376,6 +433,7 @@ impl Drasi { .as_ref(), )?; let durable_capable = options.has_state_store(); + let persistent_indexes = options.has_index_store(); // Everything is parsed before the engine exists, so a malformed // configuration fails without leaving a half-built engine behind. @@ -402,8 +460,10 @@ impl Drasi { default_plugin_dir: Mutex::new(None), stream_counter: AtomicU64::new(0), durable_capable, + persistent_indexes, closed: AtomicBool::new(false), deferred_queries: Mutex::new(Vec::new()), + deferred_reactions: Mutex::new(Vec::new()), identity_host: None, }), }; @@ -454,12 +514,13 @@ impl Drasi { &reaction.kind, ) })?; + let hold = inner.hold_reaction_until_queries(reaction.auto_start).await; let created = descriptor .create_reaction( &reaction.id, reaction.queries.clone(), &reaction.config, - reaction.auto_start, + reaction.auto_start && !hold, ) .await .map_err(engine_error)?; @@ -468,6 +529,7 @@ impl Drasi { .add_reaction(BoxedReaction(created)) .await .map_err(engine_error)?; + inner.note_deferred_reaction(&reaction.id, hold).await; } Ok(drasi) @@ -485,7 +547,8 @@ impl Drasi { let inner = self.inner(); future_into_py(py, async move { inner.core.start().await.map_err(engine_error)?; - inner.start_deferred_queries().await + inner.start_deferred_queries().await?; + inner.start_deferred_reactions().await }) } @@ -892,11 +955,16 @@ impl Drasi { let id = reaction_id .unwrap_or_else(|| format!("__stream_{query_id}_{}", inner.next_stream_id())); let (receiver, sender) = streams::channel(); + let hold = inner.hold_reaction_until_queries(true).await; inner .core - .add_reaction(StreamingReaction::new(&id, vec![query_id.clone()], sender)) + .add_reaction( + StreamingReaction::new(&id, vec![query_id.clone()], sender) + .with_auto_start(!hold), + ) .await .map_err(engine_error)?; + inner.note_deferred_reaction(&id, hold).await; Ok(Stream::new( receiver, format!("results of query '{query_id}'"), @@ -1182,11 +1250,16 @@ impl Drasi { let reaction_id = format!("__stream_{query_id}_{}", inner.next_stream_id()); let description = format!("results of query '{query_id}'"); let (rx, sender) = streams::channel(); + let hold = inner.hold_reaction_until_queries(true).await; inner .core - .add_reaction(StreamingReaction::new(&reaction_id, vec![query_id], sender)) + .add_reaction( + StreamingReaction::new(&reaction_id, vec![query_id], sender) + .with_auto_start(!hold), + ) .await .map_err(engine_error)?; + inner.note_deferred_reaction(&reaction_id, hold).await; streams::pump_callback(rx, callback, description); Ok(()) }) @@ -1698,8 +1771,9 @@ impl Drasi { /// /// Unlike `add_python_reaction`, this waits for the coroutine to finish. If /// it raises, the checkpoint is left where it was, so the event is replayed - /// after a restart rather than lost. That guarantee needs somewhere durable - /// to keep the checkpoint, so a `state_store` is required. + /// after a restart rather than lost. That guarantee needs a `state_store` + /// for the checkpoint and an `index_store` so the subscribed queries are + /// not volatile. #[pyo3(signature = (id, query_ids, callback, *, recovery_policy = "strict"))] fn add_durable_python_reaction<'py>( &self, @@ -1718,6 +1792,13 @@ impl Drasi { pass state_store={'kind': 'redb', 'path': ...} to Drasi.create", )); } + if !self.inner.persistent_indexes { + return Err(error( + DrasiErrorCode::DurableRequiresIndexStore, + "a durable reaction cannot subscribe to an in-memory query; \ + pass index_store={'kind': 'rocksdb', 'path': ...} to Drasi.create", + )); + } require_callable(py, &callback)?; require_coroutine_function(py, &callback)?; let recovery = parse_recovery_policy(recovery_policy)?; @@ -1726,13 +1807,17 @@ impl Drasi { let inner = self.inner(); inner.ensure_open()?; future_into_py(py, async move { + let hold = inner.hold_reaction_until_queries(true).await; inner .core - .add_reaction(PythonReaction::durable( - &id, query_ids, callback, recovery, locals, - )) + .add_reaction( + PythonReaction::durable(&id, query_ids, callback, recovery, locals) + .with_auto_start(!hold), + ) .await - .map_err(engine_error) + .map_err(engine_error)?; + inner.note_deferred_reaction(&id, hold).await; + Ok(()) }) } @@ -1826,8 +1911,9 @@ impl Drasi { .ok_or_else(|| { unknown_kind(DrasiErrorCode::UnknownReactionKind, "reaction", &kind) })?; + let hold = inner.hold_reaction_until_queries(auto_start).await; let reaction = descriptor - .create_reaction(&id, query_ids, &config, auto_start) + .create_reaction(&id, query_ids, &config, auto_start && !hold) .await .map_err(engine_error)?; inner @@ -1837,7 +1923,9 @@ impl Drasi { HashMap::from([("pluginKind".to_string(), kind)]), ) .await - .map_err(engine_error) + .map_err(engine_error)?; + inner.note_deferred_reaction(&id, hold).await; + Ok(()) }) } @@ -1960,15 +2048,18 @@ impl Drasi { .ok_or_else(|| { unknown_kind(DrasiErrorCode::UnknownReactionKind, "reaction", &kind) })?; + let hold = inner.hold_reaction_until_queries(auto_start).await; let reaction = descriptor - .create_reaction(&id, query_ids, &config, auto_start) + .create_reaction(&id, query_ids, &config, auto_start && !hold) .await .map_err(engine_error)?; inner .core .update_reaction(&id, BoxedReaction(reaction)) .await - .map_err(engine_error) + .map_err(engine_error)?; + inner.note_deferred_reaction(&id, hold).await; + Ok(()) }) } @@ -1986,7 +2077,9 @@ impl Drasi { .core .remove_reaction(&id, cleanup) .await - .map_err(engine_error) + .map_err(engine_error)?; + inner.forget_deferred_reaction(&id).await; + Ok(()) }) } @@ -2097,11 +2190,14 @@ impl Drasi { let inner = self.inner(); inner.ensure_open()?; future_into_py(py, async move { + let hold = inner.hold_reaction_until_queries(true).await; inner .core - .add_reaction(PythonReaction::new(&id, query_ids, callback)) + .add_reaction(PythonReaction::new(&id, query_ids, callback).with_auto_start(!hold)) .await - .map_err(engine_error) + .map_err(engine_error)?; + inner.note_deferred_reaction(&id, hold).await; + Ok(()) }) } } diff --git a/src/errors.rs b/src/errors.rs index 3527dd0..ef8ee4e 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -56,6 +56,7 @@ pub enum DrasiErrorCode { UnknownIdentityKind, IdentityConfigInvalid, DurableRequiresStateStore, + DurableRequiresIndexStore, UnknownQueryLanguage, ConfigInvalid, PluginSignatureInvalid, @@ -88,6 +89,7 @@ impl DrasiErrorCode { Self::UnknownIdentityKind => "UNKNOWN_IDENTITY_KIND", Self::IdentityConfigInvalid => "IDENTITY_CONFIG_INVALID", Self::DurableRequiresStateStore => "DURABLE_REQUIRES_STATE_STORE", + Self::DurableRequiresIndexStore => "DURABLE_REQUIRES_INDEX_STORE", Self::UnknownQueryLanguage => "UNKNOWN_QUERY_LANGUAGE", Self::ConfigInvalid => "CONFIG_INVALID", Self::PluginSignatureInvalid => "PLUGIN_SIGNATURE_INVALID", @@ -122,6 +124,7 @@ impl DrasiErrorCode { UnknownIdentityKind, IdentityConfigInvalid, DurableRequiresStateStore, + DurableRequiresIndexStore, UnknownQueryLanguage, ConfigInvalid, PluginSignatureInvalid, @@ -153,6 +156,7 @@ impl DrasiErrorCode { | IdentityKindRequired | IdentityConfigInvalid | DurableRequiresStateStore + | DurableRequiresIndexStore | ConfigInvalid => py.get_type::().into(), NoPySource diff --git a/src/lib.rs b/src/lib.rs index 675c33c..474ead1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,9 +37,9 @@ use pyo3::prelude::*; /// Published plugins are annotated with the versions they were built against, /// and the host only accepts a plugin whose versions match these on /// `major.minor`. -pub const DRASI_CORE_VERSION: &str = "0.5.7"; -pub const DRASI_LIB_VERSION: &str = "0.8.9"; -pub const DRASI_SDK_VERSION: &str = "0.10.0"; +pub const DRASI_CORE_VERSION: &str = "0.5.10"; +pub const DRASI_LIB_VERSION: &str = "0.9.3"; +pub const DRASI_SDK_VERSION: &str = "0.11.3"; #[pymodule] fn _drasi(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/src/stores.rs b/src/stores.rs index 3e5d023..d250610 100644 --- a/src/stores.rs +++ b/src/stores.rs @@ -105,6 +105,14 @@ impl CreateOptions { self.state_store.is_some() } + /// Whether a persistent index store was configured. + /// + /// Queries otherwise keep their outbox in memory. A durable reaction cannot + /// resume from that after a restart, so drasi-lib rejects the subscription. + pub fn has_index_store(&self) -> bool { + self.index_store.is_some() + } + /// Applies the options to a builder, returning the secrets for the plugin /// config resolver. /// diff --git a/tests/e2e/test_concurrency.py b/tests/e2e/test_concurrency.py index ae73b8c..b998306 100644 --- a/tests/e2e/test_concurrency.py +++ b/tests/e2e/test_concurrency.py @@ -19,6 +19,8 @@ import asyncio from typing import Any +import pytest + from drasi import Drasi, Stream from drasi.types import QueryResultEvent, SourceChange @@ -119,8 +121,14 @@ async def test_simultaneous_streams_on_one_query_each_receive_the_event(engine: async def test_a_slow_durable_reaction_does_not_block_query_updates( tmp_path: Any, ) -> None: + from drasi import host_info + + if "rocksdb" not in host_info()["index_backends"]: + pytest.skip("built without the rocksdb feature") drasi = await Drasi.create( - "t-slow-reaction", state_store={"kind": "redb", "path": str(tmp_path / "state.redb")} + "t-slow-reaction", + state_store={"kind": "redb", "path": str(tmp_path / "state.redb")}, + index_store={"kind": "rocksdb", "path": str(tmp_path / "index")}, ) try: await running_orders_engine(drasi) diff --git a/tests/e2e/test_durable_and_config.py b/tests/e2e/test_durable_and_config.py index fc1c106..820dcd9 100644 --- a/tests/e2e/test_durable_and_config.py +++ b/tests/e2e/test_durable_and_config.py @@ -22,7 +22,7 @@ import pytest -from drasi import ConfigError, Drasi, DrasiError, UnknownKindError +from drasi import ConfigError, Drasi, DrasiError, UnknownKindError, host_info from drasi.types import QueryResultEvent, SourceChange from .helpers import wait_for, wait_for_at_least_rows, wait_for_query_running @@ -47,9 +47,17 @@ def plugin_dir() -> Path: return directory +def require_persistent_index() -> None: + if "rocksdb" not in host_info()["index_backends"]: + pytest.skip("built without the rocksdb feature") + + async def durable_engine(tmp_path: Path, name: str = "durable") -> Drasi: + require_persistent_index() engine = await Drasi.create( - name, state_store={"kind": "redb", "path": str(tmp_path / "state.redb")} + name, + state_store={"kind": "redb", "path": str(tmp_path / "state.redb")}, + index_store={"kind": "rocksdb", "path": str(tmp_path / "index")}, ) await engine.start() await engine.add_python_source("orders") @@ -68,6 +76,19 @@ async def test_a_durable_reaction_needs_a_state_store(engine: Drasi) -> None: assert caught.value.code == "DURABLE_REQUIRES_STATE_STORE" +async def test_a_durable_reaction_needs_a_persistent_index(tmp_path: Path) -> None: + """An in-memory query cannot back a checkpoint that has to survive a restart.""" + drasi = await Drasi.create( + "durable-volatile", state_store={"kind": "redb", "path": str(tmp_path / "state.redb")} + ) + try: + with pytest.raises(ConfigError) as caught: + await drasi.add_durable_python_reaction("r", ["q"], lambda _: None) # pyright: ignore[reportArgumentType] # invalid on purpose + assert caught.value.code == "DURABLE_REQUIRES_INDEX_STORE" + finally: + await drasi.close() + + async def test_a_durable_reaction_awaits_its_callback(tmp_path: Path) -> None: drasi = await durable_engine(tmp_path) try: @@ -264,13 +285,16 @@ async def test_from_config_accepts_the_same_store_options(tmp_path: Path) -> Non } ) try: - # A durable reaction proves the state store was actually applied. + # Getting past the state-store check proves it was applied. A persistent + # index is a separate requirement, and this config does not set one. async def handler(_: QueryResultEvent) -> None: return None await drasi.add_python_source("s") await drasi.add_query("q", ORDERS_QUERY, ["s"]) - await drasi.add_durable_python_reaction("r", ["q"], handler) + with pytest.raises(ConfigError) as caught: + await drasi.add_durable_python_reaction("r", ["q"], handler) + assert caught.value.code == "DURABLE_REQUIRES_INDEX_STORE" finally: await drasi.close() diff --git a/tests/e2e/test_lifecycle.py b/tests/e2e/test_lifecycle.py index 4f67c23..2b2634d 100644 --- a/tests/e2e/test_lifecycle.py +++ b/tests/e2e/test_lifecycle.py @@ -84,7 +84,7 @@ async def test_an_engine_can_process_changes_after_stop_then_start() -> None: async def test_a_query_added_before_start_runs_exactly_once() -> None: """Registering a query before `start()` must leave it running, not in error. - `drasi-lib` 0.8.9 starts an auto-start query the moment it is added, with + `drasi-lib` starts an auto-start query the moment it is added, with no `is_running()` guard (`add_source` and `add_reaction` both have one), so `start()` would start it a second time. That left the query reporting `Error` ("already running") while it was in fact running, and tripped an diff --git a/tests/unit/test_registry_pins.py b/tests/unit/test_registry_pins.py index bd92b0c..213fdb3 100644 --- a/tests/unit/test_registry_pins.py +++ b/tests/unit/test_registry_pins.py @@ -35,9 +35,9 @@ def test_reads_exact_pins_from_cargo_toml() -> None: pins = pins_module.cargo_pins(ROOT / "Cargo.toml") assert pins == { - "drasi-core": "0.5.7", - "drasi-lib": "0.8.9", - "drasi-plugin-sdk": "0.10.0", + "drasi-core": "0.5.10", + "drasi-lib": "0.9.3", + "drasi-plugin-sdk": "0.11.3", } diff --git a/website/content/docs/concepts/_index.md b/website/content/docs/concepts/_index.md index 87be22d..63f9737 100644 --- a/website/content/docs/concepts/_index.md +++ b/website/content/docs/concepts/_index.md @@ -152,7 +152,9 @@ engine: that feature. `host_info()["index_backends"]` reports which are available. A durable reaction records how far it got, so after a restart it resumes rather than -replaying from the beginning or silently skipping what it missed. +replaying from the beginning or silently skipping what it missed. That only works if +the queries it watches are persistent too, so it needs both stores: `state_store` for +the checkpoint and `index_store` for the query outbox. ## Next diff --git a/website/content/docs/guides/error-handling.md b/website/content/docs/guides/error-handling.md index 187f913..e6f0571 100644 --- a/website/content/docs/guides/error-handling.md +++ b/website/content/docs/guides/error-handling.md @@ -77,6 +77,7 @@ The full set is available at runtime as `drasi.ERROR_CODES`. | `UNKNOWN_IDENTITY_KIND` | No identity provider matches that kind | | `IDENTITY_CONFIG_INVALID` | The identity block is missing fields its kind needs | | `DURABLE_REQUIRES_STATE_STORE` | A durable reaction was added without a state store | +| `DURABLE_REQUIRES_INDEX_STORE` | A durable reaction was added without a persistent index store | | `UNKNOWN_QUERY_LANGUAGE` | The language was not `cypher` or `gql` | | `CONFIG_INVALID` | The configuration was malformed | | `PLUGIN_SIGNATURE_INVALID` | Signature verification failed | diff --git a/website/content/docs/guides/plugins.md b/website/content/docs/guides/plugins.md index 0f94e3e..b2b8d85 100644 --- a/website/content/docs/guides/plugins.md +++ b/website/content/docs/guides/plugins.md @@ -59,7 +59,7 @@ Plugins are published as OCI artifacts: ```text ghcr.io/drasi-project/{type}/{kind}:{version}-{arch} -ghcr.io/drasi-project/source/postgres:0.2.7-linux-amd64 +ghcr.io/drasi-project/source/postgres:0.2.12-linux-amd64 ghcr.io/drasi-project/reaction/dashboard:0.1.3-darwin-arm64 ``` @@ -86,11 +86,11 @@ Inspect what this host offers: >>> import drasi >>> drasi.host_info() {'arch_suffix': 'darwin-arm64', - 'core_version': '0.5.7', - 'ffi_sdk_version': '0.11.0', + 'core_version': '0.5.10', + 'ffi_sdk_version': '0.14.0', 'index_backends': ['memory', 'rocksdb'], - 'lib_version': '0.8.9', - 'sdk_version': '0.10.0', + 'lib_version': '0.9.3', + 'sdk_version': '0.11.3', 'target_triple': 'aarch64-apple-darwin'} ``` diff --git a/website/content/docs/guides/python-reactions.md b/website/content/docs/guides/python-reactions.md index 6b26fee..1a2965b 100644 --- a/website/content/docs/guides/python-reactions.md +++ b/website/content/docs/guides/python-reactions.md @@ -58,12 +58,16 @@ An ordinary reaction starts from scratch after a restart: whatever was delivered the process was down is gone. A **durable** reaction records how far it got, and resumes from there. -It needs a state store, configured when the engine is created: +It needs two stores, configured when the engine is created. The state store keeps the +checkpoint. The index store keeps the query's outbox, which is what the checkpoint +points at. An in-memory query is rejected: after a restart there would be nothing to +resume from. ```python drasi = await Drasi.create( "app", - state_store={"kind": "redb", "path": "/var/lib/myapp/drasi"}, + state_store={"kind": "redb", "path": "/var/lib/myapp/drasi/state.redb"}, + index_store={"kind": "rocksdb", "path": "/var/lib/myapp/drasi/index"}, ) ``` @@ -87,7 +91,7 @@ for example when the data it referred to is no longer available after a restart: | Policy | Behaviour | | --- | --- | | `strict` | Refuse to start rather than deliver an incomplete stream. The default. | -| `auto_reset` | Start again from the current position, accepting the gap. | +| `auto_reset` | Move the checkpoint to the current result-set head and continue. The rows already in the result set are not replayed to the callback. | | `skip_gap` | Continue past the gap and carry on from what is available. | `strict` is the default: the alternatives accept data loss. @@ -97,8 +101,10 @@ The reaction loop runs off your event loop, so a plain function cannot be awaite there. Registering one raises immediately. {{% /alert %}} -Checkpoints advance only after your callback returns without raising. If it raises, the -checkpoint stays where it was and the event is delivered again on the next attempt. +Checkpoints advance only after your callback returns without raising. If it raises +under `strict` or `auto_reset`, the reaction stops in an error state and the +checkpoint stays put, so the event is delivered again the next time the reaction +starts. `skip_gap` records the event as handled and continues. ## Reading results directly