From 0d41b36614be9898fe094df2a87b1d9d656d2609 Mon Sep 17 00:00:00 2001 From: Muhideen Mujeeb Adeoye Date: Mon, 31 Aug 2026 09:52:48 +0100 Subject: [PATCH 1/9] =?UTF-8?q?=E2=9C=A8=20feat(workflow):=20prototype=20d?= =?UTF-8?q?ynamic=20orchestration=20semantics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conformance/CAPABILITY_REGISTER.md | 5 + crates/headgate-workflow/src/experimental.rs | 650 ++++++++++++++++++ crates/headgate-workflow/src/lib.rs | 2 + docs/workflow-experiments.md | 46 ++ go/headgateworkflow/experimental/reducer.go | 418 +++++++++++ .../experimental/reducer_test.go | 93 +++ 6 files changed, 1214 insertions(+) create mode 100644 crates/headgate-workflow/src/experimental.rs create mode 100644 docs/workflow-experiments.md create mode 100644 go/headgateworkflow/experimental/reducer.go create mode 100644 go/headgateworkflow/experimental/reducer_test.go diff --git a/conformance/CAPABILITY_REGISTER.md b/conformance/CAPABILITY_REGISTER.md index 9938565..ddd982a 100644 --- a/conformance/CAPABILITY_REGISTER.md +++ b/conformance/CAPABILITY_REGISTER.md @@ -114,6 +114,11 @@ not think to name. | Task-local typed data (non-persisted) | ✅ | **Round 32y: implemented in both runtimes as two deliberately separate, process-local type maps.** Rust `Extensions` is keyed by `TypeId` and returns `Arc`; `WorkerConfig.extensions` is shared across the worker while every `JobCtx::from_claim` creates a fresh job map. `JobCtx::data` applies job-then-worker shadowing, with explicit `worker_data`, `job_data`, and `insert_data` APIs. Go mirrors this with a `reflect.Type`-keyed `Extensions`, generic `SetExtension`/`Extension`, `Config.Extensions`, and handler-context `Data[T]` / `WorkerData[T]` / `JobData[T]` / `SetJobData`; typed boxes preserve typed nils and a mutex covers every map access. Both real worker-loop tests force two concurrent jobs to insert the SAME concrete type before either reads it, then require each job's own value, the unchanged worker default, a wrong-type miss, terminal completion, and an envelope snapshot with no local marker. Container tests separately pin type replacement/removal and the Go outside-handler error. **Mutation teeth:** making the job map reuse the worker map failed both concurrency tests (and Go ran under `-race`). `docs/task-data.md` records attempt lifetime, shadowing, type-identity/newtype guidance, and that this is storage only—not the still-❌ handler-extractor API. | | Task aggregation / batch handlers | ✅ | **Typed execution chunks in both runtimes.** Rust `Registry::register_batch` and Go `RegisterBatchFunc` coalesce same-kind claims from one atomic admission call until a maximum size or absolute maximum delay, then invoke one handler. Each member keeps its own context, fence, lease, timeout, checkpoint, logs, rate weight and durable outcome; positional results allow partial success/retry and a result-count mismatch fails every member rather than silently dropping one. Panics are isolated and wake every waiter. Direct Store callers keep singleton units for compatibility; worker and test-drain paths form deterministic bounded units and run them concurrently. This is River/Oban-style execution batching, not Sidekiq workflow batches or asynq's synthetic aggregate-task replacement. See `docs/batch-handlers.md`. | | **Workflows / DAG dependencies** | ✅ | **Round 32u: durable DAG dependency gating is implemented as the separate opt-in `headgate-workflow` crate and Go `workflow` package—fulfilling the architecture boundary without adding orchestration to core or changing admission.** Builders reject empty/duplicate names, missing/repeated dependencies and cycles before enqueue, then return one batch containing a durable coordinator and every application task in `pending`. The coordinator uses bounded point reads only, promotes roots, then fan-out/fan-in nodes only when every dependency is `completed`; it snoozes without consuming attempts while work is live. Failed/missing upstream jobs cause still-pending descendants to be deleted before execution and archive the coordinator as the workflow-level failure record. Zero child retention is raised to the workflow retention so completion cannot become indistinguishable from a missing dependency; explicit positive retention is preserved. A live Postgres runtime test executes `extract -> {left,right} -> join` through the real worker/Store path and asserts the join is last. Go's independent coordinator test drives the same fan-out/fan-in state machine and proves an archived branch removes the pending join and settles failed. This row claims durable static DAG dependencies, not River Pro's signals, timers, CEL waits, retry, dynamic grafting/nesting, or graph UI; those remain explicitly outside this slice. See `docs/workflows.md`. | +| Workflow signals | ❌ | Experimenting on `codex/workflow-experiments`; no durable write path or public claim yet. | +| Workflow timers | ❌ | Experimenting with store-time semantics; worker-clock timers will not be accepted. | +| Workflow graph mutation | ❌ | Experimenting with additive, revision-checked grafts; in-place mutation of executed nodes is out of scope. | +| Nested workflows | ❌ | Experimenting with child coordinators as explicit parent nodes; failure and retry propagation remain undecided. | +| Workflow-level retry | ❌ | Experimenting with failed-subgraph retry while preserving successful ancestors; no control API claim yet. | ## Failure diff --git a/crates/headgate-workflow/src/experimental.rs b/crates/headgate-workflow/src/experimental.rs new file mode 100644 index 0000000..c696c78 --- /dev/null +++ b/crates/headgate-workflow/src/experimental.rs @@ -0,0 +1,650 @@ +//! Experimental workflow semantics. This reducer is intentionally store-agnostic: it +//! settles behavior before durable adapters and control APIs make the contract permanent. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum NodeKind { + Task, + Signal { signal: String }, + Timer { wake_at_ms: i64 }, + ChildWorkflow { workflow_id: String }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct NodeSpec { + pub name: String, + #[serde(default)] + pub deps: Vec, + pub kind: NodeKind, +} + +impl NodeSpec { + pub fn task( + name: impl Into, + deps: impl IntoIterator>, + ) -> Self { + Self { + name: name.into(), + deps: deps.into_iter().map(Into::into).collect(), + kind: NodeKind::Task, + } + } + + pub fn signal( + name: impl Into, + signal: impl Into, + deps: impl IntoIterator>, + ) -> Self { + Self { + name: name.into(), + deps: deps.into_iter().map(Into::into).collect(), + kind: NodeKind::Signal { + signal: signal.into(), + }, + } + } + + pub fn timer( + name: impl Into, + wake_at_ms: i64, + deps: impl IntoIterator>, + ) -> Self { + Self { + name: name.into(), + deps: deps.into_iter().map(Into::into).collect(), + kind: NodeKind::Timer { wake_at_ms }, + } + } + + pub fn child( + name: impl Into, + workflow_id: impl Into, + deps: impl IntoIterator>, + ) -> Self { + Self { + name: name.into(), + deps: deps.into_iter().map(Into::into).collect(), + kind: NodeKind::ChildWorkflow { + workflow_id: workflow_id.into(), + }, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NodeState { + Waiting, + Active, + Succeeded, + Failed, + Blocked, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RunStatus { + Running, + Succeeded, + Failed, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct RuntimeNode { + pub spec: NodeSpec, + pub state: NodeState, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Run { + pub revision: u64, + pub generation: u32, + pub status: RunStatus, + pub store_now_ms: i64, + pub nodes: BTreeMap, + pub signals: BTreeSet, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Command { + Signal { + signal: String, + }, + AdvanceStoreTime { + now_ms: i64, + }, + SucceedNode { + name: String, + }, + FailNode { + name: String, + }, + Graft { + expected_revision: u64, + nodes: Vec, + }, + RetryFailedSubgraph { + expected_revision: u64, + }, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Action { + DispatchTask { + name: String, + generation: u32, + }, + WaitForSignal { + name: String, + signal: String, + }, + ArmTimer { + name: String, + wake_at_ms: i64, + }, + StartChildWorkflow { + name: String, + workflow_id: String, + generation: u32, + }, + WorkflowSucceeded { + generation: u32, + }, + WorkflowFailed { + name: String, + generation: u32, + }, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExperimentError(pub String); + +impl std::fmt::Display for ExperimentError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for ExperimentError {} + +impl Run { + pub fn new( + nodes: Vec, + store_now_ms: i64, + ) -> Result<(Self, Vec), ExperimentError> { + validate_graph(&nodes)?; + let mut run = Self { + revision: 1, + generation: 1, + status: RunStatus::Running, + store_now_ms, + nodes: nodes + .into_iter() + .map(|spec| { + ( + spec.name.clone(), + RuntimeNode { + spec, + state: NodeState::Waiting, + }, + ) + }) + .collect(), + signals: BTreeSet::new(), + }; + let actions = run.reconcile(); + Ok((run, actions)) + } + + pub fn apply(&mut self, command: Command) -> Result, ExperimentError> { + let mut actions = Vec::new(); + match command { + Command::Signal { signal } => { + if signal.is_empty() { + return Err(ExperimentError("signal name must not be empty".into())); + } + if !self.nodes.values().any( + |node| matches!(&node.spec.kind, NodeKind::Signal { signal: expected } if expected == &signal), + ) { + return Err(ExperimentError(format!("unknown signal `{signal}`"))); + } + self.signals.insert(signal.clone()); + for node in self.nodes.values_mut() { + if node.state == NodeState::Active + && matches!(&node.spec.kind, NodeKind::Signal { signal: expected } if expected == &signal) + { + node.state = NodeState::Succeeded; + } + } + } + Command::AdvanceStoreTime { now_ms } => { + if now_ms < self.store_now_ms { + return Err(ExperimentError("store time must not move backwards".into())); + } + self.store_now_ms = now_ms; + for node in self.nodes.values_mut() { + if node.state == NodeState::Active + && matches!(node.spec.kind, NodeKind::Timer { wake_at_ms } if wake_at_ms <= now_ms) + { + node.state = NodeState::Succeeded; + } + } + } + Command::SucceedNode { name } => self.settle_node(&name, true)?, + Command::FailNode { name } => { + self.settle_node(&name, false)?; + self.block_descendants(&name); + self.status = RunStatus::Failed; + actions.push(Action::WorkflowFailed { + name, + generation: self.generation, + }); + } + Command::Graft { + expected_revision, + nodes, + } => { + self.require_revision(expected_revision)?; + if self.status != RunStatus::Running { + return Err(ExperimentError( + "nodes may only be grafted onto a running workflow".into(), + )); + } + if nodes.is_empty() { + return Err(ExperimentError( + "graft must contain at least one node".into(), + )); + } + let mut combined: Vec = + self.nodes.values().map(|node| node.spec.clone()).collect(); + for node in &nodes { + if self.nodes.contains_key(&node.name) { + return Err(ExperimentError(format!( + "graft repeats existing node `{}`", + node.name + ))); + } + combined.push(node.clone()); + } + validate_graph(&combined)?; + for spec in nodes { + self.nodes.insert( + spec.name.clone(), + RuntimeNode { + spec, + state: NodeState::Waiting, + }, + ); + } + self.revision += 1; + } + Command::RetryFailedSubgraph { expected_revision } => { + self.require_revision(expected_revision)?; + if self.status != RunStatus::Failed { + return Err(ExperimentError( + "only a failed workflow may be retried".into(), + )); + } + for node in self.nodes.values_mut() { + if matches!(node.state, NodeState::Failed | NodeState::Blocked) { + node.state = NodeState::Waiting; + } + } + self.generation = self + .generation + .checked_add(1) + .ok_or_else(|| ExperimentError("workflow generation overflow".into()))?; + self.revision += 1; + self.status = RunStatus::Running; + } + } + actions.extend(self.reconcile()); + Ok(actions) + } + + fn require_revision(&self, expected: u64) -> Result<(), ExperimentError> { + if expected != self.revision { + return Err(ExperimentError(format!( + "revision conflict: expected {expected}, current {}", + self.revision + ))); + } + Ok(()) + } + + fn settle_node(&mut self, name: &str, succeeded: bool) -> Result<(), ExperimentError> { + let node = self + .nodes + .get_mut(name) + .ok_or_else(|| ExperimentError(format!("unknown node `{name}`")))?; + if node.state != NodeState::Active { + return Err(ExperimentError(format!("node `{name}` is not active"))); + } + if !matches!( + node.spec.kind, + NodeKind::Task | NodeKind::ChildWorkflow { .. } + ) { + return Err(ExperimentError(format!( + "node `{name}` is settled by its signal or timer" + ))); + } + node.state = if succeeded { + NodeState::Succeeded + } else { + NodeState::Failed + }; + Ok(()) + } + + fn block_descendants(&mut self, failed: &str) { + let mut queue = VecDeque::from([failed.to_string()]); + while let Some(parent) = queue.pop_front() { + let children: Vec = self + .nodes + .values() + .filter(|node| node.spec.deps.iter().any(|dep| dep == &parent)) + .map(|node| node.spec.name.clone()) + .collect(); + for child in children { + if let Some(node) = self.nodes.get_mut(&child) { + if matches!(node.state, NodeState::Waiting | NodeState::Active) { + node.state = NodeState::Blocked; + queue.push_back(child); + } + } + } + } + } + + fn reconcile(&mut self) -> Vec { + if self.status != RunStatus::Running { + return Vec::new(); + } + let mut actions = Vec::new(); + loop { + let ready: Vec = self + .nodes + .values() + .filter(|node| node.state == NodeState::Waiting) + .filter(|node| { + node.spec.deps.iter().all(|dep| { + self.nodes + .get(dep) + .is_some_and(|upstream| upstream.state == NodeState::Succeeded) + }) + }) + .map(|node| node.spec.name.clone()) + .collect(); + if ready.is_empty() { + break; + } + let mut completed_virtual = false; + for name in ready { + let node = self.nodes.get_mut(&name).expect("ready node exists"); + match &node.spec.kind { + NodeKind::Task => { + node.state = NodeState::Active; + actions.push(Action::DispatchTask { + name, + generation: self.generation, + }); + } + NodeKind::Signal { signal } if self.signals.contains(signal) => { + node.state = NodeState::Succeeded; + completed_virtual = true; + } + NodeKind::Signal { signal } => { + node.state = NodeState::Active; + actions.push(Action::WaitForSignal { + name, + signal: signal.clone(), + }); + } + NodeKind::Timer { wake_at_ms } if *wake_at_ms <= self.store_now_ms => { + node.state = NodeState::Succeeded; + completed_virtual = true; + } + NodeKind::Timer { wake_at_ms } => { + node.state = NodeState::Active; + actions.push(Action::ArmTimer { + name, + wake_at_ms: *wake_at_ms, + }); + } + NodeKind::ChildWorkflow { workflow_id } => { + node.state = NodeState::Active; + actions.push(Action::StartChildWorkflow { + name, + workflow_id: workflow_id.clone(), + generation: self.generation, + }); + } + } + } + if !completed_virtual { + break; + } + } + if self + .nodes + .values() + .all(|node| node.state == NodeState::Succeeded) + { + self.status = RunStatus::Succeeded; + actions.push(Action::WorkflowSucceeded { + generation: self.generation, + }); + } + actions + } +} + +fn validate_graph(nodes: &[NodeSpec]) -> Result<(), ExperimentError> { + if nodes.is_empty() { + return Err(ExperimentError( + "workflow must contain at least one node".into(), + )); + } + let mut names = BTreeSet::new(); + for node in nodes { + if node.name.is_empty() || !names.insert(node.name.as_str()) { + return Err(ExperimentError( + "node names must be non-empty and unique".into(), + )); + } + if matches!(&node.kind, NodeKind::Signal { signal } if signal.is_empty()) { + return Err(ExperimentError(format!( + "signal node `{}` has an empty signal", + node.name + ))); + } + if matches!(&node.kind, NodeKind::ChildWorkflow { workflow_id } if workflow_id.is_empty()) { + return Err(ExperimentError(format!( + "child node `{}` has an empty workflow id", + node.name + ))); + } + } + let mut degree: BTreeMap<&str, usize> = + nodes.iter().map(|node| (node.name.as_str(), 0)).collect(); + let mut outgoing: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for node in nodes { + let mut unique = BTreeSet::new(); + for dep in &node.deps { + if !names.contains(dep.as_str()) { + return Err(ExperimentError(format!( + "node `{}` depends on missing node `{dep}`", + node.name + ))); + } + if !unique.insert(dep.as_str()) { + return Err(ExperimentError(format!( + "node `{}` repeats dependency `{dep}`", + node.name + ))); + } + *degree + .get_mut(node.name.as_str()) + .expect("node degree exists") += 1; + outgoing.entry(dep).or_default().push(&node.name); + } + } + let mut queue: VecDeque<&str> = degree + .iter() + .filter_map(|(name, count)| (*count == 0).then_some(*name)) + .collect(); + let mut visited = 0; + while let Some(name) = queue.pop_front() { + visited += 1; + for child in outgoing.get(name).into_iter().flatten() { + let count = degree.get_mut(child).expect("child degree exists"); + *count -= 1; + if *count == 0 { + queue.push_back(child); + } + } + } + if visited != nodes.len() { + return Err(ExperimentError("workflow graph contains a cycle".into())); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn names(actions: &[Action]) -> Vec<&str> { + actions + .iter() + .filter_map(|action| match action { + Action::DispatchTask { name, .. } | Action::StartChildWorkflow { name, .. } => { + Some(name.as_str()) + } + _ => None, + }) + .collect() + } + + #[test] + fn signals_and_store_time_timers_unlock_in_dependency_order() { + let (mut run, first) = Run::new( + vec![ + NodeSpec::task("prepare", Vec::::new()), + NodeSpec::signal("approval", "approved", ["prepare"]), + NodeSpec::timer("release", 1_500, ["approval"]), + NodeSpec::task("publish", ["release"]), + ], + 1_000, + ) + .unwrap(); + assert_eq!(names(&first), ["prepare"]); + let unknown = run + .apply(Command::Signal { + signal: "typo".into(), + }) + .unwrap_err(); + assert!(unknown.0.contains("unknown signal")); + assert!( + run.apply(Command::Signal { + signal: "approved".into() + }) + .unwrap() + .is_empty() + ); + let wait = run + .apply(Command::SucceedNode { + name: "prepare".into(), + }) + .unwrap(); + assert!(wait.iter().any( + |a| matches!(a, Action::ArmTimer { name, wake_at_ms: 1_500 } if name == "release") + )); + assert!( + run.apply(Command::AdvanceStoreTime { now_ms: 1_499 }) + .unwrap() + .is_empty() + ); + assert_eq!( + names( + &run.apply(Command::AdvanceStoreTime { now_ms: 1_500 }) + .unwrap() + ), + ["publish"] + ); + } + + #[test] + fn graft_is_additive_revision_checked_and_cycle_safe() { + let (mut run, _) = Run::new(vec![NodeSpec::task("root", Vec::::new())], 0).unwrap(); + let actions = run + .apply(Command::Graft { + expected_revision: 1, + nodes: vec![NodeSpec::task("grafted", ["root"])], + }) + .unwrap(); + assert!(actions.is_empty()); + assert_eq!(run.revision, 2); + let stale = run + .apply(Command::Graft { + expected_revision: 1, + nodes: vec![NodeSpec::task("stale", ["root"])], + }) + .unwrap_err(); + assert!(stale.0.contains("revision conflict")); + let cycle = run + .apply(Command::Graft { + expected_revision: 2, + nodes: vec![NodeSpec::task("a", ["b"]), NodeSpec::task("b", ["a"])], + }) + .unwrap_err(); + assert!(cycle.0.contains("cycle")); + } + + #[test] + fn nested_failure_retries_only_failed_subgraph() { + let (mut run, first) = Run::new( + vec![ + NodeSpec::task("extract", Vec::::new()), + NodeSpec::child("child", "child-workflow", ["extract"]), + NodeSpec::task("finish", ["child"]), + ], + 0, + ) + .unwrap(); + assert_eq!(names(&first), ["extract"]); + let child = run + .apply(Command::SucceedNode { + name: "extract".into(), + }) + .unwrap(); + assert_eq!(names(&child), ["child"]); + let failed = run + .apply(Command::FailNode { + name: "child".into(), + }) + .unwrap(); + assert!(failed.iter().any( + |a| matches!(a, Action::WorkflowFailed { name, generation: 1 } if name == "child") + )); + assert_eq!(run.nodes["extract"].state, NodeState::Succeeded); + assert_eq!(run.nodes["finish"].state, NodeState::Blocked); + let retried = run + .apply(Command::RetryFailedSubgraph { + expected_revision: 1, + }) + .unwrap(); + assert_eq!(run.generation, 2); + assert_eq!(run.nodes["extract"].state, NodeState::Succeeded); + assert_eq!(names(&retried), ["child"]); + assert!( + retried + .iter() + .any(|a| matches!(a, Action::StartChildWorkflow { generation: 2, .. })) + ); + } +} diff --git a/crates/headgate-workflow/src/lib.rs b/crates/headgate-workflow/src/lib.rs index cdea1cd..55b5358 100644 --- a/crates/headgate-workflow/src/lib.rs +++ b/crates/headgate-workflow/src/lib.rs @@ -1,5 +1,7 @@ //! Durable DAG dependencies layered on headgate's ordinary pending jobs. +pub mod experimental; + use std::{ collections::{HashMap, HashSet, VecDeque}, sync::Arc, diff --git a/docs/workflow-experiments.md b/docs/workflow-experiments.md new file mode 100644 index 0000000..d16e8ec --- /dev/null +++ b/docs/workflow-experiments.md @@ -0,0 +1,46 @@ +# Dynamic workflow experiments + +This branch explores dynamic workflow behavior without changing the shipped static +coordinator or claiming durable support. Rust exposes the reducer under +`headgate_workflow::experimental`; Go mirrors it in `headgateworkflow/experimental`. + +The reducer exists to settle semantics before a migration, store port, HTTP API, or UI +makes an accidental contract permanent. + +## Current decisions + +| Capability | Experimental behavior | +| --- | --- | +| Signals | Signals are named, idempotent, and buffered. A signal received before its dependencies complete is retained and consumed when the wait node becomes eligible. | +| Timers | Timer deadlines are absolute milliseconds advanced by store time. Moving time backwards is rejected; worker clocks are not accepted as durable workflow time. | +| Graph mutation | Grafts are additive and require the caller's expected graph revision. Existing nodes cannot be rewritten, and the combined graph must still have unique names, valid dependencies, and no cycle. | +| Nested workflows | A child workflow is an explicit node. The parent dispatches it only after its dependencies succeed and settles it through the same success/failure boundary as a task. | +| Workflow retry | Retry increments the workflow generation, resets failed and dependency-blocked nodes, and preserves successful ancestors. It does not silently rerun already successful effects. | + +The reducer emits actions instead of performing I/O: dispatch a task, wait for a signal, +arm a timer, start a child workflow, or record terminal workflow state. Rust and Go tests +drive the same signal → timer chain, revision-conflicted graft, nested failure, and +failed-subgraph retry. + +## Durability boundary still to build + +A production implementation must commit the state transition and its emitted actions in +one store transaction or script. Otherwise a coordinator can persist `active` and crash +before dispatching the action, or dispatch twice after a crash. The eventual action +identity should include workflow ID, graph revision, generation, node name, and action +kind so replay is deterministic. + +The current experiment deliberately has no: + +- PostgreSQL, MySQL, or Redis persistence; +- signal, graft, retry, or child-workflow control API; +- authorization and `Idempotency-Key` contract for those mutations; +- migration from the v1 immutable coordinator payload; +- dynamic workflow UI controls; or +- conformance claim. + +Before promoting the reducer, the design still needs decisions for cancellation of active +parallel branches, propagation of parent cancellation into children, relative timers that +start after a dependency completes, event-history retention, and bounded graph/event +limits. The existing immutable coordinator remains the compatibility baseline throughout +the experiment. diff --git a/go/headgateworkflow/experimental/reducer.go b/go/headgateworkflow/experimental/reducer.go new file mode 100644 index 0000000..7f2a60b --- /dev/null +++ b/go/headgateworkflow/experimental/reducer.go @@ -0,0 +1,418 @@ +// Package experimental settles dynamic workflow semantics before durable adapters and +// control APIs make the contract permanent. It is not a persistence implementation. +package experimental + +import ( + "errors" + "fmt" + "sort" +) + +type NodeKind string + +const ( + Task NodeKind = "task" + Signal NodeKind = "signal" + Timer NodeKind = "timer" + ChildWorkflow NodeKind = "child_workflow" +) + +type NodeSpec struct { + Name string + Deps []string + Kind NodeKind + Signal string + WakeAtMs int64 + WorkflowID string +} + +func TaskNode(name string, deps ...string) NodeSpec { + return NodeSpec{Name: name, Deps: clone(deps), Kind: Task} +} + +func SignalNode(name, signal string, deps ...string) NodeSpec { + return NodeSpec{Name: name, Deps: clone(deps), Kind: Signal, Signal: signal} +} + +func TimerNode(name string, wakeAtMs int64, deps ...string) NodeSpec { + return NodeSpec{Name: name, Deps: clone(deps), Kind: Timer, WakeAtMs: wakeAtMs} +} + +func ChildNode(name, workflowID string, deps ...string) NodeSpec { + return NodeSpec{Name: name, Deps: clone(deps), Kind: ChildWorkflow, WorkflowID: workflowID} +} + +type NodeState string + +const ( + Waiting NodeState = "waiting" + Active NodeState = "active" + Succeeded NodeState = "succeeded" + Failed NodeState = "failed" + Blocked NodeState = "blocked" +) + +type RunStatus string + +const ( + Running RunStatus = "running" + RunSucceeded RunStatus = "succeeded" + RunFailed RunStatus = "failed" +) + +type RuntimeNode struct { + Spec NodeSpec + State NodeState +} + +type Run struct { + Revision uint64 + Generation uint32 + Status RunStatus + StoreNowMs int64 + Nodes map[string]*RuntimeNode + Signals map[string]struct{} +} + +type ActionType string + +const ( + DispatchTask ActionType = "dispatch_task" + WaitForSignal ActionType = "wait_for_signal" + ArmTimer ActionType = "arm_timer" + StartChildWorkflow ActionType = "start_child_workflow" + WorkflowSucceeded ActionType = "workflow_succeeded" + WorkflowFailed ActionType = "workflow_failed" +) + +type Action struct { + Type ActionType + Name string + Signal string + WakeAtMs int64 + WorkflowID string + Generation uint32 +} + +func NewRun(nodes []NodeSpec, storeNowMs int64) (*Run, []Action, error) { + if err := validateGraph(nodes); err != nil { + return nil, nil, err + } + run := &Run{ + Revision: 1, Generation: 1, Status: Running, StoreNowMs: storeNowMs, + Nodes: make(map[string]*RuntimeNode, len(nodes)), Signals: map[string]struct{}{}, + } + for _, raw := range nodes { + spec := cloneSpec(raw) + run.Nodes[spec.Name] = &RuntimeNode{Spec: spec, State: Waiting} + } + return run, run.reconcile(), nil +} + +func (r *Run) ReceiveSignal(signal string) ([]Action, error) { + if signal == "" { + return nil, errors.New("signal name must not be empty") + } + known := false + for _, node := range r.Nodes { + if node.Spec.Kind == Signal && node.Spec.Signal == signal { + known = true + break + } + } + if !known { + return nil, fmt.Errorf("unknown signal `%s`", signal) + } + r.Signals[signal] = struct{}{} + for _, node := range r.Nodes { + if node.State == Active && node.Spec.Kind == Signal && node.Spec.Signal == signal { + node.State = Succeeded + } + } + return r.reconcile(), nil +} + +func (r *Run) AdvanceStoreTime(nowMs int64) ([]Action, error) { + if nowMs < r.StoreNowMs { + return nil, errors.New("store time must not move backwards") + } + r.StoreNowMs = nowMs + for _, node := range r.Nodes { + if node.State == Active && node.Spec.Kind == Timer && node.Spec.WakeAtMs <= nowMs { + node.State = Succeeded + } + } + return r.reconcile(), nil +} + +func (r *Run) SucceedNode(name string) ([]Action, error) { + if err := r.settleNode(name, true); err != nil { + return nil, err + } + return r.reconcile(), nil +} + +func (r *Run) FailNode(name string) ([]Action, error) { + if err := r.settleNode(name, false); err != nil { + return nil, err + } + r.blockDescendants(name) + r.Status = RunFailed + return []Action{{Type: WorkflowFailed, Name: name, Generation: r.Generation}}, nil +} + +func (r *Run) Graft(expectedRevision uint64, nodes ...NodeSpec) ([]Action, error) { + if err := r.requireRevision(expectedRevision); err != nil { + return nil, err + } + if r.Status != Running { + return nil, errors.New("nodes may only be grafted onto a running workflow") + } + if len(nodes) == 0 { + return nil, errors.New("graft must contain at least one node") + } + combined := make([]NodeSpec, 0, len(r.Nodes)+len(nodes)) + for _, node := range r.Nodes { + combined = append(combined, cloneSpec(node.Spec)) + } + for _, node := range nodes { + if _, exists := r.Nodes[node.Name]; exists { + return nil, fmt.Errorf("graft repeats existing node `%s`", node.Name) + } + combined = append(combined, cloneSpec(node)) + } + if err := validateGraph(combined); err != nil { + return nil, err + } + for _, raw := range nodes { + spec := cloneSpec(raw) + r.Nodes[spec.Name] = &RuntimeNode{Spec: spec, State: Waiting} + } + r.Revision++ + return r.reconcile(), nil +} + +func (r *Run) RetryFailedSubgraph(expectedRevision uint64) ([]Action, error) { + if err := r.requireRevision(expectedRevision); err != nil { + return nil, err + } + if r.Status != RunFailed { + return nil, errors.New("only a failed workflow may be retried") + } + if r.Generation == ^uint32(0) { + return nil, errors.New("workflow generation overflow") + } + for _, node := range r.Nodes { + if node.State == Failed || node.State == Blocked { + node.State = Waiting + } + } + r.Generation++ + r.Revision++ + r.Status = Running + return r.reconcile(), nil +} + +func (r *Run) requireRevision(expected uint64) error { + if expected != r.Revision { + return fmt.Errorf("revision conflict: expected %d, current %d", expected, r.Revision) + } + return nil +} + +func (r *Run) settleNode(name string, success bool) error { + node := r.Nodes[name] + if node == nil { + return fmt.Errorf("unknown node `%s`", name) + } + if node.State != Active { + return fmt.Errorf("node `%s` is not active", name) + } + if node.Spec.Kind != Task && node.Spec.Kind != ChildWorkflow { + return fmt.Errorf("node `%s` is settled by its signal or timer", name) + } + if success { + node.State = Succeeded + } else { + node.State = Failed + } + return nil +} + +func (r *Run) blockDescendants(failed string) { + queue := []string{failed} + for len(queue) > 0 { + parent := queue[0] + queue = queue[1:] + children := make([]string, 0) + for name, node := range r.Nodes { + if contains(node.Spec.Deps, parent) { + children = append(children, name) + } + } + sort.Strings(children) + for _, child := range children { + node := r.Nodes[child] + if node.State == Waiting || node.State == Active { + node.State = Blocked + queue = append(queue, child) + } + } + } +} + +func (r *Run) reconcile() []Action { + if r.Status != Running { + return nil + } + actions := make([]Action, 0) + for { + ready := make([]string, 0) + for name, node := range r.Nodes { + if node.State != Waiting { + continue + } + complete := true + for _, dep := range node.Spec.Deps { + if r.Nodes[dep] == nil || r.Nodes[dep].State != Succeeded { + complete = false + break + } + } + if complete { + ready = append(ready, name) + } + } + sort.Strings(ready) + if len(ready) == 0 { + break + } + completedVirtual := false + for _, name := range ready { + node := r.Nodes[name] + switch node.Spec.Kind { + case Task: + node.State = Active + actions = append(actions, Action{Type: DispatchTask, Name: name, Generation: r.Generation}) + case Signal: + if _, received := r.Signals[node.Spec.Signal]; received { + node.State = Succeeded + completedVirtual = true + } else { + node.State = Active + actions = append(actions, Action{Type: WaitForSignal, Name: name, Signal: node.Spec.Signal}) + } + case Timer: + if node.Spec.WakeAtMs <= r.StoreNowMs { + node.State = Succeeded + completedVirtual = true + } else { + node.State = Active + actions = append(actions, Action{Type: ArmTimer, Name: name, WakeAtMs: node.Spec.WakeAtMs}) + } + case ChildWorkflow: + node.State = Active + actions = append(actions, Action{ + Type: StartChildWorkflow, Name: name, WorkflowID: node.Spec.WorkflowID, Generation: r.Generation, + }) + } + } + if !completedVirtual { + break + } + } + allSucceeded := true + for _, node := range r.Nodes { + if node.State != Succeeded { + allSucceeded = false + break + } + } + if allSucceeded { + r.Status = RunSucceeded + actions = append(actions, Action{Type: WorkflowSucceeded, Generation: r.Generation}) + } + return actions +} + +func validateGraph(nodes []NodeSpec) error { + if len(nodes) == 0 { + return errors.New("workflow must contain at least one node") + } + names := make(map[string]struct{}, len(nodes)) + for _, node := range nodes { + if node.Name == "" { + return errors.New("node names must be non-empty and unique") + } + if _, exists := names[node.Name]; exists { + return errors.New("node names must be non-empty and unique") + } + names[node.Name] = struct{}{} + if node.Kind == Signal && node.Signal == "" { + return fmt.Errorf("signal node `%s` has an empty signal", node.Name) + } + if node.Kind == ChildWorkflow && node.WorkflowID == "" { + return fmt.Errorf("child node `%s` has an empty workflow id", node.Name) + } + if node.Kind != Task && node.Kind != Signal && node.Kind != Timer && node.Kind != ChildWorkflow { + return fmt.Errorf("node `%s` has unknown kind `%s`", node.Name, node.Kind) + } + } + degree := make(map[string]int, len(nodes)) + outgoing := make(map[string][]string) + for _, node := range nodes { + seen := map[string]struct{}{} + for _, dep := range node.Deps { + if _, exists := names[dep]; !exists { + return fmt.Errorf("node `%s` depends on missing node `%s`", node.Name, dep) + } + if _, exists := seen[dep]; exists { + return fmt.Errorf("node `%s` repeats dependency `%s`", node.Name, dep) + } + seen[dep] = struct{}{} + degree[node.Name]++ + outgoing[dep] = append(outgoing[dep], node.Name) + } + } + ready := make([]string, 0) + for name := range names { + if degree[name] == 0 { + ready = append(ready, name) + } + } + sort.Strings(ready) + visited := 0 + for len(ready) > 0 { + name := ready[0] + ready = ready[1:] + visited++ + children := outgoing[name] + sort.Strings(children) + for _, child := range children { + degree[child]-- + if degree[child] == 0 { + ready = append(ready, child) + } + } + } + if visited != len(nodes) { + return errors.New("workflow graph contains a cycle") + } + return nil +} + +func contains(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} + +func clone(values []string) []string { return append([]string(nil), values...) } + +func cloneSpec(spec NodeSpec) NodeSpec { + spec.Deps = clone(spec.Deps) + return spec +} diff --git a/go/headgateworkflow/experimental/reducer_test.go b/go/headgateworkflow/experimental/reducer_test.go new file mode 100644 index 0000000..5ec6934 --- /dev/null +++ b/go/headgateworkflow/experimental/reducer_test.go @@ -0,0 +1,93 @@ +package experimental + +import ( + "strings" + "testing" +) + +func actionNames(actions []Action) []string { + result := make([]string, 0) + for _, action := range actions { + if action.Type == DispatchTask || action.Type == StartChildWorkflow { + result = append(result, action.Name) + } + } + return result +} + +func TestSignalsAndStoreTimeTimersUnlockInDependencyOrder(t *testing.T) { + run, first, err := NewRun([]NodeSpec{ + TaskNode("prepare"), + SignalNode("approval", "approved", "prepare"), + TimerNode("release", 1_500, "approval"), + TaskNode("publish", "release"), + }, 1_000) + if err != nil || len(first) != 1 || first[0].Name != "prepare" { + t.Fatalf("new run = %#v, %v", first, err) + } + if _, err := run.ReceiveSignal("typo"); err == nil || !strings.Contains(err.Error(), "unknown signal") { + t.Fatalf("unknown signal = %v", err) + } + if actions, err := run.ReceiveSignal("approved"); err != nil || len(actions) != 0 { + t.Fatalf("early signal = %#v, %v", actions, err) + } + wait, err := run.SucceedNode("prepare") + if err != nil || len(wait) != 1 || wait[0].Type != ArmTimer || wait[0].WakeAtMs != 1_500 { + t.Fatalf("timer arm = %#v, %v", wait, err) + } + if actions, err := run.AdvanceStoreTime(1_499); err != nil || len(actions) != 0 { + t.Fatalf("early time = %#v, %v", actions, err) + } + actions, err := run.AdvanceStoreTime(1_500) + if err != nil || len(actionNames(actions)) != 1 || actionNames(actions)[0] != "publish" { + t.Fatalf("timer fire = %#v, %v", actions, err) + } +} + +func TestGraftIsAdditiveRevisionCheckedAndCycleSafe(t *testing.T) { + run, _, err := NewRun([]NodeSpec{TaskNode("root")}, 0) + if err != nil { + t.Fatal(err) + } + if actions, err := run.Graft(1, TaskNode("grafted", "root")); err != nil || len(actions) != 0 { + t.Fatalf("graft = %#v, %v", actions, err) + } + if run.Revision != 2 { + t.Fatalf("revision = %d", run.Revision) + } + if _, err := run.Graft(1, TaskNode("stale", "root")); err == nil || !strings.Contains(err.Error(), "revision conflict") { + t.Fatalf("stale graft = %v", err) + } + if _, err := run.Graft(2, TaskNode("a", "b"), TaskNode("b", "a")); err == nil || !strings.Contains(err.Error(), "cycle") { + t.Fatalf("cyclic graft = %v", err) + } +} + +func TestNestedFailureRetriesOnlyFailedSubgraph(t *testing.T) { + run, first, err := NewRun([]NodeSpec{ + TaskNode("extract"), + ChildNode("child", "child-workflow", "extract"), + TaskNode("finish", "child"), + }, 0) + if err != nil || len(first) != 1 || first[0].Name != "extract" { + t.Fatalf("new run = %#v, %v", first, err) + } + child, err := run.SucceedNode("extract") + if err != nil || len(child) != 1 || child[0].Type != StartChildWorkflow { + t.Fatalf("child start = %#v, %v", child, err) + } + failed, err := run.FailNode("child") + if err != nil || len(failed) != 1 || failed[0].Type != WorkflowFailed { + t.Fatalf("child failure = %#v, %v", failed, err) + } + if run.Nodes["extract"].State != Succeeded || run.Nodes["finish"].State != Blocked { + t.Fatalf("states = extract %s, finish %s", run.Nodes["extract"].State, run.Nodes["finish"].State) + } + retried, err := run.RetryFailedSubgraph(1) + if err != nil || len(retried) != 1 || retried[0].Name != "child" || retried[0].Generation != 2 { + t.Fatalf("retry = %#v, %v", retried, err) + } + if run.Generation != 2 || run.Nodes["extract"].State != Succeeded { + t.Fatalf("generation = %d, extract = %s", run.Generation, run.Nodes["extract"].State) + } +} From 7470fa56afe0de18de808ed7472d39b3d448ae8d Mon Sep 17 00:00:00 2001 From: Muhideen Mujeeb Adeoye Date: Sat, 5 Sep 2026 13:50:26 +0100 Subject: [PATCH 2/9] =?UTF-8?q?=E2=9C=A8=20feat(workflow):=20add=20durable?= =?UTF-8?q?=20dynamic=20orchestration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 98 + Cargo.toml | 1 + api/headgate.openapi.yaml | 242 ++ conformance/CAPABILITY_REGISTER.md | 14 +- conformance/EVIDENCE.md | 76 + conformance/TEST_INVENTORY.tsv | 13 +- conformance/state_machine.yaml | 2 + crates/headgate-api/Cargo.toml | 1 + crates/headgate-api/src/lib.rs | 304 +- crates/headgate-api/tests/api.rs | 175 +- crates/headgate-core/src/lib.rs | 77 +- crates/headgate-crypto/tests/live.rs | 5 +- .../mysql/0013_durable_events.down.sql | 2 + .../mysql/0013_durable_events.up.sql | 17 + .../postgres/0013_durable_events.down.sql | 2 + .../postgres/0013_durable_events.up.sql | 17 + crates/headgate-migrate/src/bin/hg_migrate.rs | 2 +- crates/headgate-migrate/src/lib.rs | 21 +- crates/headgate-migrate/src/mysql.rs | 6 +- crates/headgate-migrate/src/schema.rs | 8 + crates/headgate-migrate/tests/live.rs | 30 +- .../migrations/0013_durable_events.sql | 17 + crates/headgate-mysql/src/inspect.rs | 178 +- crates/headgate-mysql/src/lib.rs | 57 +- crates/headgate-mysql/tests/bounded_pool.rs | 2 +- crates/headgate-mysql/tests/unique.rs | 3 +- .../migrations/0013_durable_events.sql | 17 + crates/headgate-postgres/src/inspect.rs | 191 +- crates/headgate-postgres/src/lib.rs | 16 +- .../headgate-postgres/tests/bounded_pool.rs | 2 +- crates/headgate-redis/lua/admin.lua | 17 +- crates/headgate-redis/src/inspect.rs | 228 +- crates/headgate-redis/src/lib.rs | 2 +- crates/headgate-shared/src/log.rs | 72 +- crates/headgate-testkit/src/database.rs | 2 +- crates/headgate-testkit/src/lib.rs | 162 +- crates/headgate-workflow/Cargo.toml | 5 + crates/headgate-workflow/src/experimental.rs | 650 ---- crates/headgate-workflow/src/lib.rs | 2969 ++++++++++++++++- crates/headgate-workflow/tests/live.rs | 513 ++- crates/headgate-workflow/tests/live_mysql.rs | 136 + crates/headgate/src/isolated.rs | 8 +- crates/headgate/src/tracked.rs | 20 +- crates/headgate/src/worker.rs | 24 +- crates/headgate/tests/runtime.rs | 3 +- examples/go/go.mod | 8 + examples/go/go.sum | 32 + go.work | 16 + go/go.work.sum => go.work.sum | 26 +- go/driver/headgatemysql/go.mod | 2 +- go/driver/headgatemysql/go.sum | 6 +- go/driver/headgatemysql/inspect.go | 105 +- go/driver/headgatepgx/go.mod | 4 +- go/driver/headgatepgx/go.sum | 6 +- go/driver/headgatepgx/inspect.go | 97 +- go/driver/headgateredis/go.mod | 2 +- go/driver/headgateredis/go.sum | 6 +- go/driver/headgateredis/inspect.go | 112 +- go/driver/headgateredis/lua/admin.lua | 17 +- go/go.work | 16 - go/headgate.go | 39 + go/headgateapi/api.go | 242 ++ go/headgateapi/api_test.go | 208 ++ go/headgateapi/go.mod | 17 +- go/headgateapi/go.sum | 35 +- go/headgatecrypto/go.mod | 5 +- go/headgatecrypto/go.sum | 3 +- go/headgatemigrate/go.mod | 2 +- go/headgatemigrate/go.sum | 6 +- go/headgatemigrate/live_mysql_test.go | 12 +- go/headgatemigrate/live_postgres_test.go | 10 +- go/headgatemigrate/migrate.go | 14 + go/headgatemigrate/migrate_test.go | 12 +- .../mysql/0013_durable_events.down.sql | 2 + .../mysql/0013_durable_events.up.sql | 17 + .../postgres/0013_durable_events.down.sql | 2 + .../postgres/0013_durable_events.up.sql | 17 + go/headgateotel/go.mod | 5 +- go/headgateotel/go.sum | 6 +- go/headgatetest/go.mod | 2 +- go/headgatetest/go.sum | 6 +- go/headgateworkflow/experimental/reducer.go | 418 --- .../experimental/reducer_test.go | 93 - go/headgateworkflow/go.mod | 46 +- go/headgateworkflow/go.sum | 55 +- go/headgateworkflow/live_matrix_test.go | 163 + go/headgateworkflow/live_mysql_test.go | 20 + go/headgateworkflow/workflow.go | 2340 ++++++++++++- go/headgateworkflow/workflow_test.go | 757 ++++- scripts/check-migrations.py | 34 +- scripts/run-scenarios.py | 3 + scripts/test-admission.sh | 3 + 92 files changed, 9653 insertions(+), 1803 deletions(-) create mode 100644 crates/headgate-migrate/migrations/mysql/0013_durable_events.down.sql create mode 100644 crates/headgate-migrate/migrations/mysql/0013_durable_events.up.sql create mode 100644 crates/headgate-migrate/migrations/postgres/0013_durable_events.down.sql create mode 100644 crates/headgate-migrate/migrations/postgres/0013_durable_events.up.sql create mode 100644 crates/headgate-mysql/migrations/0013_durable_events.sql create mode 100644 crates/headgate-postgres/migrations/0013_durable_events.sql delete mode 100644 crates/headgate-workflow/src/experimental.rs create mode 100644 crates/headgate-workflow/tests/live_mysql.rs create mode 100644 go.work rename go/go.work.sum => go.work.sum (88%) delete mode 100644 go/go.work create mode 100644 go/headgatemigrate/migrations/mysql/0013_durable_events.down.sql create mode 100644 go/headgatemigrate/migrations/mysql/0013_durable_events.up.sql create mode 100644 go/headgatemigrate/migrations/postgres/0013_durable_events.down.sql create mode 100644 go/headgatemigrate/migrations/postgres/0013_durable_events.up.sql delete mode 100644 go/headgateworkflow/experimental/reducer.go delete mode 100644 go/headgateworkflow/experimental/reducer_test.go create mode 100644 go/headgateworkflow/live_matrix_test.go create mode 100644 go/headgateworkflow/live_mysql_test.go diff --git a/Cargo.lock b/Cargo.lock index 328200b..04fb1eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,6 +79,23 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" +[[package]] +name = "antlr4rust" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093d520274bfff7278d776f7ea12981a0a0a6f96db90964658e0f38fc6e9a6a6" +dependencies = [ + "better_any", + "bit-set", + "byteorder", + "lazy_static", + "murmur3", + "once_cell", + "parking_lot", + "typed-arena", + "uuid", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -184,6 +201,27 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "better_any" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4372b9543397a4b86050cc5e7ee36953edf4bac9518e8a774c2da694977fb6e4" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "2.13.1" @@ -253,6 +291,22 @@ dependencies = [ "shlex", ] +[[package]] +name = "cel" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f93082a93da8fd78394852602ced1e2e7754ed8c29dc813fa80b01a1baf9032" +dependencies = [ + "antlr4rust", + "chrono", + "lazy_static", + "nom", + "pastey", + "regex", + "serde", + "thiserror 2.0.20", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -278,6 +332,7 @@ checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", + "serde", "windows-link", ] @@ -916,6 +971,7 @@ dependencies = [ "headgate-postgres", "headgate-redis", "headgate-ui", + "headgate-workflow", "http-body-util", "serde", "serde_json", @@ -1090,10 +1146,15 @@ dependencies = [ name = "headgate-workflow" version = "0.1.7" dependencies = [ + "cel", "futures-util", "headgate", "headgate-core", + "headgate-mysql", "headgate-postgres", + "headgate-redis", + "headgate-testkit", + "redis", "serde", "serde_json", "tokio", @@ -1533,6 +1594,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1554,6 +1621,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "murmur3" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a198f9589efc03f544388dfc4a19fe8af4323662b62f598b8dcfdac62c14771c" +dependencies = [ + "byteorder", +] + [[package]] name = "mysql_async" version = "0.35.1" @@ -1614,6 +1690,16 @@ dependencies = [ "zstd", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -1742,6 +1828,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pem" version = "3.0.6" @@ -2645,6 +2737,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + [[package]] name = "typenum" version = "1.20.1" diff --git a/Cargo.toml b/Cargo.toml index 92125ea..c901e36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,3 +30,4 @@ headgate-sql = { version = "0.1.7", path = "crates/headgate-sql" } headgate-testkit = { version = "0.1.7", path = "crates/headgate-testkit" } headgate-ui = { version = "0.1.7", path = "crates/headgate-ui" } headgate-otel = { version = "0.1.7", path = "crates/headgate-otel" } +headgate-workflow = { version = "0.1.7", path = "crates/headgate-workflow" } diff --git a/api/headgate.openapi.yaml b/api/headgate.openapi.yaml index e105afa..59e24ba 100644 --- a/api/headgate.openapi.yaml +++ b/api/headgate.openapi.yaml @@ -803,6 +803,193 @@ paths: reason: {type: string, maxLength: 64} recorded_at_ms: {type: integer} + /workflows: + get: + summary: List workflow coordinators without loading their graphs + parameters: + - {$ref: '#/components/parameters/ControlLimit'} + - {name: cursor, in: query, schema: {type: string}} + responses: + '200': + description: Bounded workflow page. + content: + application/json: + schema: + type: object + required: [workflows, next_cursor] + properties: + workflows: {type: array, maxItems: 200, items: {$ref: '#/components/schemas/WorkflowSummary'}} + next_cursor: {type: [string, "null"]} + /workflows/{id}: + get: + summary: Inspect a workflow's accepted graph and live execution state + description: Returns the base graph plus accepted additive grafts. Task payloads are never returned. + parameters: [{$ref: '#/components/parameters/Id'}] + responses: + '200': + description: Bounded workflow graph snapshot. + content: {application/json: {schema: {$ref: '#/components/schemas/WorkflowSnapshot'}}} + '404': {description: Workflow not found} + /workflows/{id}/events: + get: + summary: Read bounded durable workflow history + parameters: [{$ref: '#/components/parameters/Id'}] + responses: {'200': {description: At most 256 ordered events from the fenced coordinator checkpoint.}, '404': {description: Workflow not found}} + /workflows/{id}/nodes/{node}: + get: + summary: Inspect one workflow node + parameters: + - {$ref: '#/components/parameters/Id'} + - {name: node, in: path, required: true, schema: {type: string}} + responses: + '200': {description: Node topology and live job state, content: {application/json: {schema: {$ref: '#/components/schemas/WorkflowNode'}}}} + '404': {description: Workflow or node not found} + /workflows/{id}/nodes/{node}/dependencies: + get: + summary: List a workflow node's immediate prerequisites + parameters: + - {$ref: '#/components/parameters/Id'} + - {name: node, in: path, required: true, schema: {type: string}} + responses: + '200': + description: Immediate dependency nodes. + content: + application/json: + schema: + type: object + required: [dependencies] + properties: {dependencies: {type: array, items: {$ref: '#/components/schemas/WorkflowNode'}}} + '404': {description: Workflow or node not found} + /workflows/{id}/nodes/{node}/dependents: + get: + summary: List nodes that immediately depend on a workflow node + parameters: + - {$ref: '#/components/parameters/Id'} + - {name: node, in: path, required: true, schema: {type: string}} + responses: + '200': + description: Immediate dependent nodes. + content: + application/json: + schema: + type: object + required: [dependents] + properties: {dependents: {type: array, items: {$ref: '#/components/schemas/WorkflowNode'}}} + '404': {description: Workflow or node not found} + /workflows/{id}/signals: + get: + summary: List bounded durable workflow signal history + parameters: + - {$ref: '#/components/parameters/Id'} + - {$ref: '#/components/parameters/ControlLimit'} + - {name: cursor, in: query, schema: {type: integer, minimum: 1}} + responses: + '200': + description: Newest-first signal emissions, at most 100 retained per workflow. + content: + application/json: + schema: + type: object + required: [signals, next_cursor] + properties: + signals: {type: array, maxItems: 100, items: {$ref: '#/components/schemas/WorkflowSignal'}} + next_cursor: {type: [integer, "null"]} + post: + summary: Emit a durable buffered workflow signal + parameters: [{$ref: '#/components/parameters/Id'}, {$ref: '#/components/parameters/IdempotencyKey'}] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [signal] + properties: + signal: {type: string, minLength: 1} + payload: {description: Arbitrary JSON value delivered with the signal.} + source: {description: Caller-supplied JSON describing the emitter or calling system; authenticate and populate it at a trusted upstream boundary.} + responses: + '200': + description: Idempotent signal receipt including the original durable emission. + content: + application/json: + schema: + type: object + required: [matched, promoted, inserted, emission] + properties: + matched: {type: integer, minimum: 1} + promoted: {type: integer, minimum: 0} + inserted: {type: boolean, description: False when this idempotency key replayed an existing emission.} + emission: {$ref: '#/components/schemas/WorkflowSignal'} + /workflows/{id}/grafts: + post: + summary: Atomically submit a revision-checked additive graph graft + parameters: [{$ref: '#/components/parameters/Id'}, {$ref: '#/components/parameters/IdempotencyKey'}] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [expected_revision, tasks] + properties: + expected_revision: {type: integer, minimum: 1} + queue: {type: string} + tasks: + type: array + minItems: 1 + maxItems: 999 + items: + type: object + required: [name, kind, payload] + properties: + name: {type: string, minLength: 1, maxLength: 128} + deps: {type: array, items: {type: string}} + id: {type: string} + kind: {type: string, minLength: 1} + payload: {type: string, contentEncoding: base64} + queue: {type: string} + schema_version: {type: integer, minimum: 1} + responses: {'202': {description: Graft receipt and task batch accepted atomically}, '409': {description: Revision conflict}} + /workflows/{id}/retry: + post: + summary: Retry only a workflow's failed subgraph + parameters: [{$ref: '#/components/parameters/Id'}, {$ref: '#/components/parameters/IdempotencyKey'}] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [expected_revision] + properties: + expected_revision: {type: integer, minimum: 1} + recoveries: + type: array + maxItems: 999 + description: Explicit repair instructions for failed nodes that cannot be retried as-is. + items: + type: object + required: [node] + properties: + node: {type: string, minLength: 1, maxLength: 128} + payload: {type: string, contentEncoding: base64, description: Required when repairing an undecodable node.} + schema_version: {type: integer, minimum: 1, description: Required when repairing an undecodable node.} + release_quarantine: {type: boolean, default: false, description: Must be true to release a quarantined fingerprint.} + responses: {'200': {description: Accepted revision and generation}, '409': {description: Revision conflict}} + /workflows/{id}/cancel: + post: + summary: Cancel active workflow work and optionally linked children + parameters: [{$ref: '#/components/parameters/Id'}, {$ref: '#/components/parameters/IdempotencyKey'}] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: {propagate_children: {type: boolean, default: true}} + responses: {'200': {description: Bounded cancellation receipt}} + /healthz: get: summary: Liveness. Never touches the store. @@ -836,6 +1023,61 @@ components: POST, must not enqueue the job twice. Keys are retained long enough to cover a client retry window and the response is replayed verbatim. schemas: + WorkflowSummary: + type: object + required: [workflow_id, coordinator_job_id, state, enqueued_at_ms, scheduled_at_ms] + properties: + workflow_id: {type: string} + coordinator_job_id: {type: string} + state: {type: string} + enqueued_at_ms: {type: integer} + scheduled_at_ms: {type: integer} + finalized_at_ms: {type: integer} + WorkflowNode: + type: object + required: [name, job_id, kind, job_kind, state, dependencies, dependents] + properties: + name: {type: string} + job_id: {type: string} + kind: {type: string, enum: [task, signal, timer, child_workflow, condition]} + job_kind: {type: string} + state: {type: string, description: May be `missing` when retention removed an unfinished node before inspection.} + dependencies: {type: array, items: {type: string}} + dependents: {type: array, items: {type: string}} + signal: {type: string} + wake_at_ms: {type: integer} + delay_ms: {type: integer} + child_workflow_id: {type: string} + condition: {type: string} + completed_at_ms: {type: integer} + WorkflowSnapshot: + type: object + required: [workflow_id, coordinator_job_id, coordinator_state, revision, generation, failed, failed_subgraph_retry, nodes] + properties: + workflow_id: {type: string} + coordinator_job_id: {type: string} + coordinator_state: {type: string} + revision: {type: integer, minimum: 1} + generation: {type: integer, minimum: 1} + failed: {type: boolean} + failed_subgraph_retry: {type: boolean} + retry_policy: + type: object + required: [max_generations, backoff_ms] + properties: + max_generations: {type: integer, minimum: 1} + backoff_ms: {type: integer, minimum: 1} + nodes: {type: array, maxItems: 999, items: {$ref: '#/components/schemas/WorkflowNode'}} + WorkflowSignal: + type: object + required: [id, signal, idempotency_key, payload, source, recorded_at_ms] + properties: + id: {type: integer, minimum: 1} + signal: {type: string} + idempotency_key: {type: string} + payload: {description: Arbitrary JSON value supplied by the emitter, limited to 64 KiB after canonical serialization.} + source: {description: Caller-supplied JSON describing the emitter or calling system, limited to 16 KiB; it is not proof of identity by itself.} + recorded_at_ms: {type: integer, description: Timestamp assigned by the backing store.} State: type: string enum: [pending, scheduled, available, running, retryable, completed, archived, cancelled, quarantined, undecodable] diff --git a/conformance/CAPABILITY_REGISTER.md b/conformance/CAPABILITY_REGISTER.md index f969f44..e5492ce 100644 --- a/conformance/CAPABILITY_REGISTER.md +++ b/conformance/CAPABILITY_REGISTER.md @@ -120,12 +120,14 @@ not think to name. | Long-running task tracking | ✅ | **Round 32ab: futures/goroutines explicitly attached inside a handler are now owned by that attempt rather than detached from it.** Rust exposes `JobCtx::spawn_tracked` over an attempt-owned `JoinSet`; Go exposes `Track(ctx, func(context.Context) error)` over a context-bound, race-safe task group. Handler success closes registration and waits for every child BEFORE the success ack. The first child error fails the attempt and cancels siblings; handler error, timeout, and recovered panic cancel and join children before their outcome is acknowledged. Graceful shutdown therefore waits even after the user handler has returned. Lease loss calls `JobCtx::cancel` + `abort_all` in Rust and cancels the exact handler context in Go; the lost holder never acks. Rust's synchronous abort deliberately breaks the possible `JobCtx -> tracker -> future -> JobCtx` ownership cycle. Go is language-honest: cancellation is cooperative and code that ignores `ctx.Done()` cannot be killed, while fencing still blocks stale writes; gap 19 owns reporting that future stuck case. Late/outside registration returns `TrackedTaskClosed` / `ErrTaskTrackerClosed` / `ErrTaskTrackerUnavailable`, never a global fallback. Real live-loop tests hold tracked work behind a barrier, request shutdown, prove the runner CANNOT finish, then release it and require completion. Separate forced-renew-loss tests require child cancellation/drop, no post-cancel side effect, no ack, and the old row still `running`; error tests require retry rather than false success. **Mutation teeth:** deleting Rust's success join acknowledged a failed child as `success`; rebinding Go's tracker to `context.Background()` made the lease-loss cancellation witness time out. Focused Go tests pass `-race`. See `docs/tracked-tasks.md`. | | Task-local typed data (non-persisted) | ✅ | **Round 32y: implemented in both runtimes as two deliberately separate, process-local type maps.** Rust `Extensions` is keyed by `TypeId` and returns `Arc`; `WorkerConfig.extensions` is shared across the worker while every `JobCtx::from_claim` creates a fresh job map. `JobCtx::data` applies job-then-worker shadowing, with explicit `worker_data`, `job_data`, and `insert_data` APIs. Go mirrors this with a `reflect.Type`-keyed `Extensions`, generic `SetExtension`/`Extension`, `Config.Extensions`, and handler-context `Data[T]` / `WorkerData[T]` / `JobData[T]` / `SetJobData`; typed boxes preserve typed nils and a mutex covers every map access. Both real worker-loop tests force two concurrent jobs to insert the SAME concrete type before either reads it, then require each job's own value, the unchanged worker default, a wrong-type miss, terminal completion, and an envelope snapshot with no local marker. Container tests separately pin type replacement/removal and the Go outside-handler error. **Mutation teeth:** making the job map reuse the worker map failed both concurrency tests (and Go ran under `-race`). `docs/task-data.md` records attempt lifetime, shadowing, type-identity/newtype guidance, and that this is storage only—not the still-❌ handler-extractor API. | | Task aggregation / batch handlers | ✅ | **Typed execution chunks in both runtimes.** Rust `Registry::register_batch` and Go `RegisterBatchFunc` coalesce same-kind claims from one atomic admission call until a maximum size or absolute maximum delay, then invoke one handler. Each member keeps its own context, fence, lease, timeout, checkpoint, logs, rate weight and durable outcome; positional results allow partial success/retry and a result-count mismatch fails every member rather than silently dropping one. Panics are isolated and wake every waiter. Direct Store callers keep singleton units for compatibility; worker and test-drain paths form deterministic bounded units and run them concurrently. This is River/Oban-style execution batching, not Sidekiq workflow batches or asynq's synthetic aggregate-task replacement. See `docs/batch-handlers.md`. | -| **Workflows / DAG dependencies** | ✅ | **Round 32u: durable DAG dependency gating is implemented as the separate opt-in `headgate-workflow` crate and Go `workflow` package—fulfilling the architecture boundary without adding orchestration to core or changing admission.** Builders reject empty/duplicate names, missing/repeated dependencies and cycles before enqueue, then return one batch containing a durable coordinator and every application task in `pending`. The coordinator uses bounded point reads only, promotes roots, then fan-out/fan-in nodes only when every dependency is `completed`; it snoozes without consuming attempts while work is live. Failed or unrecorded missing upstream jobs cause still-pending descendants to be deleted before execution and archive the coordinator as the workflow-level failure record. **Workflow-aware retention hardening:** every child is retained for at least the workflow retention, and each observed completion is copied into the coordinator's fenced cursor before descendants are promoted. An early job row may therefore expire during a long retry without erasing the durable fact that it completed. Both languages prove short-retention clamping and missing-row recovery from recorded evidence. A live Postgres runtime test executes `extract -> {left,right} -> join` through the real worker/Store path and asserts the join is last. Go's independent coordinator test drives the same fan-out/fan-in state machine and proves an archived branch removes the pending join and settles failed. This row claims durable static DAG dependencies, not River Pro's signals, timers, CEL waits, retry, dynamic grafting/nesting, or graph UI; those remain explicitly outside this slice. See `docs/workflows.md`. | -| Workflow signals | ❌ | Experimenting on `codex/workflow-experiments`; no durable write path or public claim yet. | -| Workflow timers | ❌ | Experimenting with store-time semantics; worker-clock timers will not be accepted. | -| Workflow graph mutation | ❌ | Experimenting with additive, revision-checked grafts; in-place mutation of executed nodes is out of scope. | -| Nested workflows | ❌ | Experimenting with child coordinators as explicit parent nodes; failure and retry propagation remain undecided. | -| Workflow-level retry | ❌ | Experimenting with failed-subgraph retry while preserving successful ancestors; no control API claim yet. | +| **Workflows / DAG dependencies** | ✅ | **Round 32u: durable DAG dependency gating is implemented as the separate opt-in `headgate-workflow` crate and Go package—fulfilling the architecture boundary without adding orchestration to core or changing admission.** Builders reject empty/duplicate names, missing/repeated dependencies and cycles before enqueue, then return one batch containing a durable coordinator and every application task in `pending`. The coordinator uses bounded point reads only, promotes roots, then fan-out/fan-in nodes only when every dependency is `completed`; it snoozes without consuming an attempt while work is live. By default, failed or unrecorded missing upstream jobs remove still-pending descendants and archive the coordinator. Workflow retention clamps child retention, and store-stamped completion evidence is copied into the fenced cursor before descendants are promoted. The existing live PostgreSQL proof requires `extract -> {left,right} -> join` in order. This ✅ remains only the immutable base-DAG claim; dynamic features and CEL waits are tracked by the adjacent 🔶 experiment rows and documentation. See `docs/workflows.md`. | +| Workflow signals | ✅ | Durable buffered/idempotent signal jobs exist in Rust and Go. Rich SDK and `POST /workflows/{id}/signals` emissions durably retain the signal name, store timestamp, idempotency key, bounded JSON payload, and caller-supplied source metadata before unblocking work; replay returns the original record and conflicting key reuse fails. `GET /workflows/{id}/signals` and the workflow console expose the newest 100 records on PostgreSQL, Redis, and MySQL. Source identity is descriptive unless a deployment derives it at its authenticated upstream boundary. UI mutation controls are a separate deferred surface. | +| Workflow timers | ✅ | Absolute timers use ordinary scheduled jobs. Relative timers record dependency `finalized_at_ms` from store-stamped completion evidence and atomically schedule at the latest timestamp plus the delay; coordinator polling latency and worker clocks cannot move the anchor. The complete runtime path passed the six live backend/language cells. | +| Workflow conditional waits (CEL) | ✅ | Both SDKs compile bounded CEL boolean expressions before enqueue and evaluate them over typed `revision`, `generation`, `states`, and `completed` values. Conditions are ordinary pending internal jobs and cannot perform I/O. The CEL path passed inside all six live workflow matrix cells. | +| Workflow graph mutation | ✅ | Additive revision-checked grafts use one atomic deterministic receipt/task batch, validate the combined DAG, fence accepted nodes into the coordinator cursor, and record acceptance in bounded history. Authenticated/idempotency-key HTTP mutation routes exist. Accepted nodes and dependency edges remain immutable across revisions: grafts append ordinary tasks but never replace, rename, remove, or rewire existing work, and the coordinator must remain live. Grafts cannot add signals, timers, CEL conditions, or child-workflow links; those control-flow nodes must be declared in the initial graph because their delivery, store-clock, evaluation, cycle, cancellation, and retry rules need a separate versioned mutation contract. A materially different graph requires a new workflow ID. Once terminal, the execution's graph and recorded outcome are permanently immutable; explicitly preconfigured failed-subgraph retry advances a generation over the unchanged graph rather than mutating it. UI mutation controls are separate. | +| Workflow graph inspection | ✅ | Rust `list_workflows` / Go `ListWorkflows` provide bounded cursor pages without loading every graph. Rust `inspect_workflow` and Go `InspectWorkflow` return one bounded payload-free snapshot containing the accepted base graph plus grafts, coordinator/node execution state, revision, generation, retry policy, dependencies, reverse dependents, virtual-node configuration, and retained completion evidence. Both libraries provide snapshot methods and fresh-read node/dependency/dependent helpers. Matching authenticated GET routes expose the same list and topology without requiring console code or coordinator-payload parsing. | +| Nested workflows | ✅ | Child links support atomic `prepare_bundle` / `PrepareBundle`, complete-bundle cross-workflow cycle detection, bounded parent cancellation across all live branches, and failed-child retry propagation. Separately enqueued children remain supported without global cycle proof. | +| Workflow-level retry | ✅ | Manual and store-timed automatic failed-subgraph retry exist in both SDKs. Retry preserves successful ancestors, records generation/revision history, repairs quarantined nodes only after explicit fingerprint-wide release, repairs undecodable nodes only with replacement payload/schema, and propagates through failed child links. Authenticated/idempotency-key retry and cancellation routes exist; the combined retry path passed all six live cells. See `docs/workflow-experiments.md`. | ## Failure diff --git a/conformance/EVIDENCE.md b/conformance/EVIDENCE.md index af742ab..78e9609 100644 --- a/conformance/EVIDENCE.md +++ b/conformance/EVIDENCE.md @@ -696,6 +696,82 @@ NOTE: round 32ab. The graceful tests drive the real admission loop, hold a child - go: headgateworkflow/workflow_test.go::TestCoordinatorUsesDurableCompletionEvidenceAfterRetention NOTE: round 32u plus workflow-aware retention hardening. Both builders pin the durable batch shape and reject missing dependencies and cycles. They clamp short child retention to the workflow retention. The coordinator records observed completions in a fenced cursor before promoting descendants; both languages prove retained evidence still resolves a dependency after its ordinary job row disappears, while an unrecorded missing dependency remains a failure. The Rust live test enqueues one real coordinator plus four pending application jobs into Postgres, repeatedly runs the ordinary worker runtime, and requires `extract` first, `join` last, and both fan-out branches exactly once between them. Go independently drives the resolver through root promotion, two-way fan-out, and an archived branch; the still-pending join is deleted before execution and the workflow settles failed. The graph is static and bounded by coordinator payload size. Signals, timers, CEL waits, dynamic graph mutation, workflow retry, and graph UI are not claimed. +### Workflow signals +- rust: crates/headgate-api/tests/api.rs::workflow_control_routes_share_the_idempotency_boundary +- go: headgateapi/api_test.go::TestWorkflowMutationRoutesRequireIdempotencyKey +- rust: crates/headgate-workflow/src/lib.rs::signal_is_pending_work_and_early_completion_waits_for_dependencies +- rust: crates/headgate-workflow/tests/live.rs::live_postgres_buffers_and_idempotently_replays_workflow_signal +- rust: crates/headgate-workflow/tests/live.rs::workflow_experiments_postgres_matrix_cell +- rust: crates/headgate-workflow/tests/live.rs::workflow_experiments_redis_matrix_cell +- rust-mysql: crates/headgate-workflow/tests/live_mysql.rs::workflow_experiments_mysql_matrix_cell +- go: headgateworkflow/workflow_test.go::TestSignalEmissionIsDurableBufferedAndIdempotent +- go: headgateworkflow/live_matrix_test.go::TestWorkflowExperimentsPostgresMatrixCell +- go: headgateworkflow/live_matrix_test.go::TestWorkflowExperimentsRedisMatrixCell +- go-mysql: headgateworkflow/live_mysql_test.go::TestWorkflowExperimentsMySQLMatrixCell +NOTE: both SDKs use a retained pending signal job, buffer early delivery, and make replay idempotent. `TestSignalEmissionIsDurableBufferedAndIdempotent` and the two API boundary tests require a store-timestamped emission containing the idempotency key, JSON payload, and caller-supplied JSON source; replay returns the original record, conflicting content is rejected, and the bounded newest-first list reads it back. The shared matrix scenario combines early signal delivery with CEL, a relative timer, automatic retry, and durable history. On 2026-09-04 all six PostgreSQL/Redis/MySQL × Rust/Go cells ran live and passed; migration v13 adds the portable signal event store used by the richer contract. Payload is bounded to 64 KiB, source to 16 KiB, and each workflow retains 100 emissions. + +### Workflow timers +- rust: crates/headgate-workflow/src/lib.rs::timer_uses_absolute_schedule_and_buffers_until_dependencies_complete +- go: headgateworkflow/workflow_test.go::TestTimerUsesStoreScheduleAndBuffersUntilDependenciesComplete +- go: headgateworkflow/workflow_test.go::TestRelativeTimerCheckpointsBeforeStoreTimeSnooze +- rust: crates/headgate-workflow/src/lib.rs::relative_timer_anchors_to_dependency_completion +- rust: crates/headgate-workflow/tests/live.rs::workflow_experiments_postgres_matrix_cell +- rust: crates/headgate-workflow/tests/live.rs::workflow_experiments_redis_matrix_cell +- rust-mysql: crates/headgate-workflow/tests/live_mysql.rs::workflow_experiments_mysql_matrix_cell +- go: headgateworkflow/live_matrix_test.go::TestWorkflowExperimentsPostgresMatrixCell +- go: headgateworkflow/live_matrix_test.go::TestWorkflowExperimentsRedisMatrixCell +- go-mysql: headgateworkflow/live_mysql_test.go::TestWorkflowExperimentsMySQLMatrixCell +NOTE: relative timers anchor to the latest dependency's store-stamped finalization and use the narrow pending-to-scheduled store operation. The six-cell scenario covers the complete runtime path and passed live on 2026-09-04. Its first Redis runs failed because the new Lua branch read ARGV[3]/ARGV[4] instead of ARGV[2]/ARGV[3], then omitted the future-score admission route; both failures are now regression-covered by the matrix. + +### Workflow conditional waits (CEL) +- rust: crates/headgate-workflow/src/lib.rs::automatic_retry_policy_and_cel_condition_are_validated +- rust: crates/headgate-workflow/tests/live.rs::workflow_experiments_postgres_matrix_cell +- rust: crates/headgate-workflow/tests/live.rs::workflow_experiments_redis_matrix_cell +- rust-mysql: crates/headgate-workflow/tests/live_mysql.rs::workflow_experiments_mysql_matrix_cell +- go: headgateworkflow/workflow_test.go::TestAutomaticRetryPolicyAndCELConditionAreValidated +- go: headgateworkflow/live_matrix_test.go::TestWorkflowExperimentsPostgresMatrixCell +- go: headgateworkflow/live_matrix_test.go::TestWorkflowExperimentsRedisMatrixCell +- go-mysql: headgateworkflow/live_mysql_test.go::TestWorkflowExperimentsMySQLMatrixCell +NOTE: both implementations reject malformed or oversized expressions before enqueue, expose the same four typed variables, require a boolean result, and promote a condition as an ordinary pending internal job. The condition path passed in every live matrix cell on 2026-09-04. + +### Workflow graph mutation +- rust: crates/headgate-api/tests/api.rs::workflow_control_routes_share_the_idempotency_boundary +- go: headgateapi/api_test.go::TestWorkflowMutationRoutesRequireIdempotencyKey +- rust: crates/headgate-workflow/src/lib.rs::revisioned_graft_prepares_one_atomic_receipt_and_pending_tasks +- rust: crates/headgate-workflow/tests/live.rs::live_postgres_accepts_one_revisioned_workflow_graft_atomically +- go: headgateworkflow/workflow_test.go::TestRevisionedGraftPersistsGraphBeforePromotingReceipt +- go: headgateworkflow/workflow_test.go::TestInvalidCombinedGraftIsRemovedWithoutAdvancingRevision +NOTE: both SDKs derive the same deterministic next-revision receipt, atomically enqueue it with pending tasks, validate the combined DAG, and persist acceptance before promotion. The authenticated/idempotency-key HTTP route exists and acceptance enters bounded history. Grafts remain additive ordinary-task mutations on a live coordinator; that is the claimed boundary, not missing evidence. + +### Workflow graph inspection +- rust: crates/headgate-workflow/src/lib.rs::workflow_snapshot_answers_topology_queries +- rust: crates/headgate-api/tests/api.rs::workflow_control_routes_share_the_idempotency_boundary +- go: headgateworkflow/workflow_test.go::TestInspectWorkflowReturnsTopologyAndExecutionState +- go: headgateapi/api_test.go::TestWorkflowInspectionRoutesAreReadOnlyAndBoundedToKnownNodes +- go: headgateapi/api_test.go::TestWorkflowInspectionRoutesReturnTopologyWithoutPayloads +NOTE: both workflow libraries expose the accepted graph and reverse topology without payloads, and their dependency/dependent lookups fail explicitly for unknown nodes. The Rust live API proof reads an enqueued graph and its dependency route; the Go route proofs pin read-only access, 404 behavior, graph/node/dependency/dependent response shapes, while the package test checks revision, generation, completion time, state, dependencies, and dependents. + +### Workflow-level retry +- rust: crates/headgate-workflow/src/lib.rs::failed_subgraph_retry_is_explicit_in_the_coordinator_payload +- rust: crates/headgate-workflow/tests/live.rs::live_postgres_retries_only_the_failed_workflow_subgraph +- go: headgateworkflow/workflow_test.go::TestFailedSubgraphRetryPreservesSuccessAndReopensOnlyFailure +- go: headgateworkflow/workflow_test.go::TestFailedSubgraphRetryRequiresExplicitTerminalRecovery +- go: headgateapi/api_test.go::TestWorkflowMutationRoutesRequireIdempotencyKey +- rust: crates/headgate-workflow/src/lib.rs::automatic_retry_policy_and_cel_condition_are_validated +- rust: crates/headgate-workflow/tests/live.rs::workflow_experiments_postgres_matrix_cell +- go: headgateworkflow/workflow_test.go::TestAutomaticRetryPolicyAndCELConditionAreValidated +- go: headgateworkflow/live_matrix_test.go::TestWorkflowExperimentsPostgresMatrixCell +NOTE: manual and automatic retries preserve completed ancestors, advance revision/generation, and use deterministic receipts. Recovery requires explicit quarantine release or a replacement undecodable payload/schema, and child-link retries propagate. The route and bounded history exist. The automatic retry path passed all six live cells on 2026-09-04, including terminal history surviving cursor-step completion. + +### Nested workflows +- rust: crates/headgate-workflow/src/lib.rs::child_workflow_is_an_explicit_pending_node +- rust: crates/headgate-workflow/tests/live.rs::live_postgres_parent_waits_for_child_workflow +- go: headgateworkflow/workflow_test.go::TestChildWorkflowNodeMirrorsCoordinatorTerminalState +- rust: crates/headgate-workflow/src/lib.rs::atomic_bundle_rejects_cross_workflow_cycles +- go: headgateworkflow/workflow_test.go::TestAtomicBundleRejectsCrossWorkflowCycles +- go: headgateworkflow/workflow_test.go::TestCancelWorkflowPropagatesToChildrenAndAllLiveBranches +NOTE: explicit child-link jobs mirror the child coordinator. Atomic bundles require every child, reject cross-workflow cycles, and return one enqueue batch. Cancellation traverses all live parent branches and linked children by default; failed-link retry requests child retry first. These SDK/runtime semantics are the claimed capability; UI controls are tracked separately. + ### Death handler - rust: crates/headgate/tests/death_handler.rs::death_handler_runs_once_only_after_the_archive_is_durable - go: death_handler_test.go::TestDeathHandlerRunsOnceOnlyAfterArchiveIsDurable diff --git a/conformance/TEST_INVENTORY.tsv b/conformance/TEST_INVENTORY.tsv index 41d17ec..3e4259b 100644 --- a/conformance/TEST_INVENTORY.tsv +++ b/conformance/TEST_INVENTORY.tsv @@ -5,7 +5,7 @@ # deliberate hand-edit, because that is what deleting a test is. scenario conformance/scenarios/admission.yaml 9 rust crates/headgate-api/src/lib.rs 2 -rust crates/headgate-api/tests/api.rs 10 +rust crates/headgate-api/tests/api.rs 11 rust crates/headgate-core/src/lib.rs 26 rust crates/headgate-crypto/src/lib.rs 3 rust crates/headgate-crypto/tests/live.rs 1 @@ -42,8 +42,9 @@ rust crates/headgate-testkit/tests/database_postgres.rs 1 rust crates/headgate-testkit/tests/database_redis.rs 1 rust crates/headgate-testkit/tests/memstore.rs 10 rust crates/headgate-ui/tests/ui.rs 4 -rust crates/headgate-workflow/src/lib.rs 5 -rust crates/headgate-workflow/tests/live.rs 1 +rust crates/headgate-workflow/src/lib.rs 14 +rust crates/headgate-workflow/tests/live.rs 7 +rust crates/headgate-workflow/tests/live_mysql.rs 1 rust crates/headgate/src/circuit_breaker.rs 5 rust crates/headgate/src/client.rs 2 rust crates/headgate/src/isolated.rs 6 @@ -95,7 +96,7 @@ go go/driver/headgateredis/inspect_test.go 2 go go/driver/headgateredis/store_test.go 8 go go/enqueue_middleware_test.go 4 go go/fingerprint_test.go 2 -go go/headgateapi/api_test.go 18 +go go/headgateapi/api_test.go 19 go go/headgatecrypto/crypto_test.go 3 go go/headgatectl/main_test.go 1 go go/headgatemigrate/cmd/hg-migrate/main_test.go 5 @@ -116,7 +117,9 @@ go go/headgatetest/memstore_test.go 11 go go/headgatetest/task_data_test.go 1 go go/headgatetest/tracked_tasks_test.go 5 go go/headgateui/ui_test.go 5 -go go/headgateworkflow/workflow_test.go 6 +go go/headgateworkflow/live_matrix_test.go 2 +go go/headgateworkflow/live_mysql_test.go 1 +go go/headgateworkflow/workflow_test.go 18 go go/insert_hook_test.go 4 go go/log_test.go 4 go go/output_test.go 2 diff --git a/conformance/state_machine.yaml b/conformance/state_machine.yaml index 1ad9d63..e0582b5 100644 --- a/conformance/state_machine.yaml +++ b/conformance/state_machine.yaml @@ -29,10 +29,12 @@ transitions: - {from: running, on: checkpoint_stale, to: undecodable} - {from: archived, on: operator_retry, to: available} - {from: cancelled, on: operator_retry, to: available} + - {from: undecodable, on: operator_retry, to: available} - {from: quarantined, on: operator_release, to: available} - {from: available, on: operator_cancel, to: cancelled} - {from: scheduled, on: operator_cancel, to: cancelled} - {from: pending, on: operator_cancel, to: cancelled} + - {from: retryable, on: operator_cancel, to: cancelled} - {from: running, on: operator_cancel, to: cancelled} invariants: diff --git a/crates/headgate-api/Cargo.toml b/crates/headgate-api/Cargo.toml index e949e34..da935b6 100644 --- a/crates/headgate-api/Cargo.toml +++ b/crates/headgate-api/Cargo.toml @@ -18,6 +18,7 @@ license.workspace = true headgate-core = { workspace = true } # schedule_spec: the API validates specs and computes a schedule's first next_run. headgate = { workspace = true } +headgate-workflow = { workspace = true } axum = "0.8" futures-util = { version = "0.3", default-features = false, features = ["std"] } tokio = { version = "1", features = ["time", "rt-multi-thread", "macros", "net"] } diff --git a/crates/headgate-api/src/lib.rs b/crates/headgate-api/src/lib.rs index f237ae7..70a1248 100644 --- a/crates/headgate-api/src/lib.rs +++ b/crates/headgate-api/src/lib.rs @@ -15,7 +15,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use axum::extract::DefaultBodyLimit; use axum::extract::rejection::{JsonRejection, QueryRejection}; use axum::extract::{FromRequest, FromRequestParts, Path, Query, State}; -use axum::http::{HeaderValue, Method, StatusCode}; +use axum::http::{HeaderMap, HeaderValue, Method, StatusCode}; use axum::response::{IntoResponse, Response}; use axum::routing::{delete, get, post, put}; use axum::{Extension, Json, Router}; @@ -130,6 +130,25 @@ pub fn router(store: Arc, cfg: ApiConfig) -> Router { .route("/workers", get(workers)) .route("/cluster", get(cluster)) .route("/workers/{worker_id}/signal", post(signal_worker)) + .route("/workflows", get(workflow_list)) + .route("/workflows/{id}", get(workflow_detail)) + .route("/workflows/{id}/events", get(workflow_events)) + .route("/workflows/{id}/nodes/{node}", get(workflow_node)) + .route( + "/workflows/{id}/nodes/{node}/dependencies", + get(workflow_dependencies), + ) + .route( + "/workflows/{id}/nodes/{node}/dependents", + get(workflow_dependents), + ) + .route( + "/workflows/{id}/signals", + get(workflow_signals).post(workflow_signal), + ) + .route("/workflows/{id}/grafts", post(workflow_graft)) + .route("/workflows/{id}/retry", post(workflow_retry)) + .route("/workflows/{id}/cancel", post(workflow_cancel)) .route("/events", get(events)) .route("/rate-classes", get(rate_classes)) .route("/rate-classes/{name}", put(put_rate_class)) @@ -398,6 +417,7 @@ fn client_error(error: headgate::ClientError) -> Response { } } +#[allow(clippy::result_large_err)] // Axum handlers use a concrete Response as the uniform API error. fn authorize_http_enqueue( state: &ApiState, identity: Option>, @@ -410,6 +430,285 @@ fn authorize_http_enqueue( type ApiResult = Result; +fn workflow_err(error: headgate_workflow::WorkflowError) -> Response { + let message = error.to_string(); + let status = if message.contains("was not found") { + StatusCode::NOT_FOUND + } else if message.contains("revision conflict") { + StatusCode::CONFLICT + } else { + StatusCode::BAD_REQUEST + }; + err_response(status, &message) +} + +#[derive(Deserialize)] +struct WorkflowSignalBody { + signal: String, + #[serde(default)] + payload: serde_json::Value, + #[serde(default = "default_signal_source")] + source: serde_json::Value, +} + +fn default_signal_source() -> serde_json::Value { + serde_json::json!({}) +} + +async fn workflow_signal( + State(s): State, + Path(id): Path, + headers: HeaderMap, + ApiJson(body): ApiJson, +) -> ApiResult { + let key = headers + .get("idempotency-key") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default(); + let receipt = headgate_workflow::emit_signal_with( + s.store.as_ref(), + &id, + headgate_workflow::SignalEmission { + signal: body.signal, + idempotency_key: key.into(), + payload: body.payload, + source: body.source, + }, + ) + .await + .map_err(workflow_err)?; + Ok(Json(json!({ "matched": receipt.matched, "promoted": receipt.promoted, "inserted": receipt.inserted, "emission": receipt.emission })).into_response()) +} + +#[derive(Deserialize)] +struct WorkflowSignalsQuery { + cursor: Option, + limit: Option, +} + +async fn workflow_signals( + State(s): State, + Path(id): Path, + Query(query): Query, +) -> ApiResult { + let limit = query.limit.unwrap_or(100); + let signals = headgate_workflow::list_signals(s.store.as_ref(), &id, query.cursor, limit) + .await + .map_err(workflow_err)?; + let next_cursor = (signals.len() == limit as usize) + .then(|| signals.last().map(|signal| signal.id)) + .flatten(); + Ok(Json(json!({"signals": signals, "next_cursor": next_cursor})).into_response()) +} + +#[derive(Deserialize)] +struct WorkflowRetryBody { + expected_revision: u64, + #[serde(default)] + recoveries: Vec, +} + +#[derive(Deserialize)] +struct WorkflowRecoveryBody { + node: String, + #[serde(default)] + payload: Option, + #[serde(default)] + schema_version: Option, + #[serde(default)] + release_quarantine: bool, +} + +#[allow(clippy::result_large_err)] // Keeps base64 validation on the same concrete API error path. +async fn workflow_retry( + State(s): State, + Path(id): Path, + ApiJson(body): ApiJson, +) -> ApiResult { + let b64 = base64::engine::general_purpose::STANDARD; + let recoveries = body + .recoveries + .into_iter() + .map(|recovery| { + let payload = recovery + .payload + .map(|payload| { + b64.decode(payload).map_err(|_| { + err_response(StatusCode::BAD_REQUEST, "recovery payload must be base64") + }) + }) + .transpose()?; + Ok(headgate_workflow::WorkflowRecovery { + node: recovery.node, + payload, + schema_version: recovery.schema_version, + release_quarantine: recovery.release_quarantine, + }) + }) + .collect::, Response>>()?; + let receipt = headgate_workflow::request_failed_subgraph_retry_with_recovery( + s.store.as_ref(), + &id, + body.expected_revision, + &recoveries, + ) + .await + .map_err(workflow_err)?; + Ok( + Json(json!({ "revision": receipt.revision, "generation": receipt.generation })) + .into_response(), + ) +} + +#[derive(Deserialize)] +struct WorkflowCancelBody { + #[serde(default = "default_true")] + propagate_children: bool, +} + +const fn default_true() -> bool { + true +} + +async fn workflow_cancel( + State(s): State, + Path(id): Path, + ApiJson(body): ApiJson, +) -> ApiResult { + let receipt = + headgate_workflow::cancel_workflow(s.store.as_ref(), &id, body.propagate_children) + .await + .map_err(workflow_err)?; + Ok(Json(json!({ "workflows": receipt.workflows, "jobs": receipt.jobs })).into_response()) +} + +async fn workflow_events(State(s): State, Path(id): Path) -> ApiResult { + let events = headgate_workflow::workflow_events(s.store.as_ref(), &id) + .await + .map_err(workflow_err)?; + Ok(Json(json!({ "events": events })).into_response()) +} + +async fn workflow_detail(State(s): State, Path(id): Path) -> ApiResult { + let snapshot = headgate_workflow::inspect_workflow(s.store.as_ref(), &id) + .await + .map_err(workflow_err)?; + Ok(Json(snapshot).into_response()) +} + +#[derive(Deserialize)] +struct WorkflowListParams { + cursor: Option, + limit: Option, +} + +async fn workflow_list( + State(s): State, + ApiQuery(params): ApiQuery, +) -> ApiResult { + let page = headgate_workflow::list_workflows( + s.store.as_ref(), + params.cursor.as_deref(), + params.limit.unwrap_or(50), + ) + .await + .map_err(workflow_err)?; + Ok(Json(page).into_response()) +} + +async fn workflow_node( + State(s): State, + Path((id, node)): Path<(String, String)>, +) -> ApiResult { + let node = headgate_workflow::workflow_node(s.store.as_ref(), &id, &node) + .await + .map_err(workflow_err)?; + Ok(Json(node).into_response()) +} + +async fn workflow_dependencies( + State(s): State, + Path((id, node)): Path<(String, String)>, +) -> ApiResult { + let dependencies = headgate_workflow::workflow_dependencies(s.store.as_ref(), &id, &node) + .await + .map_err(workflow_err)?; + Ok(Json(json!({ "dependencies": dependencies })).into_response()) +} + +async fn workflow_dependents( + State(s): State, + Path((id, node)): Path<(String, String)>, +) -> ApiResult { + let dependents = headgate_workflow::workflow_dependents(s.store.as_ref(), &id, &node) + .await + .map_err(workflow_err)?; + Ok(Json(json!({ "dependents": dependents })).into_response()) +} + +#[derive(Deserialize)] +struct WorkflowGraftBody { + expected_revision: u64, + #[serde(default)] + queue: Option, + tasks: Vec, +} + +#[derive(Deserialize)] +struct WorkflowGraftTaskBody { + name: String, + #[serde(default)] + deps: Vec, + kind: String, + payload: String, + #[serde(default)] + id: Option, + #[serde(default)] + queue: Option, + #[serde(default)] + schema_version: Option, +} + +async fn workflow_graft( + State(s): State, + identity: Option>, + Path(id): Path, + ApiJson(body): ApiJson, +) -> ApiResult { + let mut graft = headgate_workflow::WorkflowGraft::new(&id, body.expected_revision); + if let Some(queue) = body.queue { + graft = graft.queue(queue); + } + let b64 = base64::engine::general_purpose::STANDARD; + for task in body.tasks { + let payload = b64 + .decode(task.payload) + .map_err(|_| err_response(StatusCode::BAD_REQUEST, "payload must be base64"))?; + let kind = task.kind; + let envelope = Envelope { + id: task.id.unwrap_or_default(), + fingerprint: fingerprint(&kind, &payload), + kind, + payload, + queue: task.queue.unwrap_or_default(), + schema_version: task.schema_version.unwrap_or(1), + ..Default::default() + }; + graft = graft.add(task.name, envelope, task.deps); + } + let batch = graft.prepare().map_err(workflow_err)?; + let context = headgate::EnqueueContext::http(identity.map(|Extension(value)| value)); + s.producer + .enqueue_with_context(&context, &batch) + .await + .map_err(client_error)?; + Ok(( + StatusCode::ACCEPTED, + Json(json!({ "receipt_id": batch[0].id })), + ) + .into_response()) +} + // ---------- handlers ---------- async fn readyz(State(s): State) -> ApiResult { @@ -454,6 +753,7 @@ fn default_control_page_limit() -> usize { 200 } +#[allow(clippy::result_large_err)] // Axum handlers consume this concrete response directly. fn control_page(length: usize, query: &ControlPageQuery) -> Result<(usize, usize), Response> { if query.limit == 0 || query.limit > 200 { return Err(err_response( @@ -1190,7 +1490,7 @@ async fn put_concurrency_limit( )); } let on_saturated = - SaturationStrategy::try_from(body.on_saturated.as_str()).map_err(|e| store_err(e))?; + SaturationStrategy::try_from(body.on_saturated.as_str()).map_err(store_err)?; s.store .upsert_concurrency_limit(&ConcurrencyLimitConfig { name, diff --git a/crates/headgate-api/tests/api.rs b/crates/headgate-api/tests/api.rs index cc95dba..88664c0 100644 --- a/crates/headgate-api/tests/api.rs +++ b/crates/headgate-api/tests/api.rs @@ -11,10 +11,165 @@ use headgate_core::{ ProgressStore, ProgressUpdate, Schedule, Store, }; use headgate_postgres::PgStore; +use headgate_workflow::Workflow; use http_body_util::BodyExt; use serde_json::{Value, json}; use tower::ServiceExt; +#[tokio::test] +async fn workflow_control_routes_share_the_idempotency_boundary() { + let Ok(conninfo) = std::env::var("HG_TEST_PG") else { + eprintln!("HG_TEST_PG not set; skipping workflow control API proof"); + return; + }; + let store = Arc::new(PgStore::connect(&conninfo, 2).expect("connect")); + let suffix = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let workflow_id = format!("api-workflow-{suffix}"); + let queue = format!("api-workflow-{suffix}"); + let batch = Workflow::new(&workflow_id) + .coordinator_queue(&queue) + .add( + "prepare", + Envelope { + kind: "api.workflow.prepare".into(), + payload: br#"{"secret":"not-in-graph-response"}"#.to_vec(), + queue: queue.clone(), + ..Default::default() + }, + Vec::::new(), + ) + .add_signal("approval", "approved", ["prepare"]) + .prepare() + .unwrap(); + store.enqueue(&batch).await.unwrap(); + let app = router(store as Arc, ApiConfig::default()); + + let (status, workflows) = call_with_key( + &app, + Method::GET, + "/api/v1/workflows?limit=50", + None, + "unused-on-read", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!( + workflows["workflows"] + .as_array() + .unwrap() + .iter() + .any(|workflow| workflow["workflow_id"] == workflow_id) + ); + + let (status, snapshot) = call_with_key( + &app, + Method::GET, + &format!("/api/v1/workflows/{workflow_id}"), + None, + "unused-on-read", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(snapshot["nodes"].as_array().unwrap().len(), 2); + assert!(!snapshot.to_string().contains("not-in-graph-response")); + let (status, dependencies) = call_with_key( + &app, + Method::GET, + &format!("/api/v1/workflows/{workflow_id}/nodes/approval/dependencies"), + None, + "unused-on-read", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(dependencies["dependencies"][0]["name"], "prepare"); + + let request = Request::builder() + .method(Method::POST) + .uri(format!("/api/v1/workflows/{workflow_id}/signals")) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(json!({"signal": "approved"}).to_string())) + .unwrap(); + assert_eq!( + app.clone().oneshot(request).await.unwrap().status(), + StatusCode::BAD_REQUEST + ); + + let (status, receipt) = call_with_key( + &app, + Method::POST, + &format!("/api/v1/workflows/{workflow_id}/signals"), + Some(json!({ + "signal": "approved", + "payload": {"approved": true, "reviewer": "Ada"}, + "source": {"emitter": "admin-console", "actor": "operator-42"} + })), + "workflow-signal-1", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(receipt["matched"], 1); + assert_eq!(receipt["inserted"], true); + assert_eq!(receipt["emission"]["idempotency_key"], "workflow-signal-1"); + assert_eq!(receipt["emission"]["payload"]["reviewer"], "Ada"); + assert_eq!(receipt["emission"]["source"]["actor"], "operator-42"); + + let (status, history) = call_with_key( + &app, + Method::GET, + &format!("/api/v1/workflows/{workflow_id}/signals?limit=100"), + None, + "unused-on-read", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(history["signals"].as_array().unwrap().len(), 1); + assert_eq!(history["signals"][0], receipt["emission"]); + + let (status, replay) = call_with_key( + &app, + Method::POST, + &format!("/api/v1/workflows/{workflow_id}/signals"), + Some(json!({ + "signal": "approved", + "payload": {"approved": true, "reviewer": "Ada"}, + "source": {"emitter": "admin-console", "actor": "operator-42"} + })), + "workflow-signal-1", + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(replay["inserted"], false); + assert_eq!(replay["emission"], receipt["emission"]); + + let (status, graft) = call_with_key( + &app, + Method::POST, + &format!("/api/v1/workflows/{workflow_id}/grafts"), + Some(json!({ + "expected_revision": 1, + "queue": queue, + "tasks": [{"name": "audit", "kind": "api.workflow.audit", "payload": "e30="}] + })), + "workflow-graft-1", + ) + .await; + assert_eq!(status, StatusCode::ACCEPTED); + assert_eq!(graft["receipt_id"], format!("{workflow_id}:graft:2")); + + let (status, _) = call_with_key( + &app, + Method::POST, + &format!("/api/v1/workflows/{workflow_id}/cancel"), + Some(json!({"propagate_children": true})), + "workflow-cancel-1", + ) + .await; + assert_eq!(status, StatusCode::OK); +} + async fn call_with_key( app: &axum::Router, method: Method, @@ -335,8 +490,10 @@ async fn enqueue_outage_is_service_unavailable_not_a_bad_request() { }) .unwrap(), ); - let mut config = ApiConfig::default(); - config.enqueue_circuit_breaker = Some(breaker.clone()); + let config = ApiConfig { + enqueue_circuit_breaker: Some(breaker.clone()), + ..ApiConfig::default() + }; let app = router(inspect, config); let (status, body) = call_with_key( &app, @@ -403,8 +560,10 @@ async fn enqueue_authorization_guards_http_and_periodic_paths() { }, ); let inspect: Arc = store.clone(); - let mut config = ApiConfig::default(); - config.enqueue_authorizer = authorizer; + let config = ApiConfig { + enqueue_authorizer: authorizer, + ..ApiConfig::default() + }; let app = router(inspect, config).layer(axum::Extension(headgate::EnqueueIdentity::new( "service:mailer", ))); @@ -1349,10 +1508,10 @@ async fn a_real_workers_polling_is_the_number_that_reaches_cluster() { let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(secs); loop { let ws = store.list_workers(900_000).await.unwrap(); - if let Some(w) = ws.into_iter().find(|w| w.worker_id == wid) { - if cond(&w) { - return w; - } + if let Some(w) = ws.into_iter().find(|w| w.worker_id == wid) + && cond(&w) + { + return w; } assert!( tokio::time::Instant::now() < deadline, diff --git a/crates/headgate-core/src/lib.rs b/crates/headgate-core/src/lib.rs index 0acd771..702234b 100644 --- a/crates/headgate-core/src/lib.rs +++ b/crates/headgate-core/src/lib.rs @@ -298,12 +298,17 @@ pub fn lifecycle_transition(from: State, ev: LifecycleEvent) -> Option { (State::Retryable, LifecycleEvent::BackoffDue) => Some(State::Available), // step replay a resumed job whose step set changed under it must NOT silently restart (State::Running, LifecycleEvent::CheckpointStale) => Some(State::Undecodable), - (State::Archived | State::Cancelled, LifecycleEvent::OperatorRetry) => { - Some(State::Available) - } + ( + State::Archived | State::Cancelled | State::Undecodable, + LifecycleEvent::OperatorRetry, + ) => Some(State::Available), (State::Quarantined, LifecycleEvent::OperatorRelease) => Some(State::Available), ( - State::Pending | State::Available | State::Scheduled | State::Running, + State::Pending + | State::Available + | State::Scheduled + | State::Running + | State::Retryable, LifecycleEvent::OperatorCancel, ) => Some(State::Cancelled), _ => None, @@ -1200,6 +1205,15 @@ pub fn validate_schedule_event_limit(limit: u32) -> Result<(), StoreError> { } } +pub fn validate_durable_event_limit(limit: u32) -> Result<(), StoreError> { + if limit == 0 || limit > DURABLE_EVENT_LIMIT { + return Err(StoreError::Invalid( + "durable event limit must be between 1 and 100".into(), + )); + } + Ok(()) +} + pub struct RateClassState { pub name: String, pub tokens_available: i64, @@ -1359,6 +1373,23 @@ pub struct ScheduleEvent { pub recorded_at_ms: i64, } +pub const DURABLE_EVENT_LIMIT: u32 = 100; +pub const MAX_DURABLE_EVENT_PAYLOAD_BYTES: usize = 64 * 1024; +pub const MAX_DURABLE_EVENT_SOURCE_BYTES: usize = 16 * 1024; + +/// One bounded, store-timestamped fact attached to an application scope. Payload and +/// source contain valid JSON so every backend preserves the same value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DurableEvent { + pub event_id: u64, + pub scope: String, + pub topic: String, + pub idempotency_key: String, + pub payload: Vec, + pub source: Vec, + pub recorded_at_ms: i64, +} + #[derive(Clone, Debug, Default)] pub struct WorkerMeta { pub worker_id: String, @@ -1522,7 +1553,8 @@ pub trait Inspect: Store { /// available (`operator_release`) and new enqueues are accepted again. Returns how /// many jobs were released. A released job re-quarantines on its next crash. async fn quarantine_release(&self, fingerprint: &str) -> Result; - /// `archived|cancelled → available` (`operator_retry`). Any other state is an + /// `archived|cancelled|undecodable → available` (`operator_retry`). Undecodable + /// jobs must have their payload/schema repaired before this call. Any other state is an /// error — the transition table defines exactly which rows exist. async fn operator_retry(&self, id: &str) -> Result<(), StoreError>; /// `scheduled|available|running → cancelled` (`operator_cancel`). Cancelling a @@ -1531,6 +1563,16 @@ pub trait Inspect: Store { async fn operator_cancel(&self, id: &str) -> Result<(), StoreError>; /// `pending -> available`. No timer or dependency watcher may perform this change. async fn promote_job(&self, id: &str) -> Result<(), StoreError>; + /// `pending -> scheduled` at an absolute store-clock timestamp. Workflow-relative + /// timers use this after reading the store-stamped completion time of every + /// dependency; implementations must perform the state and timestamp change as one + /// operation. The default keeps third-party inspection adapters source-compatible + /// while making lack of this capability explicit at runtime. + async fn schedule_pending_job(&self, _id: &str, _at_ms: i64) -> Result<(), StoreError> { + Err(StoreError::Invalid( + "scheduling a pending job is not supported by this backend".into(), + )) + } /// Delete a non-running job. Deleting mid-flight is refused (asynq's rule). async fn delete_job(&self, id: &str) -> Result<(), StoreError>; async fn explain_admission(&self, id: &str) -> Result, StoreError>; @@ -1589,6 +1631,29 @@ pub trait Inspect: Store { limit: u32, ) -> Result, StoreError>; + /// Append one idempotent event and retain at most [`DURABLE_EVENT_LIMIT`] events + /// per scope. The bool is true only when this call inserted the returned record. + async fn append_durable_event( + &self, + _event: &DurableEvent, + ) -> Result<(DurableEvent, bool), StoreError> { + Err(StoreError::Invalid( + "durable events are not supported by this backend".into(), + )) + } + + /// Newest first. `limit` must be in `1..=DURABLE_EVENT_LIMIT`. + async fn list_durable_events( + &self, + _scope: &str, + _before_event_id: Option, + _limit: u32, + ) -> Result, StoreError> { + Err(StoreError::Invalid( + "durable events are not supported by this backend".into(), + )) + } + // ----- worker registry + surveyed policy behavior server->worker control channel ----- /// Upsert the worker row and return any pending operator COMMAND for it — the @@ -2661,7 +2726,7 @@ mod tests { // Pinned on purpose: adding or removing a transition must be deliberate — the // yaml's own invariant requires a conformance scenario per new row. assert_eq!( - rows, 23, + rows, 25, "state_machine.yaml row count changed; update the table AND its scenarios" ); } diff --git a/crates/headgate-crypto/tests/live.rs b/crates/headgate-crypto/tests/live.rs index 8511ff9..d3dc5e7 100644 --- a/crates/headgate-crypto/tests/live.rs +++ b/crates/headgate-crypto/tests/live.rs @@ -51,7 +51,10 @@ async fn live_store_holds_ciphertext_while_handler_receives_plaintext() { .windows(plaintext.len()) .any(|w| w == plaintext) ); - store.enqueue(&[encrypted.clone()]).await.unwrap(); + store + .enqueue(std::slice::from_ref(&encrypted)) + .await + .unwrap(); let stored = store .get_job(&id, true) .await diff --git a/crates/headgate-migrate/migrations/mysql/0013_durable_events.down.sql b/crates/headgate-migrate/migrations/mysql/0013_durable_events.down.sql new file mode 100644 index 0000000..106c075 --- /dev/null +++ b/crates/headgate-migrate/migrations/mysql/0013_durable_events.down.sql @@ -0,0 +1,2 @@ +DROP TABLE headgate_durable_event; +DROP TABLE headgate_durable_event_scope; diff --git a/crates/headgate-migrate/migrations/mysql/0013_durable_events.up.sql b/crates/headgate-migrate/migrations/mysql/0013_durable_events.up.sql new file mode 100644 index 0000000..4eceb1d --- /dev/null +++ b/crates/headgate-migrate/migrations/mysql/0013_durable_events.up.sql @@ -0,0 +1,17 @@ +CREATE TABLE headgate_durable_event_scope ( + scope VARCHAR(512) NOT NULL PRIMARY KEY +); + +CREATE TABLE headgate_durable_event ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + scope VARCHAR(512) NOT NULL, + topic VARCHAR(255) NOT NULL, + idempotency_key VARCHAR(255) NOT NULL, + payload LONGBLOB NOT NULL, + source LONGBLOB NOT NULL, + recorded_at_ms BIGINT NOT NULL, + UNIQUE KEY headgate_durable_event_idempotency (scope, idempotency_key), + KEY headgate_durable_event_recent (scope, id DESC), + CONSTRAINT headgate_durable_event_scope_fk FOREIGN KEY (scope) + REFERENCES headgate_durable_event_scope(scope) ON DELETE CASCADE +); diff --git a/crates/headgate-migrate/migrations/postgres/0013_durable_events.down.sql b/crates/headgate-migrate/migrations/postgres/0013_durable_events.down.sql new file mode 100644 index 0000000..106c075 --- /dev/null +++ b/crates/headgate-migrate/migrations/postgres/0013_durable_events.down.sql @@ -0,0 +1,2 @@ +DROP TABLE headgate_durable_event; +DROP TABLE headgate_durable_event_scope; diff --git a/crates/headgate-migrate/migrations/postgres/0013_durable_events.up.sql b/crates/headgate-migrate/migrations/postgres/0013_durable_events.up.sql new file mode 100644 index 0000000..308f8d2 --- /dev/null +++ b/crates/headgate-migrate/migrations/postgres/0013_durable_events.up.sql @@ -0,0 +1,17 @@ +CREATE TABLE headgate_durable_event_scope ( + scope text PRIMARY KEY +); + +CREATE TABLE headgate_durable_event ( + id bigserial PRIMARY KEY, + scope text NOT NULL REFERENCES headgate_durable_event_scope(scope) ON DELETE CASCADE, + topic text NOT NULL, + idempotency_key text NOT NULL, + payload bytea NOT NULL, + source bytea NOT NULL, + recorded_at_ms bigint NOT NULL, + UNIQUE (scope, idempotency_key) +); + +CREATE INDEX headgate_durable_event_recent + ON headgate_durable_event (scope, id DESC); diff --git a/crates/headgate-migrate/src/bin/hg_migrate.rs b/crates/headgate-migrate/src/bin/hg_migrate.rs index 66a7a22..afb5121 100644 --- a/crates/headgate-migrate/src/bin/hg_migrate.rs +++ b/crates/headgate-migrate/src/bin/hg_migrate.rs @@ -242,7 +242,7 @@ fn print_steps(result: &headgate_migrate::MigrateResult) { async fn run_postgres(cli: &Cli, url: &str) -> Result<(), MigrationError> { let (mut client, connection) = tokio_postgres::connect(url, NoTls).await?; - let driver = tokio::spawn(async move { connection.await }); + let driver = tokio::spawn(connection); match cli.command { Command::Up => { let result = match cli.schema.as_deref() { diff --git a/crates/headgate-migrate/src/lib.rs b/crates/headgate-migrate/src/lib.rs index 4fcecc3..f4cbbe9 100644 --- a/crates/headgate-migrate/src/lib.rs +++ b/crates/headgate-migrate/src/lib.rs @@ -177,6 +177,13 @@ const POSTGRES_MIGRATIONS: &[Migration] = &[ down_sql: include_str!("../migrations/postgres/0012_worker_control_state.down.sql"), online_safe: true, }, + Migration { + version: 13, + name: "durable_events", + up_sql: include_str!("../migrations/postgres/0013_durable_events.up.sql"), + down_sql: include_str!("../migrations/postgres/0013_durable_events.down.sql"), + online_safe: true, + }, ]; const MYSQL_MIGRATIONS: &[Migration] = &[ @@ -264,6 +271,13 @@ const MYSQL_MIGRATIONS: &[Migration] = &[ down_sql: include_str!("../migrations/mysql/0012_worker_control_state.down.sql"), online_safe: false, }, + Migration { + version: 13, + name: "durable_events", + up_sql: include_str!("../migrations/mysql/0013_durable_events.up.sql"), + down_sql: include_str!("../migrations/mysql/0013_durable_events.down.sql"), + online_safe: true, + }, ]; pub const fn migrations(backend: Backend) -> &'static [Migration] { @@ -512,7 +526,7 @@ mod tests { .unwrap(); assert_eq!( up.iter().map(|s| s.migration.version).collect::>(), - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] ); let current = [ @@ -528,6 +542,7 @@ mod tests { applied(10), applied(11), applied(12), + applied(13), ]; assert!( plan( @@ -548,7 +563,7 @@ mod tests { .unwrap(); assert_eq!( down.iter().map(|s| s.migration.version).collect::>(), - [12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1] + [13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1] ); } @@ -602,7 +617,7 @@ mod tests { &[], Direction::Up, MigrateOptions { - target_version: Some(13), + target_version: Some(14), ..MigrateOptions::default() } ), diff --git a/crates/headgate-migrate/src/mysql.rs b/crates/headgate-migrate/src/mysql.rs index 1a330ff..86b7392 100644 --- a/crates/headgate-migrate/src/mysql.rs +++ b/crates/headgate-migrate/src/mysql.rs @@ -528,8 +528,10 @@ fn split_statements(sql: &str) -> Result, MigrationError> { Mode::Backtick => '`', _ => unreachable!(), }; - if ch == '\\' && next.is_some() { - statement.push(next.unwrap()); + if ch == '\\' + && let Some(next) = next + { + statement.push(next); i += 2; continue; } diff --git a/crates/headgate-migrate/src/schema.rs b/crates/headgate-migrate/src/schema.rs index c15d227..6898df4 100644 --- a/crates/headgate-migrate/src/schema.rs +++ b/crates/headgate-migrate/src/schema.rs @@ -35,6 +35,8 @@ pub(crate) const TABLES: &[&str] = &[ "headgate_duty", "headgate_schedule", "headgate_schedule_event", + "headgate_durable_event_scope", + "headgate_durable_event", "headgate_worker", "headgate_effect", "headgate_operation", @@ -94,6 +96,8 @@ required_columns!(POSTGRES_COLUMNS { "headgate_schedule_event" => [ "id", "schedule_id", "tick_ms", "job_id", "outcome", "reason", "recorded_at_ms" ], + "headgate_durable_event_scope" => ["scope"], + "headgate_durable_event" => ["id", "scope", "topic", "idempotency_key", "payload", "source", "recorded_at_ms"], "headgate_worker" => [ "worker_id", "host", "pid", "queues", "concurrency", "started_at_ms", "heartbeat_at_ms", "command", "inflight", "polls", "empty_polls", "status", @@ -164,6 +168,8 @@ required_columns!(MYSQL_COLUMNS { "headgate_schedule_event" => [ "id", "schedule_id", "tick_ms", "job_id", "outcome", "reason", "recorded_at_ms" ], + "headgate_durable_event_scope" => ["scope"], + "headgate_durable_event" => ["id", "scope", "topic", "idempotency_key", "payload", "source", "recorded_at_ms"], "headgate_worker" => [ "worker_id", "host", "pid", "queues", "concurrency", "started_at_ms", "heartbeat_at_ms", "command", "inflight", "polls", "empty_polls" @@ -203,6 +209,7 @@ pub(crate) const POSTGRES_INDEXES: &[&str] = &[ "headgate_partition_counter_recent", "headgate_schedule_due", "headgate_schedule_event_recent", + "headgate_durable_event_recent", "headgate_job_archive_queue_time", ]; @@ -225,6 +232,7 @@ pub(crate) const MYSQL_INDEXES: &[&str] = &[ "headgate_partition_counter_recent", "headgate_schedule_due", "headgate_schedule_event_recent", + "headgate_durable_event_recent", "headgate_job_archive_queue_time", ]; diff --git a/crates/headgate-migrate/tests/live.rs b/crates/headgate-migrate/tests/live.rs index 221d69f..ea9067e 100644 --- a/crates/headgate-migrate/tests/live.rs +++ b/crates/headgate-migrate/tests/live.rs @@ -20,7 +20,7 @@ async fn live_postgres_migration_lifecycle_and_drift_rejection() { let (admin, admin_driver) = tokio_postgres::connect(&conninfo, tokio_postgres::NoTls) .await .expect("admin connect"); - let admin_task = tokio::spawn(async move { admin_driver.await }); + let admin_task = tokio::spawn(admin_driver); let exists: i64 = admin .query_one( "SELECT count(*) FROM information_schema.schemata WHERE schema_name = $1", @@ -38,7 +38,7 @@ async fn live_postgres_migration_lifecycle_and_drift_rejection() { let (mut client, driver) = tokio_postgres::connect(&conninfo, tokio_postgres::NoTls) .await .expect("test connect"); - let driver_task = tokio::spawn(async move { driver.await }); + let driver_task = tokio::spawn(driver); client .batch_execute(&format!("SET search_path TO {schema}")) .await @@ -46,14 +46,14 @@ async fn live_postgres_migration_lifecycle_and_drift_rejection() { let result: Result<(), Box> = async { let up = migrate_postgres(&mut client, Direction::Up, MigrateOptions::default()).await?; - if up.steps.len() != 12 + if up.steps.len() != 13 || up.steps[0].migration.version != 1 - || up.steps[11].migration.version != 12 + || up.steps[12].migration.version != 13 { return Err(test_error(format!("fresh up steps = {:?}", up.steps))); } let validation = validate_postgres(&client).await?; - if !validation.is_ok() || validation.current_version != 12 { + if !validation.is_ok() || validation.current_version != 13 { return Err(test_error(format!("fresh validation = {validation:?}"))); } let dry = migrate_postgres( @@ -65,12 +65,12 @@ async fn live_postgres_migration_lifecycle_and_drift_rejection() { }, ) .await?; - if !dry.dry_run || dry.steps.len() != 12 { + if !dry.dry_run || dry.steps.len() != 13 { return Err(test_error(format!("down dry run = {:?}", dry.steps))); } let down = migrate_postgres(&mut client, Direction::Down, MigrateOptions::default()).await?; - if down.steps.len() != 12 { + if down.steps.len() != 13 { return Err(test_error(format!("down steps = {:?}", down.steps))); } let row = client @@ -119,7 +119,7 @@ async fn live_postgres_migration_lifecycle_and_drift_rejection() { )); } let adopted = adopt_postgres(&mut client).await?; - if adopted.last().map(|row| row.version) != Some(12) { + if adopted.last().map(|row| row.version) != Some(13) { return Err(test_error(format!("adopted history = {adopted:?}"))); } if !validate_postgres(&client).await?.is_ok() { @@ -203,14 +203,14 @@ async fn live_mysql_migration_lifecycle_and_drift_rejection() { let mut conn = pool.get_conn().await.expect("test connect"); let result: Result<(), Box> = async { let up = migrate_mysql(&mut conn, Direction::Up, MigrateOptions::default()).await?; - if up.steps.len() != 12 + if up.steps.len() != 13 || up.steps[0].migration.version != 1 - || up.steps[11].migration.version != 12 + || up.steps[12].migration.version != 13 { return Err(test_error(format!("fresh up steps = {:?}", up.steps))); } let validation = validate_mysql(&mut conn).await?; - if !validation.is_ok() || validation.current_version != 12 { + if !validation.is_ok() || validation.current_version != 13 { return Err(test_error(format!("fresh validation = {validation:?}"))); } let dry = migrate_mysql( @@ -222,11 +222,11 @@ async fn live_mysql_migration_lifecycle_and_drift_rejection() { }, ) .await?; - if !dry.dry_run || dry.steps.len() != 12 { + if !dry.dry_run || dry.steps.len() != 13 { return Err(test_error(format!("down dry run = {:?}", dry.steps))); } let down = migrate_mysql(&mut conn, Direction::Down, MigrateOptions::default()).await?; - if down.steps.len() != 12 { + if down.steps.len() != 13 { return Err(test_error(format!("down steps = {:?}", down.steps))); } let job_exists: Option = conn @@ -270,7 +270,7 @@ async fn live_mysql_migration_lifecycle_and_drift_rejection() { return Err(test_error("unversioned MySQL schema was migrated as fresh")); } let adopted = adopt_mysql(&mut conn).await?; - if adopted.last().map(|row| row.version) != Some(12) { + if adopted.last().map(|row| row.version) != Some(13) { return Err(test_error(format!("adopted history = {adopted:?}"))); } if !validate_mysql(&mut conn).await?.is_ok() { @@ -393,7 +393,7 @@ async fn live_mysql_configured_lock_namespace_avoids_an_application_lock() { let migrated = tokio::time::timeout(Duration::from_secs(30), &mut migration) .await .map_err(|_| test_error("migration still blocked after configured lock release"))??; - if migrated.steps.len() != 12 { + if migrated.steps.len() != 13 { return Err(test_error(format!( "configured migration steps = {:?}", migrated.steps diff --git a/crates/headgate-mysql/migrations/0013_durable_events.sql b/crates/headgate-mysql/migrations/0013_durable_events.sql new file mode 100644 index 0000000..4eceb1d --- /dev/null +++ b/crates/headgate-mysql/migrations/0013_durable_events.sql @@ -0,0 +1,17 @@ +CREATE TABLE headgate_durable_event_scope ( + scope VARCHAR(512) NOT NULL PRIMARY KEY +); + +CREATE TABLE headgate_durable_event ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + scope VARCHAR(512) NOT NULL, + topic VARCHAR(255) NOT NULL, + idempotency_key VARCHAR(255) NOT NULL, + payload LONGBLOB NOT NULL, + source LONGBLOB NOT NULL, + recorded_at_ms BIGINT NOT NULL, + UNIQUE KEY headgate_durable_event_idempotency (scope, idempotency_key), + KEY headgate_durable_event_recent (scope, id DESC), + CONSTRAINT headgate_durable_event_scope_fk FOREIGN KEY (scope) + REFERENCES headgate_durable_event_scope(scope) ON DELETE CASCADE +); diff --git a/crates/headgate-mysql/src/inspect.rs b/crates/headgate-mysql/src/inspect.rs index cfcd792..fbe8c65 100644 --- a/crates/headgate-mysql/src/inspect.rs +++ b/crates/headgate-mysql/src/inspect.rs @@ -8,11 +8,12 @@ use headgate_core::{ AdmissionExplain, BulkRequest, Checkpoint, CheckpointInspect, ConcurrencyLimitConfig, - HistoryBucket, Inspect, JobFilter, JobOutput, JobPage, JobProgress, JobResult, JobSummary, - MissedPolicy, OperationStatus, OutputInspect, PartitionState, ProgressInspect, QuarantineEntry, - QueueStats, QuietGroupMetrics, RateClassConfig, RateClassState, ResultInspect, - SCHEDULE_EVENT_LIMIT, SaturationStrategy, Schedule, ScheduleEvent, ScheduleEventOutcome, - StateCounts, StoreError, WorkerMeta, noisy_partition_keys, + DURABLE_EVENT_LIMIT, DurableEvent, HistoryBucket, Inspect, JobFilter, JobOutput, JobPage, + JobProgress, JobResult, JobSummary, MissedPolicy, OperationStatus, OutputInspect, + PartitionState, ProgressInspect, QuarantineEntry, QueueStats, QuietGroupMetrics, + RateClassConfig, RateClassState, ResultInspect, SCHEDULE_EVENT_LIMIT, SaturationStrategy, + Schedule, ScheduleEvent, ScheduleEventOutcome, StateCounts, StoreError, WorkerMeta, + noisy_partition_keys, }; use mysql_async::prelude::*; use mysql_async::{Params, Row, TxOpts, Value}; @@ -26,8 +27,6 @@ use headgate_shared::inspection::{ }; const QUIET_PARTITION_LIMIT: usize = SHARED_QUIET_PARTITION_LIMIT as usize; -/// Queue-position lookups cap here; "position >= 1000" is answer enough. - const JOB_COLS: &str = "j.ulid, j.kind, j.queue, CAST(j.state AS CHAR) AS state_text, \ j.schema_version, j.priority, j.attempt, j.crash_attempt, j.max_attempts, \ j.partition_key, j.rate_class, j.sticky_worker, j.weight, j.fingerprint, j.enqueued_at_ms, j.scheduled_at_ms, j.claimed_at_ms, \ @@ -732,7 +731,7 @@ impl Inspect for MysqlStore { tx.exec_drop( "INSERT INTO headgate_active_partition (queue, partition_key) SELECT queue, partition_key FROM headgate_job - WHERE ulid = ? AND state IN ('archived', 'cancelled') + WHERE ulid = ? AND state IN ('archived', 'cancelled', 'undecodable') ON DUPLICATE KEY UPDATE queue = VALUES(queue)", (id,), ) @@ -742,7 +741,7 @@ impl Inspect for MysqlStore { format!( "UPDATE headgate_job SET state = 'available', scheduled_at_ms = {NOW_MS}, finalized_at_ms = NULL - WHERE ulid = ? AND state IN ('archived', 'cancelled')" + WHERE ulid = ? AND state IN ('archived', 'cancelled', 'undecodable')" ), (id,), ) @@ -756,7 +755,7 @@ impl Inspect for MysqlStore { match self.job_state(id).await? { None => Err(StoreError::NotFound(format!("job {id}"))), Some(state) => Err(StoreError::Invalid(format!( - "operator_retry is only defined from archived or cancelled; job {id} is {state}" + "operator_retry is only defined from archived, cancelled, or undecodable; job {id} is {state}" ))), } } @@ -785,7 +784,7 @@ impl Inspect for MysqlStore { "UPDATE headgate_job SET state = 'cancelled', lease_id = NULL, lease_expires_at_ms = NULL, claimed_by = NULL, finalized_at_ms = {NOW_MS} - WHERE ulid = ? AND state IN ('pending', 'scheduled', 'available', 'running')" + WHERE ulid = ? AND state IN ('pending', 'scheduled', 'available', 'running', 'retryable')" ), (id,), ) @@ -1187,6 +1186,99 @@ impl Inspect for MysqlStore { .collect() } + async fn append_durable_event( + &self, + event: &DurableEvent, + ) -> Result<(DurableEvent, bool), StoreError> { + validate_durable_event(event)?; + let mut c = self.raw_conn().await?; + let mut tx = c + .start_transaction(TxOpts::default()) + .await + .map_err(map_err)?; + tx.exec_drop( + "INSERT IGNORE INTO headgate_durable_event_scope(scope) VALUES(?)", + (&event.scope,), + ) + .await + .map_err(map_err)?; + let _: String = tx + .exec_first( + "SELECT scope FROM headgate_durable_event_scope WHERE scope=? FOR UPDATE", + (&event.scope,), + ) + .await + .map_err(map_err)? + .ok_or_else(|| StoreError::Invalid("durable event scope was not created".into()))?; + let existing: Option = tx.exec_first( + "SELECT id,topic,payload,source,recorded_at_ms FROM headgate_durable_event WHERE scope=? AND idempotency_key=?", + (&event.scope, &event.idempotency_key), + ).await.map_err(map_err)?; + let inserted = existing.is_none(); + if inserted { + tx.exec_drop( + format!("INSERT INTO headgate_durable_event(scope,topic,idempotency_key,payload,source,recorded_at_ms) VALUES(?,?,?,?,?,{NOW_MS})"), + (&event.scope, &event.topic, &event.idempotency_key, &event.payload, &event.source), + ).await.map_err(map_err)?; + } + let row: Row = tx.exec_first( + "SELECT id,topic,payload,source,recorded_at_ms FROM headgate_durable_event WHERE scope=? AND idempotency_key=?", + (&event.scope, &event.idempotency_key), + ).await.map_err(map_err)?.ok_or_else(|| StoreError::Invalid("durable event was not stored".into()))?; + let topic: String = row.get("topic").unwrap_or_default(); + let payload: Vec = row.get("payload").unwrap_or_default(); + let source: Vec = row.get("source").unwrap_or_default(); + if topic != event.topic || payload != event.payload || source != event.source { + return Err(StoreError::Invalid( + "durable event idempotency key was reused with different content".into(), + )); + } + let id: u64 = row.get("id").unwrap_or_default(); + let recorded_at_ms: i64 = row.get("recorded_at_ms").unwrap_or_default(); + tx.exec_drop( + "DELETE e FROM headgate_durable_event e LEFT JOIN + (SELECT id FROM headgate_durable_event WHERE scope=? ORDER BY id DESC LIMIT ?) keep ON keep.id=e.id + WHERE e.scope=? AND keep.id IS NULL", + (&event.scope, DURABLE_EVENT_LIMIT, &event.scope), + ).await.map_err(map_err)?; + tx.commit().await.map_err(map_err)?; + Ok(( + DurableEvent { + event_id: id, + recorded_at_ms, + ..event.clone() + }, + inserted, + )) + } + + async fn list_durable_events( + &self, + scope: &str, + before_event_id: Option, + limit: u32, + ) -> Result, StoreError> { + headgate_core::validate_durable_event_limit(limit)?; + let mut c = self.raw_conn().await?; + let rows: Vec = c.exec( + "SELECT id,scope,topic,idempotency_key,payload,source,recorded_at_ms FROM headgate_durable_event + WHERE scope=? AND (?=0 OR id Result, StoreError> { let mut c = self.raw_conn().await?; let queues = serde_json::to_string(&w.queues).unwrap_or_else(|_| "[]".into()); @@ -1285,12 +1377,12 @@ impl Inspect for MysqlStore { worker_id: &str, command: Option<&str>, ) -> Result<(), StoreError> { - if let Some(cmd) = command { - if !headgate_core::valid_worker_command(cmd) { - return Err(StoreError::Invalid( - "command must be quiet, resume, restart, terminate, or resign".into(), - )); - } + if let Some(cmd) = command + && !headgate_core::valid_worker_command(cmd) + { + return Err(StoreError::Invalid( + "command must be quiet, resume, restart, terminate, or resign".into(), + )); } let mut c = self.raw_conn().await?; // CLIENT_FOUND_ROWS (crate contract): matched-rows semantics, so clearing an @@ -1473,6 +1565,29 @@ impl Inspect for MysqlStore { Ok(()) } + async fn schedule_pending_job(&self, id: &str, at_ms: i64) -> Result<(), StoreError> { + if at_ms <= 0 { + return Err(StoreError::Invalid( + "pending schedule timestamp must be positive".into(), + )); + } + let mut c = self.raw_conn().await?; + c.exec_drop( + "UPDATE headgate_job SET state='scheduled', scheduled_at_ms=? + WHERE ulid=? AND state='pending'", + (at_ms, id), + ) + .await + .map_err(map_err)?; + if c.affected_rows() == 1 { + Ok(()) + } else { + Err(StoreError::Invalid( + "workflow_schedule is defined only from pending".into(), + )) + } + } + async fn delete_queue(&self, queue: &str, force: bool) -> Result, StoreError> { let mut c = self.raw_conn().await?; let mut tx = c @@ -1566,6 +1681,35 @@ impl Inspect for MysqlStore { } } +fn validate_durable_event(event: &DurableEvent) -> Result<(), StoreError> { + if event.scope.is_empty() + || event.scope.len() > 512 + || event.topic.is_empty() + || event.topic.len() > 255 + || event.idempotency_key.is_empty() + || event.idempotency_key.len() > 255 + { + return Err(StoreError::Invalid( + "durable event scope, topic, or idempotency key is invalid".into(), + )); + } + if event.payload.len() > headgate_core::MAX_DURABLE_EVENT_PAYLOAD_BYTES + || serde_json::from_slice::(&event.payload).is_err() + { + return Err(StoreError::Invalid( + "durable event payload must be valid JSON of at most 65536 bytes".into(), + )); + } + if event.source.len() > headgate_core::MAX_DURABLE_EVENT_SOURCE_BYTES + || serde_json::from_slice::(&event.source).is_err() + { + return Err(StoreError::Invalid( + "durable event source must be valid JSON of at most 16384 bytes".into(), + )); + } + Ok(()) +} + #[async_trait::async_trait] impl ResultInspect for MysqlStore { async fn get_job_result(&self, id: &str) -> Result, StoreError> { diff --git a/crates/headgate-mysql/src/lib.rs b/crates/headgate-mysql/src/lib.rs index 20e9398..ddddc46 100644 --- a/crates/headgate-mysql/src/lib.rs +++ b/crates/headgate-mysql/src/lib.rs @@ -356,32 +356,6 @@ fn archive_partition(value: &str) -> Result<(String, String), StoreError> { Ok((format!("p_{value}"), format!("{year:04}-{month:02}-01"))) } -#[cfg(test)] -mod sql_shape_tests { - use super::{enqueue_backpressure_depth_sql, lazy_unique_release_sql, unique_holder_sql}; - - #[test] - fn unique_conflict_queries_stay_on_generated_indexes() { - let release = lazy_unique_release_sql(2); - assert!(release.contains("WHERE unique_throttle IN (?, ?)")); - assert!(!release.contains("WHERE unique_key IN")); - - let holder = unique_holder_sql(2); - assert!(holder.contains("unique_active IN (?, ?)")); - assert!(holder.contains("unique_throttle IN (?, ?)")); - assert!(!holder.contains("WHERE unique_key IN")); - } - - #[test] - fn enqueue_backpressure_hot_path_uses_constant_size_counters() { - let sql = enqueue_backpressure_depth_sql(2).to_ascii_lowercase(); - assert!(sql.contains("headgate_enqueue_policy")); - assert_eq!(sql.matches("headgate_enqueue_counter").count(), 2); - assert!(!sql.contains("headgate_job")); - assert!(!sql.contains("count(")); - } -} - fn map_err(e: mysql_async::Error) -> StoreError { match &e { mysql_async::Error::Io(_) | mysql_async::Error::Driver(_) => { @@ -1898,7 +1872,7 @@ async fn reclaim_tx( .map_err(map_err)? .unwrap_or(0); // lease fencing an expired lease is LeaseLost, NEVER Retry: crash_attempt++, attempt stays. - let rows: Vec<( + type ReclaimRow = ( i64, String, String, @@ -1907,7 +1881,8 @@ async fn reclaim_tx( Option, Option, i64, - )> = tx + ); + let rows: Vec = tx .exec( "SELECT id, ulid, fingerprint, crash_attempt, kind, checkpoint, unique_key, unique_window_ms @@ -2708,3 +2683,29 @@ impl MysqlStore { self.conn().await } } + +#[cfg(test)] +mod sql_shape_tests { + use super::{enqueue_backpressure_depth_sql, lazy_unique_release_sql, unique_holder_sql}; + + #[test] + fn unique_conflict_queries_stay_on_generated_indexes() { + let release = lazy_unique_release_sql(2); + assert!(release.contains("WHERE unique_throttle IN (?, ?)")); + assert!(!release.contains("WHERE unique_key IN")); + + let holder = unique_holder_sql(2); + assert!(holder.contains("unique_active IN (?, ?)")); + assert!(holder.contains("unique_throttle IN (?, ?)")); + assert!(!holder.contains("WHERE unique_key IN")); + } + + #[test] + fn enqueue_backpressure_hot_path_uses_constant_size_counters() { + let sql = enqueue_backpressure_depth_sql(2).to_ascii_lowercase(); + assert!(sql.contains("headgate_enqueue_policy")); + assert_eq!(sql.matches("headgate_enqueue_counter").count(), 2); + assert!(!sql.contains("headgate_job")); + assert!(!sql.contains("count(")); + } +} diff --git a/crates/headgate-mysql/tests/bounded_pool.rs b/crates/headgate-mysql/tests/bounded_pool.rs index 9f1f66a..1e586d8 100644 --- a/crates/headgate-mysql/tests/bounded_pool.rs +++ b/crates/headgate-mysql/tests/bounded_pool.rs @@ -268,7 +268,7 @@ async fn connection_budget_keeps_renewal_acks_and_duties_live_behind_held_transa sampler.await.expect("pool sampler"); let peak = peak_connections.load(Ordering::Relaxed); assert!( - peak >= HELD_TRANSACTIONS && peak <= POOL_BUDGET, + (HELD_TRANSACTIONS..=POOL_BUDGET).contains(&peak), "peak physical MySQL connections={peak}, want {HELD_TRANSACTIONS}..{POOL_BUDGET}" ); // A zero is allowed: the pool can schedule the short transient calls without a diff --git a/crates/headgate-mysql/tests/unique.rs b/crates/headgate-mysql/tests/unique.rs index a4bcdfb..bb6eefb 100644 --- a/crates/headgate-mysql/tests/unique.rs +++ b/crates/headgate-mysql/tests/unique.rs @@ -119,7 +119,8 @@ async fn field(id: &str, col: &str) -> String { async fn generated(id: &str) -> (bool, bool) { use mysql_async::prelude::*; let mut c = raw().await; - let row: Option<(Option>, Option>)> = c + type GeneratedUniqueKeys = (Option>, Option>); + let row: Option = c .exec_first( "SELECT unique_active, unique_throttle FROM headgate_job WHERE ulid = ?", (id,), diff --git a/crates/headgate-postgres/migrations/0013_durable_events.sql b/crates/headgate-postgres/migrations/0013_durable_events.sql new file mode 100644 index 0000000..308f8d2 --- /dev/null +++ b/crates/headgate-postgres/migrations/0013_durable_events.sql @@ -0,0 +1,17 @@ +CREATE TABLE headgate_durable_event_scope ( + scope text PRIMARY KEY +); + +CREATE TABLE headgate_durable_event ( + id bigserial PRIMARY KEY, + scope text NOT NULL REFERENCES headgate_durable_event_scope(scope) ON DELETE CASCADE, + topic text NOT NULL, + idempotency_key text NOT NULL, + payload bytea NOT NULL, + source bytea NOT NULL, + recorded_at_ms bigint NOT NULL, + UNIQUE (scope, idempotency_key) +); + +CREATE INDEX headgate_durable_event_recent + ON headgate_durable_event (scope, id DESC); diff --git a/crates/headgate-postgres/src/inspect.rs b/crates/headgate-postgres/src/inspect.rs index d325e7d..b5dec16 100644 --- a/crates/headgate-postgres/src/inspect.rs +++ b/crates/headgate-postgres/src/inspect.rs @@ -6,11 +6,12 @@ use headgate_core::{ AdmissionExplain, BulkRequest, Checkpoint, CheckpointInspect, ConcurrencyLimitConfig, - HistoryBucket, Inspect, JobFilter, JobOutput, JobPage, JobProgress, JobResult, JobSummary, - MissedPolicy, OperationStatus, OutputInspect, PartitionState, ProgressInspect, QuarantineEntry, - QueueStats, QuietGroupMetrics, RateClassConfig, RateClassState, ResultInspect, - SCHEDULE_EVENT_LIMIT, SaturationStrategy, Schedule, ScheduleEvent, ScheduleEventOutcome, - StateCounts, StoreError, WorkerMeta, noisy_partition_keys, + DURABLE_EVENT_LIMIT, DurableEvent, HistoryBucket, Inspect, JobFilter, JobOutput, JobPage, + JobProgress, JobResult, JobSummary, MissedPolicy, OperationStatus, OutputInspect, + PartitionState, ProgressInspect, QuarantineEntry, QueueStats, QuietGroupMetrics, + RateClassConfig, RateClassState, ResultInspect, SCHEDULE_EVENT_LIMIT, SaturationStrategy, + Schedule, ScheduleEvent, ScheduleEventOutcome, StateCounts, StoreError, WorkerMeta, + noisy_partition_keys, }; use tokio_postgres::types::ToSql; @@ -20,8 +21,6 @@ use crate::{NOW_MS, PgStore, decode_headers, map_pg_err}; use headgate_shared::inspection::{ MAX_PAGE, MEMORY_SAMPLE_LIMIT, POSITION_LIMIT, QUIET_PARTITION_LIMIT, SAMPLE_LIMIT, }; -/// Queue-position lookups cap here; "position >= 1000" is answer enough. - fn job_from_row(row: &tokio_postgres::Row, include_payload: bool) -> JobSummary { JobSummary { id: row.get("ulid"), @@ -271,8 +270,7 @@ impl Inspect for PgStore { // partial index. A noisy tenant's depth therefore cannot become admin work. let part_rows = c .query( - &format!( - r#" + r#" WITH names AS ( SELECT partition_key FROM headgate_active_partition WHERE queue = $1 UNION SELECT partition_key FROM headgate_inflight @@ -298,8 +296,7 @@ impl Inspect for PgStore { ON i.queue = $1 AND i.partition_key = n.partition_key LEFT JOIN rates r ON r.partition_key = n.partition_key ORDER BY n.partition_key - "# - ), + "#, &[ &queue, &(now_ms / 60000 * 60000 - 60000), @@ -671,7 +668,7 @@ impl Inspect for PgStore { WITH upd AS ( UPDATE headgate_job SET state = 'available', scheduled_at_ms = {NOW_MS}, finalized_at_ms = NULL - WHERE ulid = $1 AND state IN ('archived', 'cancelled') + WHERE ulid = $1 AND state IN ('archived', 'cancelled', 'undecodable') RETURNING queue, partition_key ), -- tenant fairness/adaptive admission retry-now makes the row available; list its partition here. @@ -690,7 +687,7 @@ impl Inspect for PgStore { match self.job_state(&c, id).await? { None => Err(StoreError::NotFound(format!("job {id}"))), Some(state) => Err(StoreError::Invalid(format!( - "operator_retry is only defined from archived or cancelled; job {id} is {state}" + "operator_retry is only defined from archived, cancelled, or undecodable; job {id} is {state}" ))), } } @@ -706,7 +703,7 @@ impl Inspect for PgStore { "WITH pick AS ( SELECT j.id, j.queue, j.partition_key, (j.state = 'running') AS was_running FROM headgate_job j - WHERE j.ulid = $1 AND j.state IN ('pending', 'scheduled', 'available', 'running') + WHERE j.ulid = $1 AND j.state IN ('pending', 'scheduled', 'available', 'running', 'retryable') FOR UPDATE ), upd AS ( @@ -1119,6 +1116,107 @@ impl Inspect for PgStore { .collect() } + async fn append_durable_event( + &self, + event: &DurableEvent, + ) -> Result<(DurableEvent, bool), StoreError> { + validate_durable_event(event)?; + let mut c = self.client().await?; + let tx = c.transaction().await.map_err(map_pg_err)?; + tx.execute( + "INSERT INTO headgate_durable_event_scope(scope) VALUES($1) ON CONFLICT DO NOTHING", + &[&event.scope], + ) + .await + .map_err(map_pg_err)?; + tx.query_one( + "SELECT scope FROM headgate_durable_event_scope WHERE scope=$1 FOR UPDATE", + &[&event.scope], + ) + .await + .map_err(map_pg_err)?; + let sql = format!( + "INSERT INTO headgate_durable_event(scope,topic,idempotency_key,payload,source,recorded_at_ms) + VALUES($1,$2,$3,$4,$5,{NOW_MS}) ON CONFLICT(scope,idempotency_key) DO NOTHING + RETURNING id,recorded_at_ms" + ); + let inserted = tx + .query( + &sql, + &[ + &event.scope as &(dyn ToSql + Sync), + &event.topic, + &event.idempotency_key, + &event.payload, + &event.source, + ], + ) + .await + .map_err(map_pg_err)? + .into_iter() + .next(); + let (id, recorded_at_ms, was_inserted) = if let Some(row) = inserted { + (row.get::<_, i64>(0) as u64, row.get(1), true) + } else { + let row = tx.query_one( + "SELECT id,topic,payload,source,recorded_at_ms FROM headgate_durable_event WHERE scope=$1 AND idempotency_key=$2", + &[&event.scope, &event.idempotency_key], + ).await.map_err(map_pg_err)?; + let topic: String = row.get(1); + let payload: Vec = row.get(2); + let source: Vec = row.get(3); + if topic != event.topic || payload != event.payload || source != event.source { + return Err(StoreError::Invalid( + "durable event idempotency key was reused with different content".into(), + )); + } + (row.get::<_, i64>(0) as u64, row.get(4), false) + }; + tx.execute( + "DELETE FROM headgate_durable_event WHERE scope=$1 AND id NOT IN + (SELECT id FROM headgate_durable_event WHERE scope=$1 ORDER BY id DESC LIMIT $2)", + &[&event.scope, &(DURABLE_EVENT_LIMIT as i64)], + ) + .await + .map_err(map_pg_err)?; + tx.commit().await.map_err(map_pg_err)?; + Ok(( + DurableEvent { + event_id: id, + recorded_at_ms, + ..event.clone() + }, + was_inserted, + )) + } + + async fn list_durable_events( + &self, + scope: &str, + before_event_id: Option, + limit: u32, + ) -> Result, StoreError> { + headgate_core::validate_durable_event_limit(limit)?; + let c = self.client().await?; + let rows = c.query( + "SELECT id,scope,topic,idempotency_key,payload,source,recorded_at_ms FROM headgate_durable_event + WHERE scope=$1 AND ($2::bigint IS NULL OR id<$2) ORDER BY id DESC LIMIT $3", + &[&scope, &before_event_id.map(|id| id as i64), &(limit as i64)], + ).await.map_err(map_pg_err)?; + Ok(rows + .into_iter() + .map(|row| DurableEvent { + event_id: row.get::<_, i64>("id") as u64, + scope: row.get("scope"), + topic: row.get("topic"), + idempotency_key: row.get("idempotency_key"), + payload: row.get("payload"), + source: row.get("source"), + recorded_at_ms: row.get("recorded_at_ms"), + }) + .collect()) + } + async fn heartbeat_worker(&self, w: &WorkerMeta) -> Result, StoreError> { let c = self.client().await?; let status = if w.status.is_empty() { @@ -1171,12 +1269,12 @@ impl Inspect for PgStore { worker_id: &str, command: Option<&str>, ) -> Result<(), StoreError> { - if let Some(cmd) = command { - if !headgate_core::valid_worker_command(cmd) { - return Err(StoreError::Invalid( - "command must be quiet, resume, restart, terminate, or resign".into(), - )); - } + if let Some(cmd) = command + && !headgate_core::valid_worker_command(cmd) + { + return Err(StoreError::Invalid( + "command must be quiet, resume, restart, terminate, or resign".into(), + )); } let c = self.client().await?; let n = c @@ -1394,6 +1492,30 @@ impl Inspect for PgStore { Ok(()) } + async fn schedule_pending_job(&self, id: &str, at_ms: i64) -> Result<(), StoreError> { + if at_ms <= 0 { + return Err(StoreError::Invalid( + "pending schedule timestamp must be positive".into(), + )); + } + let c = self.client().await?; + let n = c + .execute( + "UPDATE headgate_job SET state = 'scheduled', scheduled_at_ms = $2 + WHERE ulid = $1 AND state = 'pending'", + &[&id, &at_ms], + ) + .await + .map_err(map_pg_err)?; + if n == 1 { + Ok(()) + } else { + Err(StoreError::Invalid( + "workflow_schedule is defined only from pending".into(), + )) + } + } + async fn delete_queue(&self, queue: &str, force: bool) -> Result, StoreError> { let mut c = self.client().await?; let tx = c.transaction().await.map_err(map_pg_err)?; @@ -1486,6 +1608,35 @@ impl Inspect for PgStore { } } +fn validate_durable_event(event: &DurableEvent) -> Result<(), StoreError> { + if event.scope.is_empty() + || event.scope.len() > 512 + || event.topic.is_empty() + || event.topic.len() > 255 + || event.idempotency_key.is_empty() + || event.idempotency_key.len() > 255 + { + return Err(StoreError::Invalid( + "durable event scope, topic, or idempotency key is invalid".into(), + )); + } + if event.payload.len() > headgate_core::MAX_DURABLE_EVENT_PAYLOAD_BYTES + || serde_json::from_slice::(&event.payload).is_err() + { + return Err(StoreError::Invalid( + "durable event payload must be valid JSON of at most 65536 bytes".into(), + )); + } + if event.source.len() > headgate_core::MAX_DURABLE_EVENT_SOURCE_BYTES + || serde_json::from_slice::(&event.source).is_err() + { + return Err(StoreError::Invalid( + "durable event source must be valid JSON of at most 16384 bytes".into(), + )); + } + Ok(()) +} + #[async_trait::async_trait] impl ResultInspect for PgStore { async fn get_job_result(&self, id: &str) -> Result, StoreError> { diff --git a/crates/headgate-postgres/src/lib.rs b/crates/headgate-postgres/src/lib.rs index f3994fb..2a49677 100644 --- a/crates/headgate-postgres/src/lib.rs +++ b/crates/headgate-postgres/src/lib.rs @@ -895,6 +895,7 @@ impl PgStore { /// block a producer, never deadlock with a concurrent pruner); /// 2. in a SECOND statement, which under READ COMMITTED takes a FRESH snapshot, /// delete only those with no available job left. + /// /// One statement cannot do this. All CTEs in a statement share one snapshot, so a /// producer that committed after that snapshot is invisible, and the delete would /// strand its job — the one direction of staleness that is a correctness bug. With @@ -1416,6 +1417,7 @@ impl PgStore { /// lifecycle state machine apply the transition table on an explicit client/transaction. Every /// statement re-checks `(ulid, lease_id, fence, state='running')`, so a superseded /// holder gets `LeaseRejected` — an error the worker must handle, never a no-op. + #[allow(clippy::too_many_arguments)] // Keeps every ack policy inside the caller's transaction. async fn ack_on( &self, c: &C, @@ -2191,7 +2193,7 @@ impl headgate_core::Notifying for PgStore { match tokio::time::timeout_at(deadline, rx.recv()).await { Err(_) => return Ok(None), // timeout: the poll fallback takes it Ok(Ok(queue)) => { - if queues.is_empty() || queues.iter().any(|q| *q == queue) { + if queues.is_empty() || queues.contains(&queue) { return Ok(Some(queue)); } } @@ -2487,12 +2489,12 @@ impl PgTx { impl Drop for PgTx { fn drop(&mut self) { - if !self.done { - if let Some(client) = self.conn.take() { - // Take the connection out of the pool for good; closing it makes the - // server abort the open transaction. - let _ = Object::take(client.inner); - } + if !self.done + && let Some(client) = self.conn.take() + { + // Take the connection out of the pool for good; closing it makes the + // server abort the open transaction. + let _ = Object::take(client.inner); } } } diff --git a/crates/headgate-postgres/tests/bounded_pool.rs b/crates/headgate-postgres/tests/bounded_pool.rs index 5be7372..2b18019 100644 --- a/crates/headgate-postgres/tests/bounded_pool.rs +++ b/crates/headgate-postgres/tests/bounded_pool.rs @@ -184,7 +184,7 @@ async fn connection_budget_keeps_renewal_acks_and_duties_live_behind_held_transa let (admin, admin_driver) = tokio_postgres::connect(&conninfo, NoTls) .await .expect("admin connect"); - let admin_task = tokio::spawn(async move { admin_driver.await }); + let admin_task = tokio::spawn(admin_driver); let queue = format!("cb-rust-pg-{}", std::process::id()); let worker_id = format!("cb-rust-pg-w-{}", std::process::id()); admin diff --git a/crates/headgate-redis/lua/admin.lua b/crates/headgate-redis/lua/admin.lua index fed1cca..2311bc7 100644 --- a/crates/headgate-redis/lua/admin.lua +++ b/crates/headgate-redis/lua/admin.lua @@ -95,7 +95,7 @@ local function do_cancel(id, h) if ret > 0 then redis.call('ZADD', p .. ':ret', now + ret, id) end end --- archived|cancelled -> available (operator_retry). +-- archived|cancelled|undecodable -> available (operator_retry). local function do_retry(id, h) local st, q, part, fp = h[1], h[2], h[3], h[4] local uk, uw = h[5], tonumber(h[6] or '0') or 0 @@ -141,7 +141,7 @@ if op == 'cancel' then local id = ARGV[2] local h = job_head(id) if not h[1] then return {'NF'} end - if h[1] ~= 'pending' and h[1] ~= 'scheduled' and h[1] ~= 'available' and h[1] ~= 'running' then + if h[1] ~= 'pending' and h[1] ~= 'scheduled' and h[1] ~= 'available' and h[1] ~= 'running' and h[1] ~= 'retryable' then return {'ERR', h[1]} end do_cancel(id, h) @@ -159,6 +159,17 @@ elseif op == 'promote' then redis.call('ZADD', p .. ':avail:' .. h[2] .. ':' .. h[3], now, id) redis.call('ZADD', p .. ':metricparts:' .. h[2], now, h[3]) return {'OK'} +elseif op == 'schedule_pending' then + local id, at = ARGV[2], tonumber(ARGV[3]) + local h = job_head(id) + if not h[1] then return {'NF'} end + if h[1] ~= 'pending' or not at or at <= 0 then return {'ERR', h[1]} end + redis.call('ZREM', idx(h[2], 'pending'), id) + redis.call('ZADD', idx(h[2], 'scheduled'), at, id) + redis.call('ZADD', p .. ':sched', at, id) + add_waiting(id, h[2], h[3], h[8], at) + redis.call('HSET', jk(id), 'state', 'scheduled', 'scheduled_at_ms', at) + return {'OK'} elseif op == 'queue_delete' then local q, force, opid = ARGV[2], ARGV[3] == '1', ARGV[4] @@ -187,7 +198,7 @@ elseif op == 'retry' then local id = ARGV[2] local h = job_head(id) if not h[1] then return {'NF'} end - if h[1] ~= 'archived' and h[1] ~= 'cancelled' then return {'ERR', h[1]} end + if h[1] ~= 'archived' and h[1] ~= 'cancelled' and h[1] ~= 'undecodable' then return {'ERR', h[1]} end local holder = do_retry(id, h) if holder then return {'DUP', holder} end return {'OK'} diff --git a/crates/headgate-redis/src/inspect.rs b/crates/headgate-redis/src/inspect.rs index e2f2a65..2a9fecb 100644 --- a/crates/headgate-redis/src/inspect.rs +++ b/crates/headgate-redis/src/inspect.rs @@ -9,11 +9,12 @@ use headgate_core::{ AdmissionExplain, BulkRequest, Checkpoint, CheckpointInspect, ConcurrencyLimitConfig, - HistoryBucket, Inspect, JobFilter, JobOutput, JobPage, JobProgress, JobResult, JobSummary, - MissedPolicy, OperationStatus, OutputInspect, PartitionState, ProgressInspect, QuarantineEntry, - QueueStats, QuietGroupMetrics, RateClassConfig, RateClassState, ResultInspect, - SCHEDULE_EVENT_LIMIT, SaturationStrategy, Schedule, ScheduleEvent, ScheduleEventOutcome, - StateCounts, StoreError, WorkerMeta, noisy_partition_keys, + DURABLE_EVENT_LIMIT, DurableEvent, HistoryBucket, Inspect, JobFilter, JobOutput, JobPage, + JobProgress, JobResult, JobSummary, MissedPolicy, OperationStatus, OutputInspect, + PartitionState, ProgressInspect, QuarantineEntry, QueueStats, QuietGroupMetrics, + RateClassConfig, RateClassState, ResultInspect, SCHEDULE_EVENT_LIMIT, SaturationStrategy, + Schedule, ScheduleEvent, ScheduleEventOutcome, StateCounts, StoreError, WorkerMeta, + noisy_partition_keys, }; use crate::{JobHash, RedisStore, decode_headers, hn, hs, map_redis_err}; @@ -171,35 +172,35 @@ fn matches_filter(h: &JobHash, f: &JobFilter) -> bool { if !f.tags_any.is_empty() && !f.tags_any.iter().any(|tag| tags.contains(tag)) { return false; } - if let Some(k) = &f.kind { - if hs(h, "kind") != k { - return false; - } + if let Some(k) = &f.kind + && hs(h, "kind") != k + { + return false; } - if let Some(kp) = &f.kind_prefix { - if !hs(h, "kind").starts_with(kp.as_str()) { - return false; - } + if let Some(kp) = &f.kind_prefix + && !hs(h, "kind").starts_with(kp.as_str()) + { + return false; } - if let Some(p) = &f.partition_key { - if hs(h, "partition_key") != p { - return false; - } + if let Some(p) = &f.partition_key + && hs(h, "partition_key") != p + { + return false; } - if let Some(fp) = &f.fingerprint { - if hs(h, "fingerprint") != fp { - return false; - } + if let Some(fp) = &f.fingerprint + && hs(h, "fingerprint") != fp + { + return false; } - if let Some(rc) = &f.rate_class { - if hs(h, "rate_class") != rc { - return false; - } + if let Some(rc) = &f.rate_class + && hs(h, "rate_class") != rc + { + return false; } - if let Some(pr) = f.priority { - if hn(h, "priority") as i32 != pr { - return false; - } + if let Some(pr) = f.priority + && hn(h, "priority") as i32 != pr + { + return false; } true } @@ -870,7 +871,7 @@ impl Inspect for RedisStore { reason: hs(m, "reason").to_string(), }) .collect(); - out.sort_by(|a, b| b.quarantined_at_ms.cmp(&a.quarantined_at_ms)); + out.sort_by_key(|entry| std::cmp::Reverse(entry.quarantined_at_ms)); Ok(out) } @@ -894,7 +895,7 @@ impl Inspect for RedisStore { replaced: false, }), _ => Err(StoreError::Invalid(format!( - "operator_retry is only defined from archived or cancelled; job {id} is {}", + "operator_retry is only defined from archived, cancelled, or undecodable; job {id} is {}", res.get(1).map(String::as_str).unwrap_or("?") ))), } @@ -1241,6 +1242,91 @@ impl Inspect for RedisStore { .collect() } + async fn append_durable_event( + &self, + event: &DurableEvent, + ) -> Result<(DurableEvent, bool), StoreError> { + validate_durable_event(event)?; + let payload = std::str::from_utf8(&event.payload) + .map_err(|_| StoreError::Invalid("durable event payload must be UTF-8 JSON".into()))?; + let source = std::str::from_utf8(&event.source) + .map_err(|_| StoreError::Invalid("durable event source must be UTF-8 JSON".into()))?; + serde_json::from_str::(payload) + .map_err(|_| StoreError::Invalid("durable event payload must be valid JSON".into()))?; + serde_json::from_str::(source) + .map_err(|_| StoreError::Invalid("durable event source must be valid JSON".into()))?; + let now = self.store_now_ms().await?; + let mut conn = self.conn.clone(); + let script = redis::Script::new( + "local old=redis.call('HGET',KEYS[2],ARGV[2])\n\ + if old then return {'0',old} end\n\ + local id=redis.call('INCR',KEYS[3])\n\ + local value=cjson.encode({event_id=id,scope=ARGV[1],topic=ARGV[3],idempotency_key=ARGV[2],payload=cjson.decode(ARGV[4]),source=cjson.decode(ARGV[5]),recorded_at_ms=ARGV[6]})\n\ + redis.call('HSET',KEYS[2],ARGV[2],value)\n\ + redis.call('ZADD',KEYS[1],id,value)\n\ + local excess=redis.call('ZCARD',KEYS[1])-tonumber(ARGV[7])\n\ + if excess>0 then local gone=redis.call('ZRANGE',KEYS[1],0,excess-1) for _,v in ipairs(gone) do local e=cjson.decode(v) redis.call('HDEL',KEYS[2],e.idempotency_key) end redis.call('ZREMRANGEBYRANK',KEYS[1],0,excess-1) end\n\ + return {'1',value}", + ); + let values: Vec> = script + .key(format!("{}:durable-events:{}", self.prefix, event.scope)) + .key(format!( + "{}:durable-event-idem:{}", + self.prefix, event.scope + )) + .key(format!("{}:durable-event-seq", self.prefix)) + .arg(&event.scope) + .arg(&event.idempotency_key) + .arg(&event.topic) + .arg(payload) + .arg(source) + .arg(now) + .arg(DURABLE_EVENT_LIMIT) + .invoke_async(&mut conn) + .await + .map_err(map_redis_err)?; + let inserted = values.first().is_some_and(|v| v.as_slice() == b"1"); + let stored = decode_durable_event(values.get(1).ok_or_else(|| { + StoreError::Invalid("durable event append returned no record".into()) + })?)?; + if stored.topic != event.topic + || stored.payload != event.payload + || stored.source != event.source + { + return Err(StoreError::Invalid( + "durable event idempotency key was reused with different content".into(), + )); + } + Ok((stored, inserted)) + } + + async fn list_durable_events( + &self, + scope: &str, + before_event_id: Option, + limit: u32, + ) -> Result, StoreError> { + headgate_core::validate_durable_event_limit(limit)?; + let mut conn = self.conn.clone(); + let max = before_event_id + .map(|id| format!("({id}")) + .unwrap_or_else(|| "+inf".into()); + let values: Vec> = redis::cmd("ZREVRANGEBYSCORE") + .arg(format!("{}:durable-events:{scope}", self.prefix)) + .arg(max) + .arg("-inf") + .arg("LIMIT") + .arg(0) + .arg(limit) + .query_async(&mut conn) + .await + .map_err(map_redis_err)?; + values + .iter() + .map(|value| decode_durable_event(value)) + .collect() + } + async fn heartbeat_worker(&self, w: &WorkerMeta) -> Result, StoreError> { let mut conn = self.conn.clone(); let status = if w.status.is_empty() { @@ -1330,12 +1416,12 @@ impl Inspect for RedisStore { worker_id: &str, command: Option<&str>, ) -> Result<(), StoreError> { - if let Some(cmd) = command { - if !headgate_core::valid_worker_command(cmd) { - return Err(StoreError::Invalid( - "command must be quiet, resume, restart, terminate, or resign".into(), - )); - } + if let Some(cmd) = command + && !headgate_core::valid_worker_command(cmd) + { + return Err(StoreError::Invalid( + "command must be quiet, resume, restart, terminate, or resign".into(), + )); } let mut conn = self.conn.clone(); let n: i64 = self @@ -1516,7 +1602,7 @@ impl Inspect for RedisStore { let mut total = 0u64; for id in &ids { let ok = format!("{}:op:{id}", self.prefix); - let h = &self.hashes(&[ok.clone()]).await?[0]; + let h = &self.hashes(std::slice::from_ref(&ok)).await?[0]; let action = hs(h, "action").to_string(); let done_with = |status: &str| { let mut pipe = redis::pipe(); @@ -1613,6 +1699,27 @@ impl Inspect for RedisStore { } } + async fn schedule_pending_job(&self, id: &str, at_ms: i64) -> Result<(), StoreError> { + if at_ms <= 0 { + return Err(StoreError::Invalid( + "pending schedule timestamp must be positive".into(), + )); + } + let res = self + .admin_job_op(&["schedule_pending", id, &at_ms.to_string()]) + .await?; + match res.first().map(String::as_str) { + Some("OK") => Ok(()), + Some("NF") => Err(StoreError::NotFound(id.into())), + Some("ERR") => Err(StoreError::Invalid( + "workflow_schedule is defined only from pending".into(), + )), + _ => Err(StoreError::Backend( + "invalid schedule_pending response".into(), + )), + } + } + async fn delete_queue(&self, queue: &str, force: bool) -> Result, StoreError> { let now = self.store_now_ms().await?; let id = format!( @@ -1702,6 +1809,51 @@ impl Inspect for RedisStore { } } +fn validate_durable_event(event: &DurableEvent) -> Result<(), StoreError> { + if event.scope.is_empty() + || event.scope.len() > 512 + || event.topic.is_empty() + || event.topic.len() > 255 + || event.idempotency_key.is_empty() + || event.idempotency_key.len() > 255 + { + return Err(StoreError::Invalid( + "durable event scope, topic, or idempotency key is invalid".into(), + )); + } + if event.payload.len() > headgate_core::MAX_DURABLE_EVENT_PAYLOAD_BYTES + || serde_json::from_slice::(&event.payload).is_err() + { + return Err(StoreError::Invalid( + "durable event payload must be valid JSON of at most 65536 bytes".into(), + )); + } + if event.source.len() > headgate_core::MAX_DURABLE_EVENT_SOURCE_BYTES + || serde_json::from_slice::(&event.source).is_err() + { + return Err(StoreError::Invalid( + "durable event source must be valid JSON of at most 16384 bytes".into(), + )); + } + Ok(()) +} + +fn decode_durable_event(bytes: &[u8]) -> Result { + let value: serde_json::Value = serde_json::from_slice(bytes) + .map_err(|e| StoreError::Invalid(format!("invalid stored durable event: {e}")))?; + Ok(DurableEvent { + event_id: value["event_id"].as_u64().unwrap_or_default(), + scope: value["scope"].as_str().unwrap_or_default().into(), + topic: value["topic"].as_str().unwrap_or_default().into(), + idempotency_key: value["idempotency_key"].as_str().unwrap_or_default().into(), + payload: serde_json::to_vec(&value["payload"]) + .map_err(|e| StoreError::Invalid(e.to_string()))?, + source: serde_json::to_vec(&value["source"]) + .map_err(|e| StoreError::Invalid(e.to_string()))?, + recorded_at_ms: value["recorded_at_ms"].as_i64().unwrap_or_default(), + }) +} + #[async_trait::async_trait] impl ResultInspect for RedisStore { async fn get_job_result(&self, id: &str) -> Result, StoreError> { diff --git a/crates/headgate-redis/src/lib.rs b/crates/headgate-redis/src/lib.rs index a9957e7..70dfdde 100644 --- a/crates/headgate-redis/src/lib.rs +++ b/crates/headgate-redis/src/lib.rs @@ -943,7 +943,7 @@ impl headgate_core::Notifying for RedisStore { match tokio::time::timeout_at(deadline, rx.recv()).await { Err(_) => return Ok(None), // timeout: the poll fallback takes it Ok(Ok(queue)) => { - if queues.is_empty() || queues.iter().any(|q| *q == queue) { + if queues.is_empty() || queues.contains(&queue) { return Ok(Some(queue)); } } diff --git a/crates/headgate-shared/src/log.rs b/crates/headgate-shared/src/log.rs index ac93382..e8517b3 100644 --- a/crates/headgate-shared/src/log.rs +++ b/crates/headgate-shared/src/log.rs @@ -125,44 +125,42 @@ impl LogEntry { fields: Map::new(), truncated: false, }; - if line.len() <= MAX_LOG_BYTES { - if let Some(body) = line.strip_prefix(LOG_PREFIX) { - if let Ok(value) = serde_json::from_str::(body) { - if let (Some(level), Some(message)) = ( - value - .get("level") - .and_then(Value::as_str) - .and_then(LogLevel::parse), - value.get("message").and_then(Value::as_str), - ) { - let fields_valid = value.get("fields").is_none_or(|fields| { - fields.as_object().is_some_and(|fields| { - fields.values().all(|v| !v.is_array() && !v.is_object()) - }) - }); - if !fields_valid - || value.get("at_ms").is_some_and(|at| !at.is_i64()) - || value.get("truncated").is_some_and(|v| !v.is_boolean()) - { - return plain(); - } - return Self { - level, - at_ms: value.get("at_ms").and_then(Value::as_i64), - message: message.to_owned(), - fields: value - .get("fields") - .and_then(Value::as_object) - .cloned() - .unwrap_or_default(), - truncated: value - .get("truncated") - .and_then(Value::as_bool) - .unwrap_or(false), - }; - } - } + if line.len() <= MAX_LOG_BYTES + && let Some(body) = line.strip_prefix(LOG_PREFIX) + && let Ok(value) = serde_json::from_str::(body) + && let (Some(level), Some(message)) = ( + value + .get("level") + .and_then(Value::as_str) + .and_then(LogLevel::parse), + value.get("message").and_then(Value::as_str), + ) + { + let fields_valid = value.get("fields").is_none_or(|fields| { + fields + .as_object() + .is_some_and(|fields| fields.values().all(|v| !v.is_array() && !v.is_object())) + }); + if !fields_valid + || value.get("at_ms").is_some_and(|at| !at.is_i64()) + || value.get("truncated").is_some_and(|v| !v.is_boolean()) + { + return plain(); } + return Self { + level, + at_ms: value.get("at_ms").and_then(Value::as_i64), + message: message.to_owned(), + fields: value + .get("fields") + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(), + truncated: value + .get("truncated") + .and_then(Value::as_bool) + .unwrap_or(false), + }; } plain() } diff --git a/crates/headgate-testkit/src/database.rs b/crates/headgate-testkit/src/database.rs index e7c7422..9427d50 100644 --- a/crates/headgate-testkit/src/database.rs +++ b/crates/headgate-testkit/src/database.rs @@ -75,7 +75,7 @@ impl PostgresTestDatabase { .map_err(|error| TestDatabaseError::new(format!("create schema {schema}: {error}")))?; let mut test_config = admin_config.clone(); - test_config.options(&format!("-c search_path={schema}")); + test_config.options(format!("-c search_path={schema}")); let migrated = migrate_postgres_in_schema( &mut admin, &schema, diff --git a/crates/headgate-testkit/src/lib.rs b/crates/headgate-testkit/src/lib.rs index 51deeb6..a90fe3f 100644 --- a/crates/headgate-testkit/src/lib.rs +++ b/crates/headgate-testkit/src/lib.rs @@ -571,10 +571,11 @@ fn default_backoff(attempt: i64, base: i64, cap: i64) -> i64 { fn release_unique(inner: &mut Inner, id: &str) { let Some(j) = inner.jobs.get(id) else { return }; - if let Some(k) = headgate_core::effective_unique_key(&j.env) { - if j.env.unique_window_ms == 0 && inner.unique.get(&k).map(String::as_str) == Some(id) { - inner.unique.remove(&k); - } + if let Some(k) = headgate_core::effective_unique_key(&j.env) + && j.env.unique_window_ms == 0 + && inner.unique.get(&k).map(String::as_str) == Some(id) + { + inner.unique.remove(&k); } } @@ -641,57 +642,55 @@ impl Store for MemStore { }; if let Some(id) = holder { let mut replaced = false; - if e.unique_replace != 0 || e.unique_debounce_ms > 0 { - if let Some(job) = inner.jobs.get_mut(&id) { - if matches!(job.state.as_str(), "scheduled" | "available" | "retryable") - { - let mask = e.unique_replace; - if e.unique_debounce_ms > 0 { - job.env.schema_version = if e.schema_version == 0 { - 1 - } else { - e.schema_version - }; - job.env.payload.clone_from(&e.payload); - job.env.fingerprint.clone_from(&e.fingerprint); - job.env.tags = headgate_core::canonical_tags(&e.tags); - job.env.scheduled_at_ms = now + e.unique_debounce_ms; - job.state = "scheduled".into(); - replaced = true; - } - if mask & headgate_core::UNIQUE_REPLACE_PAYLOAD != 0 { - job.env.schema_version = if e.schema_version == 0 { - 1 - } else { - e.schema_version - }; - job.env.payload.clone_from(&e.payload); - job.env.fingerprint.clone_from(&e.fingerprint); - replaced = true; - } - if mask & headgate_core::UNIQUE_REPLACE_SCHEDULED_AT != 0 - && job.state == "scheduled" - { - job.env.scheduled_at_ms = if e.scheduled_at_ms == 0 { - now - } else { - e.scheduled_at_ms - }; - replaced = true; - } - if mask & headgate_core::UNIQUE_REPLACE_PRIORITY != 0 { - job.env.priority = e.priority; - replaced = true; - } - if mask & headgate_core::UNIQUE_REPLACE_MAX_ATTEMPTS != 0 { - job.env.max_attempts = if e.max_attempts == 0 { - 25 - } else { - e.max_attempts - }; - replaced = true; - } - } + if (e.unique_replace != 0 || e.unique_debounce_ms > 0) + && let Some(job) = inner.jobs.get_mut(&id) + && matches!(job.state.as_str(), "scheduled" | "available" | "retryable") + { + let mask = e.unique_replace; + if e.unique_debounce_ms > 0 { + job.env.schema_version = if e.schema_version == 0 { + 1 + } else { + e.schema_version + }; + job.env.payload.clone_from(&e.payload); + job.env.fingerprint.clone_from(&e.fingerprint); + job.env.tags = headgate_core::canonical_tags(&e.tags); + job.env.scheduled_at_ms = now + e.unique_debounce_ms; + job.state = "scheduled".into(); + replaced = true; + } + if mask & headgate_core::UNIQUE_REPLACE_PAYLOAD != 0 { + job.env.schema_version = if e.schema_version == 0 { + 1 + } else { + e.schema_version + }; + job.env.payload.clone_from(&e.payload); + job.env.fingerprint.clone_from(&e.fingerprint); + replaced = true; + } + if mask & headgate_core::UNIQUE_REPLACE_SCHEDULED_AT != 0 + && job.state == "scheduled" + { + job.env.scheduled_at_ms = if e.scheduled_at_ms == 0 { + now + } else { + e.scheduled_at_ms + }; + replaced = true; + } + if mask & headgate_core::UNIQUE_REPLACE_PRIORITY != 0 { + job.env.priority = e.priority; + replaced = true; + } + if mask & headgate_core::UNIQUE_REPLACE_MAX_ATTEMPTS != 0 { + job.env.max_attempts = if e.max_attempts == 0 { + 25 + } else { + e.max_attempts + }; + replaced = true; } } return Err(StoreError::Duplicate { @@ -882,17 +881,17 @@ impl Store for MemStore { let j = &inner.jobs[&id]; (j.env.rate_class.clone(), j.rate_charge) }; - if charge > 0 { - if let Some(b) = inner.rate.get_mut(&rc) { - let gained = if b.limit > 0 && b.window > 0 { - (now - b.refilled).max(0) * b.limit / b.window - } else { - 0 - }; - let avail = b.burst.min(b.tokens + gained); - b.tokens = b.burst.min(avail + charge - actual as i64); - b.refilled = now; - } + if charge > 0 + && let Some(b) = inner.rate.get_mut(&rc) + { + let gained = if b.limit > 0 && b.window > 0 { + (now - b.refilled).max(0) * b.limit / b.window + } else { + 0 + }; + let avail = b.burst.min(b.tokens + gained); + b.tokens = b.burst.min(avail + charge - actual as i64); + b.refilled = now; } inner.jobs.get_mut(&id).unwrap().rate_charge = 0; } @@ -1134,10 +1133,11 @@ impl Store for MemStore { } let now = self.now(); let mut inner = self.inner.lock().unwrap(); - if let Some((h, expires)) = inner.duties.get(name) { - if *expires > now && h != holder { - return Ok(false); - } + if let Some((h, expires)) = inner.duties.get(name) + && *expires > now + && h != holder + { + return Ok(false); } inner .duties @@ -1213,17 +1213,17 @@ impl headgate_core::ResultStore for MemStore { let job = &inner.jobs[&id]; (job.env.rate_class.clone(), job.rate_charge) }; - if charge > 0 { - if let Some(bucket) = inner.rate.get_mut(&rc) { - let gained = if bucket.limit > 0 && bucket.window > 0 { - (now - bucket.refilled).max(0) * bucket.limit / bucket.window - } else { - 0 - }; - let available = bucket.burst.min(bucket.tokens + gained); - bucket.tokens = bucket.burst.min(available + charge - actual as i64); - bucket.refilled = now; - } + if charge > 0 + && let Some(bucket) = inner.rate.get_mut(&rc) + { + let gained = if bucket.limit > 0 && bucket.window > 0 { + (now - bucket.refilled).max(0) * bucket.limit / bucket.window + } else { + 0 + }; + let available = bucket.burst.min(bucket.tokens + gained); + bucket.tokens = bucket.burst.min(available + charge - actual as i64); + bucket.refilled = now; } inner.jobs.get_mut(&id).unwrap().rate_charge = 0; } diff --git a/crates/headgate-workflow/Cargo.toml b/crates/headgate-workflow/Cargo.toml index 395271d..08bbb4a 100644 --- a/crates/headgate-workflow/Cargo.toml +++ b/crates/headgate-workflow/Cargo.toml @@ -18,7 +18,12 @@ headgate-core = { workspace = true } futures-util = "0.3" serde = { version = "1", features = ["derive"] } serde_json = "1" +cel = "0.14.4" [dev-dependencies] +headgate-mysql = { workspace = true } headgate-postgres = { workspace = true } +headgate-redis = { workspace = true } +headgate-testkit = { workspace = true } +redis = { version = "0.27", features = ["tokio-comp", "connection-manager"] } tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } diff --git a/crates/headgate-workflow/src/experimental.rs b/crates/headgate-workflow/src/experimental.rs deleted file mode 100644 index c696c78..0000000 --- a/crates/headgate-workflow/src/experimental.rs +++ /dev/null @@ -1,650 +0,0 @@ -//! Experimental workflow semantics. This reducer is intentionally store-agnostic: it -//! settles behavior before durable adapters and control APIs make the contract permanent. - -use std::collections::{BTreeMap, BTreeSet, VecDeque}; - -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum NodeKind { - Task, - Signal { signal: String }, - Timer { wake_at_ms: i64 }, - ChildWorkflow { workflow_id: String }, -} - -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct NodeSpec { - pub name: String, - #[serde(default)] - pub deps: Vec, - pub kind: NodeKind, -} - -impl NodeSpec { - pub fn task( - name: impl Into, - deps: impl IntoIterator>, - ) -> Self { - Self { - name: name.into(), - deps: deps.into_iter().map(Into::into).collect(), - kind: NodeKind::Task, - } - } - - pub fn signal( - name: impl Into, - signal: impl Into, - deps: impl IntoIterator>, - ) -> Self { - Self { - name: name.into(), - deps: deps.into_iter().map(Into::into).collect(), - kind: NodeKind::Signal { - signal: signal.into(), - }, - } - } - - pub fn timer( - name: impl Into, - wake_at_ms: i64, - deps: impl IntoIterator>, - ) -> Self { - Self { - name: name.into(), - deps: deps.into_iter().map(Into::into).collect(), - kind: NodeKind::Timer { wake_at_ms }, - } - } - - pub fn child( - name: impl Into, - workflow_id: impl Into, - deps: impl IntoIterator>, - ) -> Self { - Self { - name: name.into(), - deps: deps.into_iter().map(Into::into).collect(), - kind: NodeKind::ChildWorkflow { - workflow_id: workflow_id.into(), - }, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum NodeState { - Waiting, - Active, - Succeeded, - Failed, - Blocked, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RunStatus { - Running, - Succeeded, - Failed, -} - -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct RuntimeNode { - pub spec: NodeSpec, - pub state: NodeState, -} - -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct Run { - pub revision: u64, - pub generation: u32, - pub status: RunStatus, - pub store_now_ms: i64, - pub nodes: BTreeMap, - pub signals: BTreeSet, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum Command { - Signal { - signal: String, - }, - AdvanceStoreTime { - now_ms: i64, - }, - SucceedNode { - name: String, - }, - FailNode { - name: String, - }, - Graft { - expected_revision: u64, - nodes: Vec, - }, - RetryFailedSubgraph { - expected_revision: u64, - }, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum Action { - DispatchTask { - name: String, - generation: u32, - }, - WaitForSignal { - name: String, - signal: String, - }, - ArmTimer { - name: String, - wake_at_ms: i64, - }, - StartChildWorkflow { - name: String, - workflow_id: String, - generation: u32, - }, - WorkflowSucceeded { - generation: u32, - }, - WorkflowFailed { - name: String, - generation: u32, - }, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ExperimentError(pub String); - -impl std::fmt::Display for ExperimentError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.0) - } -} - -impl std::error::Error for ExperimentError {} - -impl Run { - pub fn new( - nodes: Vec, - store_now_ms: i64, - ) -> Result<(Self, Vec), ExperimentError> { - validate_graph(&nodes)?; - let mut run = Self { - revision: 1, - generation: 1, - status: RunStatus::Running, - store_now_ms, - nodes: nodes - .into_iter() - .map(|spec| { - ( - spec.name.clone(), - RuntimeNode { - spec, - state: NodeState::Waiting, - }, - ) - }) - .collect(), - signals: BTreeSet::new(), - }; - let actions = run.reconcile(); - Ok((run, actions)) - } - - pub fn apply(&mut self, command: Command) -> Result, ExperimentError> { - let mut actions = Vec::new(); - match command { - Command::Signal { signal } => { - if signal.is_empty() { - return Err(ExperimentError("signal name must not be empty".into())); - } - if !self.nodes.values().any( - |node| matches!(&node.spec.kind, NodeKind::Signal { signal: expected } if expected == &signal), - ) { - return Err(ExperimentError(format!("unknown signal `{signal}`"))); - } - self.signals.insert(signal.clone()); - for node in self.nodes.values_mut() { - if node.state == NodeState::Active - && matches!(&node.spec.kind, NodeKind::Signal { signal: expected } if expected == &signal) - { - node.state = NodeState::Succeeded; - } - } - } - Command::AdvanceStoreTime { now_ms } => { - if now_ms < self.store_now_ms { - return Err(ExperimentError("store time must not move backwards".into())); - } - self.store_now_ms = now_ms; - for node in self.nodes.values_mut() { - if node.state == NodeState::Active - && matches!(node.spec.kind, NodeKind::Timer { wake_at_ms } if wake_at_ms <= now_ms) - { - node.state = NodeState::Succeeded; - } - } - } - Command::SucceedNode { name } => self.settle_node(&name, true)?, - Command::FailNode { name } => { - self.settle_node(&name, false)?; - self.block_descendants(&name); - self.status = RunStatus::Failed; - actions.push(Action::WorkflowFailed { - name, - generation: self.generation, - }); - } - Command::Graft { - expected_revision, - nodes, - } => { - self.require_revision(expected_revision)?; - if self.status != RunStatus::Running { - return Err(ExperimentError( - "nodes may only be grafted onto a running workflow".into(), - )); - } - if nodes.is_empty() { - return Err(ExperimentError( - "graft must contain at least one node".into(), - )); - } - let mut combined: Vec = - self.nodes.values().map(|node| node.spec.clone()).collect(); - for node in &nodes { - if self.nodes.contains_key(&node.name) { - return Err(ExperimentError(format!( - "graft repeats existing node `{}`", - node.name - ))); - } - combined.push(node.clone()); - } - validate_graph(&combined)?; - for spec in nodes { - self.nodes.insert( - spec.name.clone(), - RuntimeNode { - spec, - state: NodeState::Waiting, - }, - ); - } - self.revision += 1; - } - Command::RetryFailedSubgraph { expected_revision } => { - self.require_revision(expected_revision)?; - if self.status != RunStatus::Failed { - return Err(ExperimentError( - "only a failed workflow may be retried".into(), - )); - } - for node in self.nodes.values_mut() { - if matches!(node.state, NodeState::Failed | NodeState::Blocked) { - node.state = NodeState::Waiting; - } - } - self.generation = self - .generation - .checked_add(1) - .ok_or_else(|| ExperimentError("workflow generation overflow".into()))?; - self.revision += 1; - self.status = RunStatus::Running; - } - } - actions.extend(self.reconcile()); - Ok(actions) - } - - fn require_revision(&self, expected: u64) -> Result<(), ExperimentError> { - if expected != self.revision { - return Err(ExperimentError(format!( - "revision conflict: expected {expected}, current {}", - self.revision - ))); - } - Ok(()) - } - - fn settle_node(&mut self, name: &str, succeeded: bool) -> Result<(), ExperimentError> { - let node = self - .nodes - .get_mut(name) - .ok_or_else(|| ExperimentError(format!("unknown node `{name}`")))?; - if node.state != NodeState::Active { - return Err(ExperimentError(format!("node `{name}` is not active"))); - } - if !matches!( - node.spec.kind, - NodeKind::Task | NodeKind::ChildWorkflow { .. } - ) { - return Err(ExperimentError(format!( - "node `{name}` is settled by its signal or timer" - ))); - } - node.state = if succeeded { - NodeState::Succeeded - } else { - NodeState::Failed - }; - Ok(()) - } - - fn block_descendants(&mut self, failed: &str) { - let mut queue = VecDeque::from([failed.to_string()]); - while let Some(parent) = queue.pop_front() { - let children: Vec = self - .nodes - .values() - .filter(|node| node.spec.deps.iter().any(|dep| dep == &parent)) - .map(|node| node.spec.name.clone()) - .collect(); - for child in children { - if let Some(node) = self.nodes.get_mut(&child) { - if matches!(node.state, NodeState::Waiting | NodeState::Active) { - node.state = NodeState::Blocked; - queue.push_back(child); - } - } - } - } - } - - fn reconcile(&mut self) -> Vec { - if self.status != RunStatus::Running { - return Vec::new(); - } - let mut actions = Vec::new(); - loop { - let ready: Vec = self - .nodes - .values() - .filter(|node| node.state == NodeState::Waiting) - .filter(|node| { - node.spec.deps.iter().all(|dep| { - self.nodes - .get(dep) - .is_some_and(|upstream| upstream.state == NodeState::Succeeded) - }) - }) - .map(|node| node.spec.name.clone()) - .collect(); - if ready.is_empty() { - break; - } - let mut completed_virtual = false; - for name in ready { - let node = self.nodes.get_mut(&name).expect("ready node exists"); - match &node.spec.kind { - NodeKind::Task => { - node.state = NodeState::Active; - actions.push(Action::DispatchTask { - name, - generation: self.generation, - }); - } - NodeKind::Signal { signal } if self.signals.contains(signal) => { - node.state = NodeState::Succeeded; - completed_virtual = true; - } - NodeKind::Signal { signal } => { - node.state = NodeState::Active; - actions.push(Action::WaitForSignal { - name, - signal: signal.clone(), - }); - } - NodeKind::Timer { wake_at_ms } if *wake_at_ms <= self.store_now_ms => { - node.state = NodeState::Succeeded; - completed_virtual = true; - } - NodeKind::Timer { wake_at_ms } => { - node.state = NodeState::Active; - actions.push(Action::ArmTimer { - name, - wake_at_ms: *wake_at_ms, - }); - } - NodeKind::ChildWorkflow { workflow_id } => { - node.state = NodeState::Active; - actions.push(Action::StartChildWorkflow { - name, - workflow_id: workflow_id.clone(), - generation: self.generation, - }); - } - } - } - if !completed_virtual { - break; - } - } - if self - .nodes - .values() - .all(|node| node.state == NodeState::Succeeded) - { - self.status = RunStatus::Succeeded; - actions.push(Action::WorkflowSucceeded { - generation: self.generation, - }); - } - actions - } -} - -fn validate_graph(nodes: &[NodeSpec]) -> Result<(), ExperimentError> { - if nodes.is_empty() { - return Err(ExperimentError( - "workflow must contain at least one node".into(), - )); - } - let mut names = BTreeSet::new(); - for node in nodes { - if node.name.is_empty() || !names.insert(node.name.as_str()) { - return Err(ExperimentError( - "node names must be non-empty and unique".into(), - )); - } - if matches!(&node.kind, NodeKind::Signal { signal } if signal.is_empty()) { - return Err(ExperimentError(format!( - "signal node `{}` has an empty signal", - node.name - ))); - } - if matches!(&node.kind, NodeKind::ChildWorkflow { workflow_id } if workflow_id.is_empty()) { - return Err(ExperimentError(format!( - "child node `{}` has an empty workflow id", - node.name - ))); - } - } - let mut degree: BTreeMap<&str, usize> = - nodes.iter().map(|node| (node.name.as_str(), 0)).collect(); - let mut outgoing: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); - for node in nodes { - let mut unique = BTreeSet::new(); - for dep in &node.deps { - if !names.contains(dep.as_str()) { - return Err(ExperimentError(format!( - "node `{}` depends on missing node `{dep}`", - node.name - ))); - } - if !unique.insert(dep.as_str()) { - return Err(ExperimentError(format!( - "node `{}` repeats dependency `{dep}`", - node.name - ))); - } - *degree - .get_mut(node.name.as_str()) - .expect("node degree exists") += 1; - outgoing.entry(dep).or_default().push(&node.name); - } - } - let mut queue: VecDeque<&str> = degree - .iter() - .filter_map(|(name, count)| (*count == 0).then_some(*name)) - .collect(); - let mut visited = 0; - while let Some(name) = queue.pop_front() { - visited += 1; - for child in outgoing.get(name).into_iter().flatten() { - let count = degree.get_mut(child).expect("child degree exists"); - *count -= 1; - if *count == 0 { - queue.push_back(child); - } - } - } - if visited != nodes.len() { - return Err(ExperimentError("workflow graph contains a cycle".into())); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn names(actions: &[Action]) -> Vec<&str> { - actions - .iter() - .filter_map(|action| match action { - Action::DispatchTask { name, .. } | Action::StartChildWorkflow { name, .. } => { - Some(name.as_str()) - } - _ => None, - }) - .collect() - } - - #[test] - fn signals_and_store_time_timers_unlock_in_dependency_order() { - let (mut run, first) = Run::new( - vec![ - NodeSpec::task("prepare", Vec::::new()), - NodeSpec::signal("approval", "approved", ["prepare"]), - NodeSpec::timer("release", 1_500, ["approval"]), - NodeSpec::task("publish", ["release"]), - ], - 1_000, - ) - .unwrap(); - assert_eq!(names(&first), ["prepare"]); - let unknown = run - .apply(Command::Signal { - signal: "typo".into(), - }) - .unwrap_err(); - assert!(unknown.0.contains("unknown signal")); - assert!( - run.apply(Command::Signal { - signal: "approved".into() - }) - .unwrap() - .is_empty() - ); - let wait = run - .apply(Command::SucceedNode { - name: "prepare".into(), - }) - .unwrap(); - assert!(wait.iter().any( - |a| matches!(a, Action::ArmTimer { name, wake_at_ms: 1_500 } if name == "release") - )); - assert!( - run.apply(Command::AdvanceStoreTime { now_ms: 1_499 }) - .unwrap() - .is_empty() - ); - assert_eq!( - names( - &run.apply(Command::AdvanceStoreTime { now_ms: 1_500 }) - .unwrap() - ), - ["publish"] - ); - } - - #[test] - fn graft_is_additive_revision_checked_and_cycle_safe() { - let (mut run, _) = Run::new(vec![NodeSpec::task("root", Vec::::new())], 0).unwrap(); - let actions = run - .apply(Command::Graft { - expected_revision: 1, - nodes: vec![NodeSpec::task("grafted", ["root"])], - }) - .unwrap(); - assert!(actions.is_empty()); - assert_eq!(run.revision, 2); - let stale = run - .apply(Command::Graft { - expected_revision: 1, - nodes: vec![NodeSpec::task("stale", ["root"])], - }) - .unwrap_err(); - assert!(stale.0.contains("revision conflict")); - let cycle = run - .apply(Command::Graft { - expected_revision: 2, - nodes: vec![NodeSpec::task("a", ["b"]), NodeSpec::task("b", ["a"])], - }) - .unwrap_err(); - assert!(cycle.0.contains("cycle")); - } - - #[test] - fn nested_failure_retries_only_failed_subgraph() { - let (mut run, first) = Run::new( - vec![ - NodeSpec::task("extract", Vec::::new()), - NodeSpec::child("child", "child-workflow", ["extract"]), - NodeSpec::task("finish", ["child"]), - ], - 0, - ) - .unwrap(); - assert_eq!(names(&first), ["extract"]); - let child = run - .apply(Command::SucceedNode { - name: "extract".into(), - }) - .unwrap(); - assert_eq!(names(&child), ["child"]); - let failed = run - .apply(Command::FailNode { - name: "child".into(), - }) - .unwrap(); - assert!(failed.iter().any( - |a| matches!(a, Action::WorkflowFailed { name, generation: 1 } if name == "child") - )); - assert_eq!(run.nodes["extract"].state, NodeState::Succeeded); - assert_eq!(run.nodes["finish"].state, NodeState::Blocked); - let retried = run - .apply(Command::RetryFailedSubgraph { - expected_revision: 1, - }) - .unwrap(); - assert_eq!(run.generation, 2); - assert_eq!(run.nodes["extract"].state, NodeState::Succeeded); - assert_eq!(names(&retried), ["child"]); - assert!( - retried - .iter() - .any(|a| matches!(a, Action::StartChildWorkflow { generation: 2, .. })) - ); - } -} diff --git a/crates/headgate-workflow/src/lib.rs b/crates/headgate-workflow/src/lib.rs index 85c7086..9252d24 100644 --- a/crates/headgate-workflow/src/lib.rs +++ b/crates/headgate-workflow/src/lib.rs @@ -1,7 +1,5 @@ //! Durable DAG dependencies layered on headgate's ordinary pending jobs. -pub mod experimental; - use std::{ collections::{HashMap, HashSet, VecDeque}, sync::Arc, @@ -10,13 +8,18 @@ use std::{ use futures_util::{StreamExt, TryStreamExt, stream}; use headgate::{CodecError, Control, Envelope, JobCtx, JobError, Registry, Task}; -use headgate_core::{Inspect, MAX_ENQUEUE_BATCH_SIZE, MAX_JOB_IDENTIFIER_LEN}; +use headgate_core::{ + DurableEvent, Inspect, JobFilter, MAX_ENQUEUE_BATCH_SIZE, MAX_JOB_IDENTIFIER_LEN, +}; use serde::{Deserialize, Serialize}; const DEFAULT_RETENTION_MS: i64 = 7 * 24 * 60 * 60 * 1000; const MAX_WORKFLOW_NODES: usize = MAX_ENQUEUE_BATCH_SIZE - 1; const MAX_WORKFLOW_EDGES: usize = 10_000; +const MAX_WORKFLOW_EVENTS: usize = 256; const WORKFLOW_CONCURRENCY: usize = 16; +const MAX_SIGNAL_PAYLOAD_BYTES: usize = 64 * 1024; +const MAX_SIGNAL_SOURCE_BYTES: usize = 16 * 1024; #[derive(Debug)] pub struct WorkflowError(String); @@ -31,10 +34,20 @@ impl std::error::Error for WorkflowError {} #[derive(Clone)] struct DraftNode { name: String, - envelope: Envelope, + kind: DraftNodeKind, deps: Vec, } +#[derive(Clone)] +enum DraftNodeKind { + Task(Box), + Signal { signal: String }, + TimerAt { wake_at_ms: i64 }, + TimerAfter { delay_ms: i64 }, + ChildWorkflow { workflow_id: String }, + Condition { expression: String }, +} + /// A validated DAG builder. `prepare` returns one atomic enqueue batch containing the /// durable coordinator plus every child in `pending` state. pub struct Workflow { @@ -42,6 +55,280 @@ pub struct Workflow { nodes: Vec, coordinator_queue: String, retention_ms: i64, + failed_subgraph_retry: bool, + retry_policy: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct WorkflowRetryPolicy { + pub max_generations: u32, + pub backoff_ms: i64, +} + +/// A revision-checked set of tasks to graft onto a running workflow. +/// +/// Enqueue the returned batch atomically. The receipt and its pending tasks then either +/// all exist or none do; the coordinator accepts only the receipt for its next revision. +pub struct WorkflowGraft { + workflow_id: String, + expected_revision: u64, + nodes: Vec, + queue: String, + retention_ms: i64, +} + +/// Prepare parent and child workflows as one atomic store enqueue. Every child link in +/// the bundle must name another member, which makes the complete cross-workflow graph +/// available for cycle detection before any row is written. +pub fn prepare_bundle(workflows: Vec) -> Result, WorkflowError> { + if workflows.is_empty() { + return Err(WorkflowError( + "workflow bundle must contain at least one workflow".into(), + )); + } + let ids: HashSet<&str> = workflows + .iter() + .map(|workflow| workflow.id.as_str()) + .collect(); + if ids.len() != workflows.len() || ids.contains("") { + return Err(WorkflowError( + "workflow bundle ids must be non-empty and unique".into(), + )); + } + let mut indegree: HashMap<&str, usize> = ids.iter().map(|id| (*id, 0)).collect(); + let mut outgoing: HashMap<&str, Vec<&str>> = HashMap::new(); + for workflow in &workflows { + let mut children = HashSet::new(); + for child in workflow.nodes.iter().filter_map(|node| match &node.kind { + DraftNodeKind::ChildWorkflow { workflow_id } => Some(workflow_id.as_str()), + _ => None, + }) { + if !ids.contains(child) { + return Err(WorkflowError(format!( + "atomic workflow bundle is missing child `{child}`" + ))); + } + if children.insert(child) { + *indegree.get_mut(child).expect("bundle child exists") += 1; + outgoing + .entry(workflow.id.as_str()) + .or_default() + .push(child); + } + } + } + let mut ready: VecDeque<&str> = indegree + .iter() + .filter_map(|(id, degree)| (*degree == 0).then_some(*id)) + .collect(); + let mut visited = 0; + while let Some(id) = ready.pop_front() { + visited += 1; + for child in outgoing.get(id).into_iter().flatten() { + let degree = indegree.get_mut(child).expect("bundle child exists"); + *degree -= 1; + if *degree == 0 { + ready.push_back(child); + } + } + } + if visited != workflows.len() { + return Err(WorkflowError( + "cross-workflow child graph contains a cycle".into(), + )); + } + let mut batch = Vec::new(); + for workflow in workflows { + batch.extend(workflow.prepare()?); + if batch.len() > MAX_ENQUEUE_BATCH_SIZE { + return Err(WorkflowError(format!( + "workflow bundle must contain at most {MAX_ENQUEUE_BATCH_SIZE} jobs" + ))); + } + } + Ok(batch) +} + +impl WorkflowGraft { + pub fn new(workflow_id: impl Into, expected_revision: u64) -> Self { + Self { + workflow_id: workflow_id.into(), + expected_revision, + nodes: Vec::new(), + queue: "headgate-workflow".into(), + retention_ms: DEFAULT_RETENTION_MS, + } + } + + pub fn queue(mut self, queue: impl Into) -> Self { + self.queue = queue.into(); + self + } + + pub fn retention(mut self, duration: Duration) -> Result { + let ms = i64::try_from(duration.as_millis()) + .map_err(|_| WorkflowError("workflow graft retention is too large".into()))?; + if ms <= 0 { + return Err(WorkflowError( + "workflow graft retention must be at least 1ms".into(), + )); + } + self.retention_ms = ms; + Ok(self) + } + + pub fn add( + mut self, + name: impl Into, + envelope: Envelope, + deps: impl IntoIterator>, + ) -> Self { + self.nodes.push(DraftNode { + name: name.into(), + kind: DraftNodeKind::Task(Box::new(envelope)), + deps: deps.into_iter().map(Into::into).collect(), + }); + self + } + + pub fn prepare(self) -> Result, WorkflowError> { + if self.workflow_id.is_empty() { + return Err(WorkflowError("workflow id must not be empty".into())); + } + if self.expected_revision == 0 { + return Err(WorkflowError( + "workflow graft expected revision must be at least 1".into(), + )); + } + if self.nodes.is_empty() || self.nodes.len() > MAX_WORKFLOW_NODES { + return Err(WorkflowError(format!( + "workflow graft must contain 1-{MAX_WORKFLOW_NODES} tasks" + ))); + } + let next_revision = self + .expected_revision + .checked_add(1) + .ok_or_else(|| WorkflowError("workflow graft revision would overflow".into()))?; + let mut names = HashSet::with_capacity(self.nodes.len()); + let mut specs = Vec::with_capacity(self.nodes.len()); + let mut children = Vec::with_capacity(self.nodes.len()); + for node in self.nodes { + if node.name.is_empty() || node.name.len() > 128 || !names.insert(node.name.clone()) { + return Err(WorkflowError( + "workflow graft task names must be non-empty and unique".into(), + )); + } + let DraftNodeKind::Task(envelope) = node.kind else { + return Err(WorkflowError( + "workflow graft currently accepts ordinary tasks only".into(), + )); + }; + let mut envelope = *envelope; + if envelope.id.is_empty() { + envelope.id = format!("{}:g{}:{}", self.workflow_id, next_revision, node.name); + } + envelope.pending = true; + envelope.scheduled_at_ms = 0; + if envelope.retention_ms < self.retention_ms { + envelope.retention_ms = self.retention_ms; + } + envelope = headgate::prepare_envelope(envelope) + .map_err(|error| WorkflowError(error.to_string()))?; + specs.push(NodeSpec { + name: node.name, + job_id: envelope.id.clone(), + deps: node.deps, + kind: NodeType::Task, + signal: None, + wake_at_ms: None, + delay_ms: None, + child_workflow_id: None, + condition: None, + }); + children.push(envelope); + } + validate_graft_nodes(&specs)?; + let graft = GraftTask { + workflow_id: self.workflow_id.clone(), + expected_revision: self.expected_revision, + nodes: specs, + }; + let payload = graft + .encode() + .map_err(|error| WorkflowError(error.to_string()))?; + let receipt = headgate::prepare_envelope(Envelope { + id: graft_receipt_id(&self.workflow_id, next_revision), + kind: GraftTask::TYPE.into(), + schema_version: GraftTask::VERSION, + fingerprint: headgate::fingerprint(GraftTask::TYPE, &payload), + payload, + queue: self.queue, + pending: true, + retention_ms: self.retention_ms, + ..Default::default() + }) + .map_err(|error| WorkflowError(error.to_string()))?; + let mut batch = Vec::with_capacity(children.len() + 1); + batch.push(receipt); + batch.extend(children); + Ok(batch) + } +} + +fn validate_graft_nodes(nodes: &[NodeSpec]) -> Result<(), WorkflowError> { + let names: HashSet<&str> = nodes.iter().map(|node| node.name.as_str()).collect(); + let mut indegree: HashMap<&str, usize> = + nodes.iter().map(|node| (node.name.as_str(), 0)).collect(); + let mut outgoing: HashMap<&str, Vec<&str>> = HashMap::new(); + let mut edges = 0usize; + for node in nodes { + let mut unique = HashSet::new(); + for dep in &node.deps { + edges = edges.saturating_add(1); + if !unique.insert(dep.as_str()) { + return Err(WorkflowError(format!( + "workflow graft task `{}` repeats dependency `{dep}`", + node.name + ))); + } + if dep == &node.name { + return Err(WorkflowError(format!( + "workflow graft task `{}` depends on itself", + node.name + ))); + } + if names.contains(dep.as_str()) { + *indegree.get_mut(node.name.as_str()).expect("known node") += 1; + outgoing.entry(dep).or_default().push(&node.name); + } + } + } + if edges > MAX_WORKFLOW_EDGES { + return Err(WorkflowError(format!( + "workflow graft must contain at most {MAX_WORKFLOW_EDGES} dependency edges" + ))); + } + let mut ready: VecDeque<&str> = indegree + .iter() + .filter_map(|(name, degree)| (*degree == 0).then_some(*name)) + .collect(); + let mut visited = 0; + while let Some(name) = ready.pop_front() { + visited += 1; + for child in outgoing.get(name).into_iter().flatten() { + let degree = indegree.get_mut(child).expect("known child"); + *degree -= 1; + if *degree == 0 { + ready.push_back(child); + } + } + } + if visited != nodes.len() { + return Err(WorkflowError( + "workflow graft dependency graph contains a cycle".into(), + )); + } + Ok(()) } impl Workflow { @@ -51,6 +338,8 @@ impl Workflow { nodes: Vec::new(), coordinator_queue: "headgate-workflow".into(), retention_ms: DEFAULT_RETENTION_MS, + failed_subgraph_retry: false, + retry_policy: None, } } @@ -71,6 +360,35 @@ impl Workflow { Ok(self) } + /// Retain dependency-blocked pending jobs so a failed generation can be retried + /// without rerunning successful ancestors. + pub fn failed_subgraph_retry(mut self) -> Self { + self.failed_subgraph_retry = true; + self + } + + /// Automatically retry the failed subgraph after a store-timed snooze. The limit + /// includes the initial generation, so `max_generations = 3` permits two retries. + pub fn automatic_retry( + mut self, + max_generations: u32, + backoff: Duration, + ) -> Result { + let backoff_ms = i64::try_from(backoff.as_millis()) + .map_err(|_| WorkflowError("workflow retry backoff is too large".into()))?; + if max_generations < 2 || backoff_ms <= 0 { + return Err(WorkflowError( + "automatic workflow retry requires at least 2 generations and 1ms backoff".into(), + )); + } + self.failed_subgraph_retry = true; + self.retry_policy = Some(WorkflowRetryPolicy { + max_generations, + backoff_ms, + }); + Ok(self) + } + pub fn add( mut self, name: impl Into, @@ -79,7 +397,101 @@ impl Workflow { ) -> Self { self.nodes.push(DraftNode { name: name.into(), - envelope, + kind: DraftNodeKind::Task(Box::new(envelope)), + deps: deps.into_iter().map(Into::into).collect(), + }); + self + } + + /// Add a durable workflow signal. Emission may happen before its dependencies + /// complete; the coordinator buffers that fact and consumes it once the node is + /// eligible. + pub fn add_signal( + mut self, + name: impl Into, + signal: impl Into, + deps: impl IntoIterator>, + ) -> Self { + self.nodes.push(DraftNode { + name: name.into(), + kind: DraftNodeKind::Signal { + signal: signal.into(), + }, + deps: deps.into_iter().map(Into::into).collect(), + }); + self + } + + /// Add an absolute store-time timer. The ordinary scheduled-job promoter supplies + /// the clock, so worker clock skew cannot fire the timer early or late. + pub fn add_timer_at( + mut self, + name: impl Into, + wake_at_ms: i64, + deps: impl IntoIterator>, + ) -> Self { + self.nodes.push(DraftNode { + name: name.into(), + kind: DraftNodeKind::TimerAt { wake_at_ms }, + deps: deps.into_iter().map(Into::into).collect(), + }); + self + } + + /// Add a relative timer anchored to the latest dependency finalization timestamp. + /// The coordinator durably records that store timestamp before scheduling the timer. + pub fn add_timer_after( + mut self, + name: impl Into, + delay: Duration, + deps: impl IntoIterator>, + ) -> Result { + let delay_ms = i64::try_from(delay.as_millis()) + .map_err(|_| WorkflowError("workflow timer delay is too large".into()))?; + if delay_ms <= 0 { + return Err(WorkflowError( + "workflow timer delay must be at least 1ms".into(), + )); + } + self.nodes.push(DraftNode { + name: name.into(), + kind: DraftNodeKind::TimerAfter { delay_ms }, + deps: deps.into_iter().map(Into::into).collect(), + }); + Ok(self) + } + + /// Add an explicit child-workflow link. The child workflow is enqueued separately; + /// this node mirrors its coordinator's terminal state into the parent. + pub fn add_child( + mut self, + name: impl Into, + workflow_id: impl Into, + deps: impl IntoIterator>, + ) -> Self { + self.nodes.push(DraftNode { + name: name.into(), + kind: DraftNodeKind::ChildWorkflow { + workflow_id: workflow_id.into(), + }, + deps: deps.into_iter().map(Into::into).collect(), + }); + self + } + + /// Wait until a CEL expression over `revision`, `generation`, `completed`, and + /// `states` evaluates to true. + pub fn add_condition( + mut self, + name: impl Into, + expression: impl Into, + deps: impl IntoIterator>, + ) -> Self { + self.nodes.push(DraftNode { + name: name.into(), + kind: DraftNodeKind::Condition { + expression: expression.into(), + }, deps: deps.into_iter().map(Into::into).collect(), }); self @@ -99,21 +511,172 @@ impl Workflow { let mut specs = Vec::with_capacity(self.nodes.len()); let mut children = Vec::with_capacity(self.nodes.len()); for node in self.nodes { - let mut envelope = node.envelope; + let (mut envelope, kind, signal, wake_at_ms, delay_ms, child_workflow_id, condition) = + match node.kind { + DraftNodeKind::Task(envelope) => { + (*envelope, NodeType::Task, None, None, None, None, None) + } + DraftNodeKind::Signal { signal } => { + let task = SignalTask { + workflow_id: self.id.clone(), + signal: signal.clone(), + }; + let payload = task + .encode() + .map_err(|error| WorkflowError(error.to_string()))?; + ( + Envelope { + kind: SignalTask::TYPE.into(), + schema_version: SignalTask::VERSION, + fingerprint: headgate::fingerprint(SignalTask::TYPE, &payload), + payload, + queue: self.coordinator_queue.clone(), + ..Default::default() + }, + NodeType::Signal, + Some(signal), + None, + None, + None, + None, + ) + } + DraftNodeKind::TimerAt { wake_at_ms } => { + let task = TimerTask { + workflow_id: self.id.clone(), + wake_at_ms: Some(wake_at_ms), + delay_ms: None, + }; + let payload = task + .encode() + .map_err(|error| WorkflowError(error.to_string()))?; + ( + Envelope { + kind: TimerTask::TYPE.into(), + schema_version: TimerTask::VERSION, + fingerprint: headgate::fingerprint(TimerTask::TYPE, &payload), + payload, + queue: self.coordinator_queue.clone(), + scheduled_at_ms: wake_at_ms, + ..Default::default() + }, + NodeType::Timer, + None, + Some(wake_at_ms), + None, + None, + None, + ) + } + DraftNodeKind::TimerAfter { delay_ms } => { + let task = TimerTask { + workflow_id: self.id.clone(), + wake_at_ms: None, + delay_ms: Some(delay_ms), + }; + let payload = task + .encode() + .map_err(|error| WorkflowError(error.to_string()))?; + ( + Envelope { + kind: TimerTask::TYPE.into(), + schema_version: TimerTask::VERSION, + fingerprint: headgate::fingerprint(TimerTask::TYPE, &payload), + payload, + queue: self.coordinator_queue.clone(), + ..Default::default() + }, + NodeType::Timer, + None, + None, + Some(delay_ms), + None, + None, + ) + } + DraftNodeKind::ChildWorkflow { workflow_id } => { + if workflow_id == self.id { + return Err(WorkflowError( + "workflow cannot contain itself as a child".into(), + )); + } + let task = ChildWorkflowTask { + parent_workflow_id: self.id.clone(), + child_workflow_id: workflow_id.clone(), + }; + let payload = task + .encode() + .map_err(|error| WorkflowError(error.to_string()))?; + ( + Envelope { + kind: ChildWorkflowTask::TYPE.into(), + schema_version: ChildWorkflowTask::VERSION, + fingerprint: headgate::fingerprint( + ChildWorkflowTask::TYPE, + &payload, + ), + payload, + queue: self.coordinator_queue.clone(), + ..Default::default() + }, + NodeType::ChildWorkflow, + None, + None, + None, + Some(workflow_id), + None, + ) + } + DraftNodeKind::Condition { expression } => { + let task = ConditionTask { + workflow_id: self.id.clone(), + expression: expression.clone(), + }; + let payload = task + .encode() + .map_err(|error| WorkflowError(error.to_string()))?; + ( + Envelope { + kind: ConditionTask::TYPE.into(), + schema_version: ConditionTask::VERSION, + fingerprint: headgate::fingerprint(ConditionTask::TYPE, &payload), + payload, + queue: self.coordinator_queue.clone(), + ..Default::default() + }, + NodeType::Condition, + None, + None, + None, + None, + Some(expression), + ) + } + }; if envelope.id.is_empty() { envelope.id = format!("{}:{}", self.id, node.name); } if envelope.retention_ms < self.retention_ms { envelope.retention_ms = self.retention_ms; } - envelope.pending = true; - envelope.scheduled_at_ms = 0; + if kind == NodeType::Timer && wake_at_ms.is_some() { + envelope.pending = false; + } else { + envelope.pending = true; + envelope.scheduled_at_ms = 0; + } envelope = headgate::prepare_envelope(envelope).map_err(|e| WorkflowError(e.to_string()))?; specs.push(NodeSpec { name: node.name, job_id: envelope.id.clone(), deps: node.deps, + kind, + signal, + wake_at_ms, + delay_ms, + child_workflow_id, + condition, }); children.push(envelope); } @@ -121,6 +684,8 @@ impl Workflow { let task = CoordinatorTask { workflow_id: self.id.clone(), nodes: specs, + failed_subgraph_retry: self.failed_subgraph_retry, + retry_policy: self.retry_policy, }; let payload = task.encode().map_err(|e| WorkflowError(e.to_string()))?; let coordinator = headgate::prepare_envelope(Envelope { @@ -163,6 +728,34 @@ fn validate_graph(nodes: &[DraftNode]) -> Result<(), WorkflowError> { node.name ))); } + if matches!(&node.kind, DraftNodeKind::Signal { signal } if signal.is_empty()) { + return Err(WorkflowError(format!( + "workflow signal node `{}` has an empty signal", + node.name + ))); + } + if matches!(&node.kind, DraftNodeKind::TimerAt { wake_at_ms } if *wake_at_ms <= 0) { + return Err(WorkflowError(format!( + "workflow timer node `{}` must have a positive absolute wake time", + node.name + ))); + } + if matches!(node.kind, DraftNodeKind::TimerAfter { .. }) && node.deps.is_empty() { + return Err(WorkflowError(format!( + "relative workflow timer `{}` requires at least one dependency", + node.name + ))); + } + if matches!(&node.kind, DraftNodeKind::ChildWorkflow { workflow_id } if workflow_id.is_empty()) + { + return Err(WorkflowError(format!( + "workflow child node `{}` has an empty workflow id", + node.name + ))); + } + if let DraftNodeKind::Condition { expression } = &node.kind { + validate_condition(expression)?; + } edges = edges.saturating_add(node.deps.len()); let mut unique = HashSet::new(); for dep in &node.deps { @@ -210,22 +803,1119 @@ fn validate_graph(nodes: &[DraftNode]) -> Result<(), WorkflowError> { Ok(()) } +fn validate_condition(expression: &str) -> Result<(), WorkflowError> { + if expression.is_empty() || expression.len() > 1_024 { + return Err(WorkflowError( + "workflow CEL condition must contain 1-1024 bytes".into(), + )); + } + cel::Program::compile(expression) + .map(|_| ()) + .map_err(|error| WorkflowError(format!("invalid workflow CEL condition: {error}"))) +} + #[derive(Clone, Debug, Serialize, Deserialize)] struct NodeSpec { name: String, job_id: String, deps: Vec, + #[serde(default)] + kind: NodeType, + #[serde(default, skip_serializing_if = "Option::is_none")] + signal: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + wake_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + delay_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + child_workflow_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + condition: Option, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum NodeType { + #[default] + Task, + Signal, + Timer, + ChildWorkflow, + Condition, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct CoordinatorTask { pub workflow_id: String, nodes: Vec, + #[serde(default)] + failed_subgraph_retry: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + retry_policy: Option, } -#[derive(Default, Serialize, Deserialize)] -struct WorkflowCursor { - completed: Vec, +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SignalTask { + pub workflow_id: String, + pub signal: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TimerTask { + pub workflow_id: String, + pub wake_at_ms: Option, + pub delay_ms: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ChildWorkflowTask { + pub parent_workflow_id: String, + pub child_workflow_id: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ConditionTask { + pub workflow_id: String, + pub expression: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct GraftTask { + pub workflow_id: String, + pub expected_revision: u64, + nodes: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct RetryTask { + pub workflow_id: String, + pub expected_revision: u64, +} + +impl Task for SignalTask { + const TYPE: &'static str = "headgate:workflow-signal"; + + fn encode(&self) -> Result, CodecError> { + serde_json::to_vec(self).map_err(|error| CodecError::Malformed(error.to_string())) + } + + fn decode(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(|error| CodecError::Malformed(error.to_string())) + } +} + +impl Task for TimerTask { + const TYPE: &'static str = "headgate:workflow-timer"; + + fn encode(&self) -> Result, CodecError> { + serde_json::to_vec(self).map_err(|error| CodecError::Malformed(error.to_string())) + } + + fn decode(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(|error| CodecError::Malformed(error.to_string())) + } +} + +impl Task for ChildWorkflowTask { + const TYPE: &'static str = "headgate:workflow-child"; + + fn encode(&self) -> Result, CodecError> { + serde_json::to_vec(self).map_err(|error| CodecError::Malformed(error.to_string())) + } + + fn decode(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(|error| CodecError::Malformed(error.to_string())) + } +} + +impl Task for ConditionTask { + const TYPE: &'static str = "headgate:workflow-condition"; + + fn encode(&self) -> Result, CodecError> { + serde_json::to_vec(self).map_err(|error| CodecError::Malformed(error.to_string())) + } + + fn decode(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(|error| CodecError::Malformed(error.to_string())) + } +} + +impl Task for GraftTask { + const TYPE: &'static str = "headgate:workflow-graft"; + + fn encode(&self) -> Result, CodecError> { + serde_json::to_vec(self).map_err(|error| CodecError::Malformed(error.to_string())) + } + + fn decode(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(|error| CodecError::Malformed(error.to_string())) + } +} + +impl Task for RetryTask { + const TYPE: &'static str = "headgate:workflow-retry"; + + fn encode(&self) -> Result, CodecError> { + serde_json::to_vec(self).map_err(|error| CodecError::Malformed(error.to_string())) + } + + fn decode(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(|error| CodecError::Malformed(error.to_string())) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SignalReceipt { + pub matched: usize, + pub promoted: usize, + pub inserted: bool, + pub emission: WorkflowSignal, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SignalEmission { + pub signal: String, + pub idempotency_key: String, + pub payload: serde_json::Value, + pub source: serde_json::Value, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct WorkflowSignal { + pub id: u64, + pub signal: String, + pub idempotency_key: String, + pub payload: serde_json::Value, + pub source: serde_json::Value, + pub recorded_at_ms: i64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RetryReceipt { + pub revision: u64, + pub generation: u32, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct WorkflowRecovery { + pub node: String, + pub payload: Option>, + pub schema_version: Option, + pub release_quarantine: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +pub struct CancelReceipt { + pub workflows: usize, + pub jobs: usize, +} + +/// Cancel a workflow and, by default, every linked child workflow. Traversal is +/// iterative and bounded by the same node cap as creation; running jobs lose their +/// lease through the ordinary operator-cancel path. +pub async fn cancel_workflow( + inspect: &dyn Inspect, + workflow_id: &str, + propagate_children: bool, +) -> Result { + if workflow_id.is_empty() { + return Err(WorkflowError("workflow id must not be empty".into())); + } + let mut pending = VecDeque::from([workflow_id.to_string()]); + let mut visited = HashSet::new(); + let mut jobs = 0; + while let Some(current) = pending.pop_front() { + if !visited.insert(current.clone()) { + continue; + } + if visited.len() > MAX_WORKFLOW_NODES { + return Err(WorkflowError( + "workflow cancellation exceeds the bounded nested-workflow limit".into(), + )); + } + let coordinator_id = format!("{current}:coordinator"); + let coordinator = inspect + .get_job(&coordinator_id, true) + .await + .map_err(|error| WorkflowError(error.to_string()))? + .ok_or_else(|| WorkflowError(format!("workflow `{current}` was not found")))?; + let payload = coordinator + .payload + .as_deref() + .ok_or_else(|| WorkflowError("workflow coordinator payload was not returned".into()))?; + let task = CoordinatorTask::decode(payload) + .map_err(|error| WorkflowError(format!("invalid workflow coordinator: {error}")))?; + if propagate_children { + pending.extend( + task.nodes + .iter() + .filter_map(|node| node.child_workflow_id.clone()), + ); + } + for job_id in task + .nodes + .iter() + .map(|node| node.job_id.as_str()) + .chain(std::iter::once(coordinator_id.as_str())) + { + let Some(job) = inspect + .get_job(job_id, false) + .await + .map_err(|error| WorkflowError(error.to_string()))? + else { + continue; + }; + if matches!( + job.state.as_str(), + "pending" | "scheduled" | "available" | "running" | "retryable" + ) { + inspect + .operator_cancel(job_id) + .await + .map_err(|error| WorkflowError(error.to_string()))?; + jobs += 1; + } + } + } + Ok(CancelReceipt { + workflows: visited.len(), + jobs, + }) +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct WorkflowEvent { + pub sequence: u64, + pub event: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub node: Option, + pub revision: u64, + pub generation: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub at_ms: Option, +} + +/// The durable role a node plays in a workflow graph. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkflowNodeKind { + Task, + Signal, + Timer, + ChildWorkflow, + Condition, +} + +/// One node in an inspected workflow graph. Dependencies and dependents contain node +/// names, while `job_id` is the underlying Headgate job to inspect or control. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct WorkflowNode { + pub name: String, + pub job_id: String, + pub kind: WorkflowNodeKind, + pub job_kind: String, + pub state: String, + pub dependencies: Vec, + pub dependents: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signal: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub wake_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub delay_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub child_workflow_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub condition: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub completed_at_ms: Option, +} + +/// A bounded point-in-time view of a workflow and its complete accepted graph, +/// including additive grafts accepted in later revisions. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct WorkflowSnapshot { + pub workflow_id: String, + pub coordinator_job_id: String, + pub coordinator_state: String, + pub revision: u64, + pub generation: u32, + pub failed: bool, + pub failed_subgraph_retry: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry_policy: Option, + pub nodes: Vec, +} + +/// One coordinator entry returned by [`list_workflows`]. Fetch its graph with +/// [`inspect_workflow`] only when node-level detail is needed. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct WorkflowSummary { + pub workflow_id: String, + pub coordinator_job_id: String, + pub state: String, + pub enqueued_at_ms: i64, + pub scheduled_at_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finalized_at_ms: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct WorkflowPage { + pub workflows: Vec, + #[serde(default)] + pub next_cursor: Option, +} + +impl WorkflowSnapshot { + pub fn node(&self, name: &str) -> Option<&WorkflowNode> { + self.nodes.iter().find(|node| node.name == name) + } + + pub fn dependencies(&self, name: &str) -> Option> { + let node = self.node(name)?; + Some( + node.dependencies + .iter() + .filter_map(|dependency| self.node(dependency)) + .collect(), + ) + } + + pub fn dependents(&self, name: &str) -> Option> { + let node = self.node(name)?; + Some( + node.dependents + .iter() + .filter_map(|dependent| self.node(dependent)) + .collect(), + ) + } +} + +/// List workflow coordinators without loading every graph. The page is capped at 200; +/// use [`inspect_workflow`] for a selected workflow. +pub async fn list_workflows( + inspect: &dyn Inspect, + cursor: Option<&str>, + limit: u32, +) -> Result { + if !(1..=200).contains(&limit) { + return Err(WorkflowError( + "workflow list limit must be between 1 and 200".into(), + )); + } + let page = inspect + .list_jobs( + &JobFilter { + kind: Some(CoordinatorTask::TYPE.into()), + ..Default::default() + }, + cursor, + limit, + ) + .await + .map_err(|error| WorkflowError(error.to_string()))?; + Ok(WorkflowPage { + workflows: page + .jobs + .into_iter() + .map(|job| WorkflowSummary { + workflow_id: job + .id + .strip_suffix(":coordinator") + .unwrap_or(&job.id) + .to_string(), + coordinator_job_id: job.id, + state: job.state, + enqueued_at_ms: job.enqueued_at_ms, + scheduled_at_ms: job.scheduled_at_ms, + finalized_at_ms: job.finalized_at_ms, + }) + .collect(), + next_cursor: page.next_cursor, + }) +} + +/// Inspect the accepted graph and live execution state without exposing task payloads. +pub async fn inspect_workflow( + inspect: &dyn Inspect, + workflow_id: &str, +) -> Result { + if workflow_id.is_empty() { + return Err(WorkflowError("workflow id must not be empty".into())); + } + let coordinator_job_id = format!("{workflow_id}:coordinator"); + let coordinator = inspect + .get_job(&coordinator_job_id, true) + .await + .map_err(|error| WorkflowError(error.to_string()))? + .ok_or_else(|| WorkflowError(format!("workflow `{workflow_id}` was not found")))?; + let payload = coordinator + .payload + .as_deref() + .ok_or_else(|| WorkflowError("workflow coordinator payload was not returned".into()))?; + let base = CoordinatorTask::decode(payload) + .map_err(|error| WorkflowError(format!("invalid workflow coordinator: {error}")))?; + let cursor = load_workflow_cursor(inspect, &coordinator_job_id, &coordinator.state).await?; + let effective = effective_workflow(&base, &cursor); + let mut dependents: HashMap> = HashMap::new(); + for node in &effective.nodes { + for dependency in &node.deps { + dependents + .entry(dependency.clone()) + .or_default() + .push(node.name.clone()); + } + } + let completed: HashSet<&str> = cursor.completed.iter().map(String::as_str).collect(); + let mut indexed_nodes = stream::iter(effective.nodes.into_iter().enumerate().map( + |(index, node)| { + let recorded_completion = completed.contains(node.name.as_str()); + let completed_at_ms = cursor.completed_at_ms.get(&node.name).copied(); + let node_dependents = dependents.get(&node.name).cloned().unwrap_or_default(); + async move { + let job = inspect + .get_job(&node.job_id, false) + .await + .map_err(|error| WorkflowError(error.to_string()))?; + let (state, job_kind) = job.map_or_else( + || { + ( + if recorded_completion { + "completed" + } else { + "missing" + } + .to_string(), + String::new(), + ) + }, + |job| (job.state, job.kind), + ); + Ok::<_, WorkflowError>(( + index, + WorkflowNode { + name: node.name, + job_id: node.job_id, + kind: match node.kind { + NodeType::Task => WorkflowNodeKind::Task, + NodeType::Signal => WorkflowNodeKind::Signal, + NodeType::Timer => WorkflowNodeKind::Timer, + NodeType::ChildWorkflow => WorkflowNodeKind::ChildWorkflow, + NodeType::Condition => WorkflowNodeKind::Condition, + }, + job_kind, + state, + dependencies: node.deps, + dependents: node_dependents, + signal: node.signal, + wake_at_ms: node.wake_at_ms, + delay_ms: node.delay_ms, + child_workflow_id: node.child_workflow_id, + condition: node.condition, + completed_at_ms, + }, + )) + } + }, + )) + .buffer_unordered(WORKFLOW_CONCURRENCY) + .try_collect::>() + .await?; + indexed_nodes.sort_unstable_by_key(|(index, _)| *index); + let nodes = indexed_nodes.into_iter().map(|(_, node)| node).collect(); + Ok(WorkflowSnapshot { + workflow_id: workflow_id.to_string(), + coordinator_job_id, + coordinator_state: coordinator.state, + revision: cursor.revision, + generation: cursor.generation, + failed: cursor.failed, + failed_subgraph_retry: base.failed_subgraph_retry, + retry_policy: base.retry_policy, + nodes, + }) +} + +pub async fn workflow_node( + inspect: &dyn Inspect, + workflow_id: &str, + node: &str, +) -> Result { + inspect_workflow(inspect, workflow_id) + .await? + .node(node) + .cloned() + .ok_or_else(|| WorkflowError(format!("workflow node `{node}` was not found"))) +} + +pub async fn workflow_dependencies( + inspect: &dyn Inspect, + workflow_id: &str, + node: &str, +) -> Result, WorkflowError> { + let snapshot = inspect_workflow(inspect, workflow_id).await?; + snapshot + .dependencies(node) + .map(|nodes| nodes.into_iter().cloned().collect()) + .ok_or_else(|| WorkflowError(format!("workflow node `{node}` was not found"))) +} + +pub async fn workflow_dependents( + inspect: &dyn Inspect, + workflow_id: &str, + node: &str, +) -> Result, WorkflowError> { + let snapshot = inspect_workflow(inspect, workflow_id).await?; + snapshot + .dependents(node) + .map(|nodes| nodes.into_iter().cloned().collect()) + .ok_or_else(|| WorkflowError(format!("workflow node `{node}` was not found"))) +} + +async fn load_workflow_cursor( + inspect: &dyn Inspect, + coordinator_job_id: &str, + coordinator_state: &str, +) -> Result { + let checkpoint_inspect = inspect.as_checkpoint_inspect().ok_or_else(|| { + WorkflowError("workflow inspection requires checkpoint inspection support".into()) + })?; + let Some(checkpoint) = checkpoint_inspect + .get_job_checkpoint(coordinator_job_id) + .await + .map_err(|error| WorkflowError(error.to_string()))? + else { + return Ok(WorkflowCursor::default()); + }; + if checkpoint + .cursor_step + .as_deref() + .is_some_and(|step| step != "headgate:workflow-state") + { + return Err(WorkflowError( + "workflow coordinator has no workflow-state checkpoint".into(), + )); + } + let bytes = if let Some(cursor) = checkpoint.cursor { + Some(cursor) + } else if let Some(outputs) = inspect.as_output_inspect() { + outputs + .get_job_output(coordinator_job_id) + .await + .map_err(|error| WorkflowError(error.to_string()))? + .map(|output| output.bytes) + } else { + None + }; + if bytes.is_none() + && matches!( + coordinator_state, + "completed" | "archived" | "cancelled" | "quarantined" | "undecodable" + ) + { + return Err(WorkflowError( + "terminal workflow has no durable coordinator output".into(), + )); + } + bytes.map_or_else( + || Ok(WorkflowCursor::default()), + |bytes| { + serde_json::from_slice(&bytes) + .map_err(|error| WorkflowError(format!("invalid workflow cursor: {error}"))) + }, + ) +} + +/// Read the bounded durable event history kept in the fenced coordinator checkpoint. +pub async fn workflow_events( + inspect: &dyn Inspect, + workflow_id: &str, +) -> Result, WorkflowError> { + let checkpoint_inspect = inspect.as_checkpoint_inspect().ok_or_else(|| { + WorkflowError("workflow history requires checkpoint inspection support".into()) + })?; + let checkpoint = checkpoint_inspect + .get_job_checkpoint(&format!("{workflow_id}:coordinator")) + .await + .map_err(|error| WorkflowError(error.to_string()))? + .ok_or_else(|| WorkflowError(format!("workflow `{workflow_id}` was not found")))?; + if checkpoint + .cursor_step + .as_deref() + .is_some_and(|step| step != "headgate:workflow-state") + { + return Err(WorkflowError( + "workflow coordinator has no workflow-state checkpoint".into(), + )); + } + let bytes = if let Some(cursor) = checkpoint.cursor { + cursor + } else { + inspect + .as_output_inspect() + .ok_or_else(|| { + WorkflowError("workflow history requires output inspection support".into()) + })? + .get_job_output(&format!("{workflow_id}:coordinator")) + .await + .map_err(|error| WorkflowError(error.to_string()))? + .ok_or_else(|| WorkflowError("workflow has no durable history".into()))? + .bytes + }; + let cursor = serde_json::from_slice::(&bytes) + .map_err(|error| WorkflowError(format!("invalid workflow cursor: {error}")))?; + Ok(cursor.events) +} + +/// Request retry of only the failed and dependency-blocked portion of a retry-enabled +/// workflow. The receipt enqueue happens before the archived coordinator is reopened, +/// so interruption can be retried without losing the request. +pub async fn request_failed_subgraph_retry( + inspect: &dyn Inspect, + workflow_id: &str, + expected_revision: u64, +) -> Result { + request_failed_subgraph_retry_with_recovery(inspect, workflow_id, expected_revision, &[]).await +} + +pub async fn request_failed_subgraph_retry_with_recovery( + inspect: &dyn Inspect, + workflow_id: &str, + expected_revision: u64, + recoveries: &[WorkflowRecovery], +) -> Result { + if workflow_id.is_empty() || expected_revision == 0 { + return Err(WorkflowError( + "workflow id and expected revision must be set".into(), + )); + } + let coordinator_id = format!("{workflow_id}:coordinator"); + let coordinator = inspect + .get_job(&coordinator_id, true) + .await + .map_err(|error| WorkflowError(error.to_string()))? + .ok_or_else(|| WorkflowError(format!("workflow `{workflow_id}` was not found")))?; + let payload = coordinator + .payload + .ok_or_else(|| WorkflowError("workflow coordinator payload was not returned".into()))?; + let task = CoordinatorTask::decode(&payload) + .map_err(|error| WorkflowError(format!("invalid workflow coordinator: {error}")))?; + if !task.failed_subgraph_retry { + return Err(WorkflowError( + "workflow was not created with failed-subgraph retry enabled".into(), + )); + } + if coordinator.state != "archived" { + return Err(WorkflowError(format!( + "workflow retry requires an archived coordinator, found `{}`", + coordinator.state + ))); + } + let nodes: HashMap<&str, &NodeSpec> = task + .nodes + .iter() + .map(|node| (node.name.as_str(), node)) + .collect(); + let mut recovered = HashSet::new(); + for recovery in recoveries { + if !recovered.insert(recovery.node.as_str()) { + return Err(WorkflowError(format!( + "workflow recovery repeats node `{}`", + recovery.node + ))); + } + let node = nodes.get(recovery.node.as_str()).ok_or_else(|| { + WorkflowError(format!( + "workflow recovery names unknown node `{}`", + recovery.node + )) + })?; + let job = inspect + .get_job(&node.job_id, true) + .await + .map_err(|error| WorkflowError(error.to_string()))? + .ok_or_else(|| WorkflowError(format!("workflow node `{}` is missing", node.job_id)))?; + match job.state.as_str() { + "quarantined" if recovery.release_quarantine => { + inspect + .quarantine_release(&job.fingerprint) + .await + .map_err(|error| WorkflowError(error.to_string()))?; + } + "quarantined" => { + return Err(WorkflowError(format!( + "workflow node `{}` requires explicit quarantine release", + recovery.node + ))); + } + "undecodable" => { + let replacement = recovery.payload.as_deref().ok_or_else(|| { + WorkflowError(format!( + "undecodable workflow node `{}` requires replacement payload", + recovery.node + )) + })?; + let version = recovery.schema_version.ok_or_else(|| { + WorkflowError(format!( + "undecodable workflow node `{}` requires schema_version", + recovery.node + )) + })?; + inspect + .edit_payload( + &node.job_id, + replacement, + version, + &headgate::fingerprint(&job.kind, replacement), + ) + .await + .map_err(|error| WorkflowError(error.to_string()))?; + inspect + .operator_retry(&node.job_id) + .await + .map_err(|error| WorkflowError(error.to_string()))?; + } + "archived" | "cancelled" => {} + // A retry request may be replayed after its recovery mutation completed but + // before the coordinator was reopened. Treat that boundary as idempotent. + "available" => {} + state => { + return Err(WorkflowError(format!( + "workflow node `{}` does not require recovery from `{state}`", + recovery.node + ))); + } + } + } + for node in &task.nodes { + let job = inspect + .get_job(&node.job_id, false) + .await + .map_err(|error| WorkflowError(error.to_string()))?; + if let Some(job) = job + && matches!(job.state.as_str(), "quarantined" | "undecodable") + { + return Err(WorkflowError(format!( + "workflow node `{}` requires recovery from `{}`", + node.name, job.state + ))); + } + } + let checkpoint_inspect = inspect.as_checkpoint_inspect().ok_or_else(|| { + WorkflowError("workflow retry requires checkpoint inspection support".into()) + })?; + let checkpoint = checkpoint_inspect + .get_job_checkpoint(&coordinator_id) + .await + .map_err(|error| WorkflowError(error.to_string()))? + .ok_or_else(|| WorkflowError("workflow coordinator checkpoint is missing".into()))?; + if checkpoint.cursor_step.as_deref() != Some("headgate:workflow-state") { + return Err(WorkflowError( + "workflow coordinator has no workflow-state checkpoint".into(), + )); + } + let bytes = checkpoint + .cursor + .ok_or_else(|| WorkflowError("workflow coordinator cursor is missing".into()))?; + let cursor: WorkflowCursor = serde_json::from_slice(&bytes) + .map_err(|error| WorkflowError(format!("invalid workflow cursor: {error}")))?; + if !cursor.failed || cursor.revision != expected_revision { + return Err(WorkflowError(format!( + "workflow retry revision conflict: expected {expected_revision}, current {}", + cursor.revision + ))); + } + let next_revision = expected_revision + .checked_add(1) + .ok_or_else(|| WorkflowError("workflow retry revision would overflow".into()))?; + let generation = cursor + .generation + .checked_add(1) + .ok_or_else(|| WorkflowError("workflow generation would overflow".into()))?; + let retry = RetryTask { + workflow_id: workflow_id.into(), + expected_revision, + }; + let payload = retry + .encode() + .map_err(|error| WorkflowError(error.to_string()))?; + let receipt = headgate::prepare_envelope(Envelope { + id: retry_receipt_id(workflow_id, next_revision), + kind: RetryTask::TYPE.into(), + schema_version: RetryTask::VERSION, + fingerprint: headgate::fingerprint(RetryTask::TYPE, &payload), + payload, + queue: coordinator.queue, + pending: true, + retention_ms: DEFAULT_RETENTION_MS, + ..Default::default() + }) + .map_err(|error| WorkflowError(error.to_string()))?; + inspect + .enqueue(&[receipt]) + .await + .map_err(|error| WorkflowError(error.to_string()))?; + if let Err(error) = inspect.operator_retry(&coordinator_id).await { + let current = inspect + .get_job(&coordinator_id, false) + .await + .map_err(|read_error| WorkflowError(read_error.to_string()))?; + if !current + .as_ref() + .is_some_and(|job| matches!(job.state.as_str(), "available" | "running")) + { + return Err(WorkflowError(error.to_string())); + } + } + Ok(RetryReceipt { + revision: next_revision, + generation, + }) +} + +/// Durably emit a named signal for an existing workflow. Repeating an emission after +/// its signal jobs become available, running, or completed is an idempotent success. +pub async fn emit_signal( + inspect: &dyn Inspect, + workflow_id: &str, + signal: &str, +) -> Result { + emit_signal_with( + inspect, + workflow_id, + SignalEmission { + signal: signal.into(), + idempotency_key: format!("legacy:{signal}"), + payload: serde_json::Value::Null, + source: serde_json::json!({}), + }, + ) + .await +} + +/// Record payload and emitter metadata before releasing matching signal nodes. Replays +/// return the original record and may safely retry promotion. +pub async fn emit_signal_with( + inspect: &dyn Inspect, + workflow_id: &str, + emission: SignalEmission, +) -> Result { + let signal = emission.signal.as_str(); + if workflow_id.is_empty() || signal.is_empty() { + return Err(WorkflowError( + "workflow id and signal must not be empty".into(), + )); + } + let coordinator_id = format!("{workflow_id}:coordinator"); + let coordinator = inspect + .get_job(&coordinator_id, true) + .await + .map_err(|error| WorkflowError(error.to_string()))? + .ok_or_else(|| WorkflowError(format!("workflow `{workflow_id}` was not found")))?; + let payload = coordinator + .payload + .ok_or_else(|| WorkflowError("workflow coordinator payload was not returned".into()))?; + let task = CoordinatorTask::decode(&payload) + .map_err(|error| WorkflowError(format!("invalid workflow coordinator: {error}")))?; + let jobs: Vec<&str> = task + .nodes + .iter() + .filter(|node| node.kind == NodeType::Signal && node.signal.as_deref() == Some(signal)) + .map(|node| node.job_id.as_str()) + .collect(); + if jobs.is_empty() { + return Err(WorkflowError(format!( + "workflow `{workflow_id}` has no signal `{signal}`" + ))); + } + if emission.idempotency_key.is_empty() { + return Err(WorkflowError( + "signal idempotency key must not be empty".into(), + )); + } + let payload = + serde_json::to_vec(&emission.payload).map_err(|e| WorkflowError(e.to_string()))?; + let source = serde_json::to_vec(&emission.source).map_err(|e| WorkflowError(e.to_string()))?; + if payload.len() > MAX_SIGNAL_PAYLOAD_BYTES { + return Err(WorkflowError( + "signal payload must be at most 65536 bytes".into(), + )); + } + if source.len() > MAX_SIGNAL_SOURCE_BYTES { + return Err(WorkflowError( + "signal source must be at most 16384 bytes".into(), + )); + } + let (stored, inserted) = inspect + .append_durable_event(&DurableEvent { + event_id: 0, + scope: workflow_signal_scope(workflow_id), + topic: signal.into(), + idempotency_key: emission.idempotency_key, + payload, + source, + recorded_at_ms: 0, + }) + .await + .map_err(|e| WorkflowError(e.to_string()))?; + let mut promoted = 0; + for job_id in &jobs { + let job = inspect + .get_job(job_id, false) + .await + .map_err(|error| WorkflowError(error.to_string()))? + .ok_or_else(|| WorkflowError(format!("signal job `{job_id}` was not found")))?; + match job.state.as_str() { + "pending" => match inspect.promote_job(job_id).await { + Ok(()) => promoted += 1, + Err(error) => { + let current = inspect + .get_job(job_id, false) + .await + .map_err(|read_error| WorkflowError(read_error.to_string()))?; + if !current.as_ref().is_some_and(|job| { + matches!(job.state.as_str(), "available" | "running" | "completed") + }) { + return Err(WorkflowError(error.to_string())); + } + } + }, + "available" | "running" | "completed" => {} + state => { + return Err(WorkflowError(format!( + "signal job `{job_id}` cannot be emitted from state `{state}`" + ))); + } + } + } + Ok(SignalReceipt { + matched: jobs.len(), + promoted, + inserted, + emission: workflow_signal(stored)?, + }) +} + +pub async fn list_signals( + inspect: &dyn Inspect, + workflow_id: &str, + before_id: Option, + limit: u32, +) -> Result, WorkflowError> { + if workflow_id.is_empty() { + return Err(WorkflowError("workflow id must not be empty".into())); + } + inspect + .list_durable_events(&workflow_signal_scope(workflow_id), before_id, limit) + .await + .map_err(|e| WorkflowError(e.to_string()))? + .into_iter() + .map(workflow_signal) + .collect() +} + +fn workflow_signal_scope(workflow_id: &str) -> String { + format!("workflow:{workflow_id}:signals") +} + +fn workflow_signal(event: DurableEvent) -> Result { + Ok(WorkflowSignal { + id: event.event_id, + signal: event.topic, + idempotency_key: event.idempotency_key, + payload: serde_json::from_slice(&event.payload) + .map_err(|e| WorkflowError(format!("invalid signal payload: {e}")))?, + source: serde_json::from_slice(&event.source) + .map_err(|e| WorkflowError(format!("invalid signal source: {e}")))?, + recorded_at_ms: event.recorded_at_ms, + }) +} + +#[derive(Serialize, Deserialize)] +struct WorkflowCursor { + #[serde(default = "initial_workflow_revision")] + revision: u64, + #[serde(default)] + completed: Vec, + #[serde(default)] + completed_at_ms: HashMap, + #[serde(default)] + grafts: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pending_graft_receipt: Option, + #[serde(default = "initial_workflow_generation")] + generation: u32, + #[serde(default)] + failed: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pending_retry_receipt: Option, + #[serde(default)] + automatic_retry_pending: bool, + #[serde(default)] + events: Vec, +} + +impl Default for WorkflowCursor { + fn default() -> Self { + Self { + revision: initial_workflow_revision(), + completed: Vec::new(), + completed_at_ms: HashMap::new(), + grafts: Vec::new(), + pending_graft_receipt: None, + generation: initial_workflow_generation(), + failed: false, + pending_retry_receipt: None, + automatic_retry_pending: false, + events: Vec::new(), + } + } +} + +const fn initial_workflow_revision() -> u64 { + 1 +} + +const fn initial_workflow_generation() -> u32 { + 1 +} + +fn graft_receipt_id(workflow_id: &str, revision: u64) -> String { + format!("{workflow_id}:graft:{revision}") +} + +fn retry_receipt_id(workflow_id: &str, revision: u64) -> String { + format!("{workflow_id}:retry:{revision}") +} + +fn record_event( + cursor: &mut WorkflowCursor, + event: &str, + node: Option, + at_ms: Option, +) -> Result<(), WorkflowError> { + let sequence = cursor + .events + .last() + .map_or(1, |entry| entry.sequence.saturating_add(1)); + if sequence == u64::MAX && cursor.events.last().is_some() { + return Err(WorkflowError("workflow event sequence overflow".into())); + } + cursor.events.push(WorkflowEvent { + sequence, + event: event.into(), + node, + revision: cursor.revision, + generation: cursor.generation, + at_ms, + }); + if cursor.events.len() > MAX_WORKFLOW_EVENTS { + let excess = cursor.events.len() - MAX_WORKFLOW_EVENTS; + cursor.events.drain(..excess); + } + Ok(()) } impl Task for CoordinatorTask { @@ -248,44 +1938,521 @@ pub fn register_coordinator( if poll_interval.as_millis() == 0 { return Err("workflow poll interval must be at least 1ms".into()); } + register_virtual_handlers(registry)?; + let child_inspect = inspect.clone(); + registry.register::( + move |_ctx: JobCtx, task: ChildWorkflowTask| { + let inspect = child_inspect.clone(); + async move { + if task.child_workflow_id.is_empty() + || task.child_workflow_id == task.parent_workflow_id + { + return Err::<(), JobError>(Box::new(WorkflowError( + "invalid child workflow link".into(), + ))); + } + let child_id = format!("{}:coordinator", task.child_workflow_id); + let child = inspect.get_job(&child_id, false).await?.ok_or_else(|| { + WorkflowError(format!("child workflow `{child_id}` was not found")) + })?; + match child.state.as_str() { + "completed" => Ok(()), + "archived" | "cancelled" | "quarantined" | "undecodable" => { + Err(Control::Skip.into()) + } + _ => Err(Control::Snooze(poll_interval).into()), + } + } + }, + )?; registry.register::(move |ctx: JobCtx, task: CoordinatorTask| { let inspect = inspect.clone(); async move { let cursor_ctx = ctx.clone(); ctx.step_cursor("headgate:workflow-state", move |cursor| async move { - let cursor = cursor + let mut cursor = cursor .map(|bytes| serde_json::from_slice::(&bytes)) .transpose() .map_err(|error| -> JobError { Box::new(error) })? .unwrap_or_default(); - let mut completed = completed_set(&task, &cursor.completed); - match tick_with_evidence(inspect.as_ref(), &task, &mut completed, Some(&cursor_ctx)) - .await? + if cursor.events.is_empty() { + record_event(&mut cursor, "workflow_started", None, None)?; + persist_workflow_cursor(&cursor_ctx, &cursor).await?; + } + if cursor.automatic_retry_pending { + enqueue_automatic_retry( + inspect.as_ref(), + &task, + &mut cursor, + &cursor_ctx, + cursor_ctx.queue(), + ) + .await?; + } + if let Some(result) = + reconcile_retry(inspect.as_ref(), &task, &mut cursor, &cursor_ctx).await? + { + return match result { + Tick::Waiting => Err(Control::Snooze(poll_interval).into()), + Tick::Succeeded => Ok(()), + Tick::Failed => Err(Control::Skip.into()), + }; + } + if let Some(result) = + reconcile_graft(inspect.as_ref(), &task, &mut cursor, &cursor_ctx).await? + { + return match result { + Tick::Waiting => Err(Control::Snooze(poll_interval).into()), + Tick::Succeeded => Ok(()), + Tick::Failed => Err(Control::Skip.into()), + }; + } + let effective = effective_workflow(&task, &cursor); + match tick_with_evidence( + inspect.as_ref(), + &effective, + &mut cursor, + Some(&cursor_ctx), + ) + .await? { Tick::Waiting => Err(Control::Snooze(poll_interval).into()), - Tick::Succeeded => Ok(()), - Tick::Failed => Err(Control::Skip.into()), + Tick::Succeeded => { + record_event(&mut cursor, "workflow_succeeded", None, None)?; + persist_workflow_cursor(&cursor_ctx, &cursor).await?; + Ok(()) + } + Tick::Failed => { + if task.failed_subgraph_retry { + cursor.failed = true; + if task + .retry_policy + .is_some_and(|policy| cursor.generation < policy.max_generations) + { + cursor.automatic_retry_pending = true; + record_event(&mut cursor, "automatic_retry_scheduled", None, None)?; + } else { + record_event(&mut cursor, "workflow_failed", None, None)?; + } + persist_workflow_cursor(&cursor_ctx, &cursor).await?; + } else { + record_event(&mut cursor, "workflow_failed", None, None)?; + persist_workflow_cursor(&cursor_ctx, &cursor).await?; + } + if cursor.automatic_retry_pending { + let policy = task.retry_policy.expect("automatic retry policy"); + return Err(Control::Snooze(Duration::from_millis( + u64::try_from(policy.backoff_ms) + .map_err(|_| "invalid workflow retry backoff")?, + )) + .into()); + } + Err(Control::Skip.into()) + } } }) .await } - }) + }) +} + +fn register_virtual_handlers(registry: &mut Registry) -> Result<(), String> { + registry.register::(|_ctx: JobCtx, _task: SignalTask| async move { + Ok::<(), JobError>(()) + })?; + registry.register::(|_ctx: JobCtx, _task: TimerTask| async move { + Ok::<(), JobError>(()) + })?; + registry.register::(|_ctx: JobCtx, _task: ConditionTask| async move { + Ok::<(), JobError>(()) + })?; + registry.register::(|_ctx: JobCtx, _task: GraftTask| async move { + Ok::<(), JobError>(()) + })?; + registry.register::(|_ctx: JobCtx, _task: RetryTask| async move { + Ok::<(), JobError>(()) + })?; + Ok(()) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Tick { + Waiting, + Succeeded, + Failed, +} + +fn effective_workflow(base: &CoordinatorTask, cursor: &WorkflowCursor) -> CoordinatorTask { + let mut nodes = Vec::with_capacity(base.nodes.len() + cursor.grafts.len()); + nodes.extend(base.nodes.iter().cloned()); + nodes.extend(cursor.grafts.iter().cloned()); + CoordinatorTask { + workflow_id: base.workflow_id.clone(), + nodes, + failed_subgraph_retry: base.failed_subgraph_retry, + retry_policy: base.retry_policy, + } +} + +async fn enqueue_automatic_retry( + inspect: &dyn Inspect, + base: &CoordinatorTask, + cursor: &mut WorkflowCursor, + ctx: &JobCtx, + queue: &str, +) -> Result<(), JobError> { + if !cursor.failed { + return Err(Box::new(WorkflowError( + "automatic retry is pending for a non-failed workflow".into(), + ))); + } + let next_revision = cursor + .revision + .checked_add(1) + .ok_or_else(|| WorkflowError("workflow retry revision would overflow".into()))?; + let retry = RetryTask { + workflow_id: base.workflow_id.clone(), + expected_revision: cursor.revision, + }; + let payload = retry.encode()?; + let receipt = headgate::prepare_envelope(Envelope { + id: retry_receipt_id(&base.workflow_id, next_revision), + kind: RetryTask::TYPE.into(), + schema_version: RetryTask::VERSION, + fingerprint: headgate::fingerprint(RetryTask::TYPE, &payload), + payload, + queue: queue.into(), + pending: true, + retention_ms: DEFAULT_RETENTION_MS, + ..Default::default() + })?; + inspect.enqueue(&[receipt]).await?; + cursor.automatic_retry_pending = false; + persist_workflow_cursor(ctx, cursor).await +} + +async fn persist_workflow_cursor(ctx: &JobCtx, cursor: &WorkflowCursor) -> Result<(), JobError> { + let bytes = serde_json::to_vec(cursor).map_err(|error| -> JobError { Box::new(error) })?; + ctx.set_cursor(bytes.clone()).await?; + ctx.persist_output(1, bytes).await?; + Ok(()) +} + +async fn reject_graft( + inspect: &dyn Inspect, + receipt_id: &str, + nodes: &[NodeSpec], +) -> Result<(), JobError> { + for job_id in nodes + .iter() + .map(|node| node.job_id.as_str()) + .chain(std::iter::once(receipt_id)) + { + let Some(job) = inspect.get_job(job_id, false).await? else { + continue; + }; + if matches!( + job.state.as_str(), + "pending" | "scheduled" | "available" | "retryable" + ) { + inspect.delete_job(job_id).await?; + } else { + return Err(Box::new(WorkflowError(format!( + "rejected workflow graft job `{job_id}` is already `{}`", + job.state + )))); + } + } + Ok(()) +} + +async fn failed_nodes_to_retry( + inspect: &dyn Inspect, + workflow: &CoordinatorTask, +) -> Result, JobError> { + let mut retry = Vec::new(); + for node in &workflow.nodes { + let job = inspect.get_job(&node.job_id, false).await?.ok_or_else(|| { + WorkflowError(format!( + "retry-enabled workflow node `{}` is missing", + node.job_id + )) + })?; + match job.state.as_str() { + "archived" | "cancelled" => retry.push(node.job_id.clone()), + "pending" | "scheduled" | "retryable" | "available" | "running" | "completed" => {} + state => { + return Err(Box::new(WorkflowError(format!( + "workflow node `{}` cannot be retried from `{state}`", + node.job_id + )))); + } + } + } + Ok(retry) +} + +async fn retry_failed_children( + inspect: &dyn Inspect, + workflow: &CoordinatorTask, +) -> Result<(), JobError> { + let checkpoint_inspect = inspect.as_checkpoint_inspect().ok_or_else(|| { + WorkflowError("child retry propagation requires checkpoint inspection support".into()) + })?; + for node in workflow + .nodes + .iter() + .filter(|node| node.kind == NodeType::ChildWorkflow) + { + let Some(child_workflow_id) = node.child_workflow_id.as_deref() else { + continue; + }; + let Some(link) = inspect.get_job(&node.job_id, false).await? else { + continue; + }; + if !matches!(link.state.as_str(), "archived" | "cancelled") { + continue; + } + let child_id = format!("{child_workflow_id}:coordinator"); + let Some(child) = inspect.get_job(&child_id, false).await? else { + return Err(Box::new(WorkflowError(format!( + "child workflow `{child_workflow_id}` is missing" + )))); + }; + if child.state != "archived" { + continue; + } + let checkpoint = checkpoint_inspect + .get_job_checkpoint(&child_id) + .await? + .ok_or_else(|| { + WorkflowError(format!( + "child workflow `{child_workflow_id}` has no checkpoint" + )) + })?; + let cursor: WorkflowCursor = serde_json::from_slice( + checkpoint + .cursor + .as_deref() + .ok_or_else(|| WorkflowError("child workflow cursor is missing".into()))?, + )?; + request_failed_subgraph_retry(inspect, child_workflow_id, cursor.revision).await?; + } + Ok(()) +} + +async fn reopen_failed_nodes(inspect: &dyn Inspect, jobs: &[String]) -> Result<(), JobError> { + for job_id in jobs { + inspect.operator_retry(job_id).await?; + } + Ok(()) +} + +async fn reconcile_retry( + inspect: &dyn Inspect, + base: &CoordinatorTask, + cursor: &mut WorkflowCursor, + ctx: &JobCtx, +) -> Result, JobError> { + if let Some(receipt_id) = cursor.pending_retry_receipt.clone() { + let receipt = inspect.get_job(&receipt_id, false).await?.ok_or_else(|| { + WorkflowError(format!( + "accepted workflow retry receipt `{receipt_id}` is missing" + )) + })?; + let workflow = effective_workflow(base, cursor); + match receipt.state.as_str() { + "pending" => { + let jobs = failed_nodes_to_retry(inspect, &workflow).await?; + reopen_failed_nodes(inspect, &jobs).await?; + inspect.promote_job(&receipt_id).await?; + return Ok(Some(Tick::Waiting)); + } + "available" | "running" => return Ok(Some(Tick::Waiting)), + "completed" => { + cursor.pending_retry_receipt = None; + persist_workflow_cursor(ctx, cursor).await?; + } + state => { + return Err(Box::new(WorkflowError(format!( + "accepted workflow retry receipt `{receipt_id}` entered `{state}`" + )))); + } + } + } + + let next_revision = cursor + .revision + .checked_add(1) + .ok_or_else(|| WorkflowError("workflow revision would overflow".into()))?; + let receipt_id = retry_receipt_id(&base.workflow_id, next_revision); + let Some(receipt) = inspect.get_job(&receipt_id, true).await? else { + return Ok(None); + }; + if receipt.state != "pending" { + return Err(Box::new(WorkflowError(format!( + "unaccepted workflow retry receipt `{receipt_id}` entered `{}`", + receipt.state + )))); + } + let payload = receipt.payload.ok_or_else(|| { + WorkflowError(format!( + "workflow retry receipt `{receipt_id}` did not return its payload" + )) + })?; + let retry = match RetryTask::decode(&payload) { + Ok(retry) => retry, + Err(_) => { + reject_graft(inspect, &receipt_id, &[]).await?; + return Ok(Some(Tick::Failed)); + } + }; + if !base.failed_subgraph_retry + || !cursor.failed + || retry.workflow_id != base.workflow_id + || retry.expected_revision != cursor.revision + { + reject_graft(inspect, &receipt_id, &[]).await?; + return Ok(Some(if cursor.failed { + Tick::Failed + } else { + Tick::Waiting + })); + } + let competing_graft_id = graft_receipt_id(&base.workflow_id, next_revision); + if let Some(competing) = inspect.get_job(&competing_graft_id, true).await? { + if competing.state != "pending" { + return Err(Box::new(WorkflowError(format!( + "competing workflow graft receipt `{competing_graft_id}` entered `{}`", + competing.state + )))); + } + let nodes = competing + .payload + .as_deref() + .and_then(|payload| GraftTask::decode(payload).ok()) + .map(|graft| graft.nodes) + .unwrap_or_default(); + reject_graft(inspect, &competing_graft_id, &nodes).await?; + } + let workflow = effective_workflow(base, cursor); + retry_failed_children(inspect, &workflow).await?; + let jobs = match failed_nodes_to_retry(inspect, &workflow).await { + Ok(jobs) => jobs, + Err(_) => { + reject_graft(inspect, &receipt_id, &[]).await?; + return Ok(Some(Tick::Failed)); + } + }; + cursor.generation = cursor + .generation + .checked_add(1) + .ok_or_else(|| WorkflowError("workflow generation would overflow".into()))?; + cursor.revision = next_revision; + cursor.failed = false; + record_event(cursor, "workflow_retry_accepted", None, None)?; + cursor.pending_retry_receipt = Some(receipt_id.clone()); + persist_workflow_cursor(ctx, cursor).await?; + reopen_failed_nodes(inspect, &jobs).await?; + inspect.promote_job(&receipt_id).await?; + Ok(Some(Tick::Waiting)) } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum Tick { - Waiting, - Succeeded, - Failed, +/// Reconcile at most one revision per tick. The cursor is persisted before the receipt +/// is promoted, so a crash can only replay an accepted revision, never lose it. +async fn reconcile_graft( + inspect: &dyn Inspect, + base: &CoordinatorTask, + cursor: &mut WorkflowCursor, + ctx: &JobCtx, +) -> Result, JobError> { + if cursor.revision == 0 { + return Err(Box::new(WorkflowError( + "workflow cursor contains revision zero".into(), + ))); + } + if let Some(receipt_id) = cursor.pending_graft_receipt.clone() { + let receipt = inspect.get_job(&receipt_id, false).await?.ok_or_else(|| { + WorkflowError(format!( + "accepted workflow graft receipt `{receipt_id}` is missing" + )) + })?; + match receipt.state.as_str() { + "pending" => { + inspect.promote_job(&receipt_id).await?; + return Ok(Some(Tick::Waiting)); + } + "available" | "running" => return Ok(Some(Tick::Waiting)), + "completed" => { + cursor.pending_graft_receipt = None; + persist_workflow_cursor(ctx, cursor).await?; + } + state => { + return Err(Box::new(WorkflowError(format!( + "accepted workflow graft receipt `{receipt_id}` entered `{state}`" + )))); + } + } + } + + let next_revision = cursor + .revision + .checked_add(1) + .ok_or_else(|| WorkflowError("workflow revision would overflow".into()))?; + let receipt_id = graft_receipt_id(&base.workflow_id, next_revision); + let Some(receipt) = inspect.get_job(&receipt_id, true).await? else { + return Ok(None); + }; + if receipt.state != "pending" { + return Err(Box::new(WorkflowError(format!( + "unaccepted workflow graft receipt `{receipt_id}` entered `{}`", + receipt.state + )))); + } + let payload = receipt.payload.ok_or_else(|| { + WorkflowError(format!( + "workflow graft receipt `{receipt_id}` did not return its payload" + )) + })?; + let graft = match GraftTask::decode(&payload) { + Ok(graft) => graft, + Err(_) => { + reject_graft(inspect, &receipt_id, &[]).await?; + return Ok(Some(Tick::Waiting)); + } + }; + if graft.workflow_id != base.workflow_id + || graft.expected_revision != cursor.revision + || graft.nodes.is_empty() + || cursor.failed + { + reject_graft(inspect, &receipt_id, &graft.nodes).await?; + return Ok(Some(Tick::Waiting)); + } + let mut candidate = effective_workflow(base, cursor); + candidate.nodes.extend(graft.nodes.iter().cloned()); + if validate_coordinator(&candidate).is_err() { + reject_graft(inspect, &receipt_id, &graft.nodes).await?; + return Ok(Some(Tick::Waiting)); + } + + cursor.revision = next_revision; + cursor.grafts.extend(graft.nodes); + record_event(cursor, "workflow_graft_accepted", None, None)?; + cursor.pending_graft_receipt = Some(receipt_id.clone()); + persist_workflow_cursor(ctx, cursor).await?; + inspect.promote_job(&receipt_id).await?; + Ok(Some(Tick::Waiting)) } async fn tick_with_evidence( inspect: &dyn Inspect, workflow: &CoordinatorTask, - completed: &mut HashSet, + cursor: &mut WorkflowCursor, persist_ctx: Option<&JobCtx>, ) -> Result { validate_coordinator(workflow).map_err(|error| -> JobError { Box::new(error) })?; + let mut completed = completed_set(workflow, &cursor.completed); let reads: Vec<(String, String)> = workflow .nodes .iter() @@ -299,58 +2466,76 @@ async fn tick_with_evidence( .try_collect() .await?; let state: HashMap> = entries.into_iter().collect(); - let mut changed = false; + let before = completed.clone(); + let changed = record_completion_evidence( + workflow, + &state, + &mut completed, + &mut cursor.completed_at_ms, + ); for node in &workflow.nodes { - if state - .get(node.name.as_str()) - .and_then(Option::as_ref) - .is_some_and(|job| job.state == "completed") - && completed.insert(node.name.clone()) - { - changed = true; + if !before.contains(node.name.as_str()) && completed.contains(node.name.as_str()) { + record_event( + cursor, + "node_completed", + Some(node.name.clone()), + cursor.completed_at_ms.get(&node.name).copied(), + )?; } } if changed && let Some(ctx) = persist_ctx { - let cursor = WorkflowCursor { - completed: completed_names(workflow, completed), - }; - let bytes = serde_json::to_vec(&cursor).map_err(|error| -> JobError { Box::new(error) })?; - ctx.set_cursor(bytes).await?; + cursor.completed = completed_names(workflow, &completed); + persist_workflow_cursor(ctx, cursor).await?; } + let failed_nodes = failed_set(workflow, &state, &completed); let mut mutations = Vec::new(); for node in &workflow.nodes { - if effective_state( + let current = effective_state( + node, state.get(node.name.as_str()).and_then(Option::as_ref), - node.name.as_str(), - completed, - ) != Some("pending") - { - continue; - } - let dep_failed = node.deps.iter().any(|dep| { - matches!( - effective_state( - state.get(dep.as_str()).and_then(Option::as_ref), - dep, - completed, - ), - None | Some("archived" | "cancelled" | "quarantined" | "undecodable") - ) - }); + &completed, + ); + let dep_failed = failed_nodes.contains(node.name.as_str()); if dep_failed { - mutations.push((node.job_id.clone(), true)); + if !workflow.failed_subgraph_retry + && matches!( + current, + Some("pending" | "scheduled" | "available" | "retryable") + ) + { + mutations.push((node.job_id.clone(), true)); + } continue; } - let deps_complete = node.deps.iter().all(|dep| { - effective_state( - state.get(dep.as_str()).and_then(Option::as_ref), - dep, - completed, - ) == Some("completed") - }); - if deps_complete { + if matches!(node.kind, NodeType::Task | NodeType::ChildWorkflow) + && current == Some("pending") + && dependencies_complete(workflow, node, &state, &completed) + { mutations.push((node.job_id.clone(), false)); } + if node.kind == NodeType::Condition + && current == Some("pending") + && dependencies_complete(workflow, node, &state, &completed) + && evaluate_condition(node, cursor, workflow, &state, &completed)? + { + mutations.push((node.job_id.clone(), false)); + } + if node.kind == NodeType::Timer + && node.delay_ms.is_some() + && current == Some("pending") + && dependencies_complete(workflow, node, &state, &completed) + { + let anchor = dependency_completion_anchor(node, &cursor.completed_at_ms)?; + let wake_at_ms = anchor + .checked_add(node.delay_ms.unwrap_or_default()) + .ok_or_else(|| { + WorkflowError(format!("workflow timer `{}` deadline overflow", node.name)) + })?; + inspect + .schedule_pending_job(&node.job_id, wake_at_ms) + .await?; + return Ok(Tick::Waiting); + } } if !mutations.is_empty() { stream::iter(mutations) @@ -368,10 +2553,14 @@ async fn tick_with_evidence( } let mut failed = false; for node in &workflow.nodes { + if failed_nodes.contains(node.name.as_str()) { + failed = true; + continue; + } match effective_state( + node, state.get(node.name.as_str()).and_then(Option::as_ref), - node.name.as_str(), - completed, + &completed, ) { Some("completed") => {} None | Some("archived" | "cancelled" | "quarantined" | "undecodable") => failed = true, @@ -385,6 +2574,213 @@ async fn tick_with_evidence( }) } +fn record_completion_evidence( + workflow: &CoordinatorTask, + state: &HashMap>, + completed: &mut HashSet, + completed_at_ms: &mut HashMap, +) -> bool { + let mut changed = false; + for node in &workflow.nodes { + if matches!(node.kind, NodeType::Task | NodeType::ChildWorkflow) + && let Some(job) = state + .get(node.name.as_str()) + .and_then(Option::as_ref) + .filter(|job| job.state == "completed") + { + changed |= completed.insert(node.name.clone()); + if let Some(at_ms) = job.finalized_at_ms { + changed |= completed_at_ms.insert(node.name.clone(), at_ms) != Some(at_ms); + } + } + } + loop { + let eligible: Vec = workflow + .nodes + .iter() + .filter(|node| { + matches!( + node.kind, + NodeType::Signal | NodeType::Timer | NodeType::Condition + ) + }) + .filter(|node| !completed.contains(node.name.as_str())) + .filter(|node| node.deps.iter().all(|dep| completed.contains(dep))) + .filter(|node| { + state + .get(node.name.as_str()) + .and_then(Option::as_ref) + .is_some_and(|job| job.state == "completed") + }) + .map(|node| node.name.clone()) + .collect(); + if eligible.is_empty() { + break; + } + changed = true; + for name in &eligible { + if let Some(at_ms) = state + .get(name.as_str()) + .and_then(Option::as_ref) + .and_then(|job| job.finalized_at_ms) + { + completed_at_ms.insert(name.clone(), at_ms); + } + } + completed.extend(eligible); + } + changed +} + +fn dependency_completion_anchor( + node: &NodeSpec, + completed_at_ms: &HashMap, +) -> Result { + let mut anchor = None; + for dependency in &node.deps { + let completed_at = completed_at_ms.get(dependency).copied().ok_or_else(|| { + Box::new(WorkflowError(format!( + "workflow timer `{}` has no durable completion timestamp for `{dependency}`", + node.name + ))) as JobError + })?; + anchor = Some(anchor.map_or(completed_at, |current: i64| current.max(completed_at))); + } + anchor.ok_or_else(|| { + Box::new(WorkflowError(format!( + "relative workflow timer `{}` requires at least one dependency", + node.name + ))) as JobError + }) +} + +fn evaluate_condition( + node: &NodeSpec, + cursor: &WorkflowCursor, + workflow: &CoordinatorTask, + state: &HashMap>, + completed: &HashSet, +) -> Result { + let expression = node.condition.as_deref().ok_or_else(|| { + Box::new(WorkflowError(format!( + "workflow condition `{}` has no expression", + node.name + ))) as JobError + })?; + let program = cel::Program::compile(expression).map_err(|error| { + Box::new(WorkflowError(format!( + "invalid workflow CEL condition `{}`: {error}", + node.name + ))) as JobError + })?; + let states: HashMap = workflow + .nodes + .iter() + .map(|candidate| { + let state = effective_state( + candidate, + state.get(candidate.name.as_str()).and_then(Option::as_ref), + completed, + ) + .unwrap_or("missing") + .to_string(); + (candidate.name.clone(), state) + }) + .collect(); + let completion: HashMap = workflow + .nodes + .iter() + .map(|candidate| { + ( + candidate.name.clone(), + completed.contains(candidate.name.as_str()), + ) + }) + .collect(); + let mut context = cel::Context::default(); + context.add_variable_from_value("revision", cursor.revision); + context.add_variable_from_value("generation", u64::from(cursor.generation)); + context.add_variable_from_value("states", states); + context.add_variable_from_value("completed", completion); + match program.execute(&context).map_err(|error| { + Box::new(WorkflowError(format!( + "workflow CEL condition `{}` failed: {error}", + node.name + ))) as JobError + })? { + cel::Value::Bool(value) => Ok(value), + _ => Err(Box::new(WorkflowError(format!( + "workflow CEL condition `{}` must return bool", + node.name + )))), + } +} + +fn dependencies_complete( + workflow: &CoordinatorTask, + node: &NodeSpec, + state: &HashMap>, + completed: &HashSet, +) -> bool { + node.deps.iter().all(|dep| { + let upstream = workflow + .nodes + .iter() + .find(|node| node.name == *dep) + .expect("validated dependency"); + effective_state( + upstream, + state.get(dep.as_str()).and_then(Option::as_ref), + completed, + ) == Some("completed") + }) +} + +#[cfg(test)] +fn dependency_failed( + workflow: &CoordinatorTask, + node: &NodeSpec, + state: &HashMap>, + completed: &HashSet, +) -> bool { + node.deps + .iter() + .any(|dep| failed_set(workflow, state, completed).contains(dep)) +} + +fn failed_set( + workflow: &CoordinatorTask, + state: &HashMap>, + completed: &HashSet, +) -> HashSet { + let mut failed: HashSet = workflow + .nodes + .iter() + .filter(|node| { + matches!( + effective_state( + node, + state.get(node.name.as_str()).and_then(Option::as_ref), + completed, + ), + None | Some("archived" | "cancelled" | "quarantined" | "undecodable") + ) + }) + .map(|node| node.name.clone()) + .collect(); + loop { + let before = failed.len(); + for node in &workflow.nodes { + if node.deps.iter().any(|dep| failed.contains(dep)) { + failed.insert(node.name.clone()); + } + } + if failed.len() == before { + return failed; + } + } +} + fn completed_set(workflow: &CoordinatorTask, names: &[String]) -> HashSet { let valid: HashSet<&str> = workflow .nodes @@ -408,12 +2804,21 @@ fn completed_names(workflow: &CoordinatorTask, completed: &HashSet) -> V } fn effective_state<'a>( + node: &NodeSpec, job: Option<&'a headgate_core::JobSummary>, - name: &str, completed: &HashSet, ) -> Option<&'a str> { - job.map(|job| job.state.as_str()) - .or_else(|| completed.contains(name).then_some("completed")) + if completed.contains(node.name.as_str()) { + return Some("completed"); + } + match (node.kind, job.map(|job| job.state.as_str())) { + // An early signal is durable evidence, but it is not consumed until all of the + // signal node's dependencies have completed. + (NodeType::Signal | NodeType::Timer | NodeType::Condition, Some("completed")) => { + Some("pending") + } + (_, state) => state, + } } fn validate_coordinator(workflow: &CoordinatorTask) -> Result<(), WorkflowError> { @@ -427,6 +2832,13 @@ fn validate_coordinator(workflow: &CoordinatorTask) -> Result<(), WorkflowError> "workflow coordinator must contain 1-{MAX_WORKFLOW_NODES} tasks" ))); } + if workflow.retry_policy.is_some_and(|policy| { + policy.max_generations < 2 || policy.backoff_ms <= 0 || !workflow.failed_subgraph_retry + }) { + return Err(WorkflowError( + "workflow coordinator contains an invalid retry policy".into(), + )); + } let mut names = HashSet::with_capacity(workflow.nodes.len()); let mut edges = 0usize; for node in &workflow.nodes { @@ -435,6 +2847,25 @@ fn validate_coordinator(workflow: &CoordinatorTask) -> Result<(), WorkflowError> || node.job_id.is_empty() || node.job_id.len() > MAX_JOB_IDENTIFIER_LEN || !names.insert(node.name.as_str()) + || (node.kind == NodeType::Signal && node.signal.as_deref().is_none_or(str::is_empty)) + || (node.kind == NodeType::Timer + && match (node.wake_at_ms, node.delay_ms) { + (Some(at), None) => at <= 0, + (None, Some(delay)) => delay <= 0 || node.deps.is_empty(), + _ => true, + }) + || (node.kind != NodeType::Signal && node.signal.is_some()) + || (node.kind != NodeType::Timer + && (node.wake_at_ms.is_some() || node.delay_ms.is_some())) + || (node.kind == NodeType::ChildWorkflow + && node.child_workflow_id.as_deref().is_none_or(str::is_empty)) + || (node.kind != NodeType::ChildWorkflow && node.child_workflow_id.is_some()) + || (node.kind == NodeType::Condition + && node + .condition + .as_deref() + .is_none_or(|condition| validate_condition(condition).is_err())) + || (node.kind != NodeType::Condition && node.condition.is_some()) { return Err(WorkflowError( "workflow coordinator contains an invalid task".into(), @@ -447,14 +2878,47 @@ fn validate_coordinator(workflow: &CoordinatorTask) -> Result<(), WorkflowError> "workflow coordinator must contain at most {MAX_WORKFLOW_EDGES} dependency edges" ))); } - if workflow + let mut indegree: HashMap<&str, usize> = workflow .nodes .iter() - .flat_map(|node| &node.deps) - .any(|dep| !names.contains(dep.as_str())) - { + .map(|node| (node.name.as_str(), 0)) + .collect(); + let mut outgoing: HashMap<&str, Vec<&str>> = HashMap::new(); + for node in &workflow.nodes { + let mut unique = HashSet::new(); + for dep in &node.deps { + if !names.contains(dep.as_str()) { + return Err(WorkflowError( + "workflow coordinator contains a missing dependency".into(), + )); + } + if !unique.insert(dep.as_str()) { + return Err(WorkflowError( + "workflow coordinator repeats a dependency".into(), + )); + } + *indegree.get_mut(node.name.as_str()).expect("known node") += 1; + outgoing.entry(dep).or_default().push(&node.name); + } + } + let mut ready: VecDeque<&str> = indegree + .iter() + .filter_map(|(name, degree)| (*degree == 0).then_some(*name)) + .collect(); + let mut visited = 0; + while let Some(name) = ready.pop_front() { + visited += 1; + for child in outgoing.get(name).into_iter().flatten() { + let degree = indegree.get_mut(child).expect("known child"); + *degree -= 1; + if *degree == 0 { + ready.push_back(child); + } + } + } + if visited != workflow.nodes.len() { return Err(WorkflowError( - "workflow coordinator contains a missing dependency".into(), + "workflow coordinator dependency graph contains a cycle".into(), )); } Ok(()) @@ -464,6 +2928,81 @@ fn validate_coordinator(workflow: &CoordinatorTask) -> Result<(), WorkflowError> mod tests { use super::*; + #[derive(Clone)] + struct GraftStep(String); + + impl Task for GraftStep { + const TYPE: &'static str = "workflow:test-graft-step"; + + fn encode(&self) -> Result, CodecError> { + Ok(self.0.as_bytes().to_vec()) + } + + fn decode(bytes: &[u8]) -> Result { + Ok(Self(String::from_utf8_lossy(bytes).into_owned())) + } + } + + fn graft_env(value: &str) -> Envelope { + let task = GraftStep(value.into()); + Envelope { + kind: GraftStep::TYPE.into(), + payload: task.encode().unwrap(), + queue: "headgate-workflow".into(), + ..Default::default() + } + } + + #[test] + fn workflow_snapshot_answers_topology_queries() { + let snapshot = WorkflowSnapshot { + workflow_id: "wf".into(), + coordinator_job_id: "wf:coordinator".into(), + coordinator_state: "running".into(), + revision: 2, + generation: 1, + failed: false, + failed_subgraph_retry: true, + retry_policy: None, + nodes: vec![ + WorkflowNode { + name: "prepare".into(), + job_id: "wf:prepare".into(), + kind: WorkflowNodeKind::Task, + job_kind: "task:prepare".into(), + state: "completed".into(), + dependencies: vec![], + dependents: vec!["publish".into()], + signal: None, + wake_at_ms: None, + delay_ms: None, + child_workflow_id: None, + condition: None, + completed_at_ms: Some(42), + }, + WorkflowNode { + name: "publish".into(), + job_id: "wf:publish".into(), + kind: WorkflowNodeKind::Task, + job_kind: "task:publish".into(), + state: "pending".into(), + dependencies: vec!["prepare".into()], + dependents: vec![], + signal: None, + wake_at_ms: None, + delay_ms: None, + child_workflow_id: None, + condition: None, + completed_at_ms: None, + }, + ], + }; + assert_eq!(snapshot.node("prepare").unwrap().state, "completed"); + assert_eq!(snapshot.dependencies("publish").unwrap()[0].name, "prepare"); + assert_eq!(snapshot.dependents("prepare").unwrap()[0].name, "publish"); + assert!(snapshot.dependencies("missing").is_none()); + } + fn env(kind: &str) -> Envelope { Envelope { kind: kind.into(), @@ -473,6 +3012,35 @@ mod tests { } } + fn summary(id: &str, state: &str) -> headgate_core::JobSummary { + headgate_core::JobSummary { + id: id.into(), + kind: "test".into(), + queue: "default".into(), + state: state.into(), + schema_version: 1, + priority: 0, + attempt: 0, + crash_attempt: 0, + max_attempts: 1, + partition_key: String::new(), + rate_class: String::new(), + sticky_worker: String::new(), + weight: 1, + fingerprint: "fp".into(), + enqueued_at_ms: 0, + scheduled_at_ms: 0, + claimed_at_ms: None, + periodic_schedule_id: String::new(), + periodic_tick_ms: 0, + finalized_at_ms: None, + payload: None, + headers: Default::default(), + errors_json: "[]".into(), + tags: Vec::new(), + } + } + #[test] fn prepare_builds_one_coordinator_and_pending_fan_out_fan_in() { let batch = Workflow::new("wf1") @@ -503,6 +3071,127 @@ mod tests { assert_eq!(batch[1].retention_ms, 3 * 60 * 60 * 1000); } + #[test] + fn signal_is_pending_work_and_early_completion_waits_for_dependencies() { + let batch = Workflow::new("wf-signals") + .add("prepare", env("task:prepare"), Vec::::new()) + .add_signal("approval", "approved", ["prepare"]) + .add("publish", env("task:publish"), ["approval"]) + .prepare() + .unwrap(); + assert_eq!(batch[2].kind, SignalTask::TYPE); + assert!(batch[2].pending); + let task = CoordinatorTask::decode(&batch[0].payload).unwrap(); + assert_eq!(task.nodes[1].kind, NodeType::Signal); + assert_eq!(task.nodes[1].signal.as_deref(), Some("approved")); + + let mut state = HashMap::from([ + ("prepare".into(), Some(summary("prepare", "available"))), + ("approval".into(), Some(summary("approval", "completed"))), + ("publish".into(), Some(summary("publish", "pending"))), + ]); + let mut completed = HashSet::new(); + let mut completed_at_ms = HashMap::new(); + assert!(!record_completion_evidence( + &task, + &state, + &mut completed, + &mut completed_at_ms + )); + assert!(!completed.contains("approval")); + + state.insert("prepare".into(), Some(summary("prepare", "completed"))); + assert!(record_completion_evidence( + &task, + &state, + &mut completed, + &mut completed_at_ms + )); + assert!(completed.contains("prepare")); + assert!(completed.contains("approval")); + } + + #[test] + fn timer_uses_absolute_schedule_and_buffers_until_dependencies_complete() { + let batch = Workflow::new("wf-timer") + .add("prepare", env("task:prepare"), Vec::::new()) + .add_timer_at("release", 1_500, ["prepare"]) + .add("publish", env("task:publish"), ["release"]) + .prepare() + .unwrap(); + assert_eq!(batch[2].kind, TimerTask::TYPE); + assert!(!batch[2].pending); + assert_eq!(batch[2].scheduled_at_ms, 1_500); + let task = CoordinatorTask::decode(&batch[0].payload).unwrap(); + assert_eq!(task.nodes[1].kind, NodeType::Timer); + assert_eq!(task.nodes[1].wake_at_ms, Some(1_500)); + + let mut state = HashMap::from([ + ("prepare".into(), Some(summary("prepare", "available"))), + ("release".into(), Some(summary("release", "completed"))), + ("publish".into(), Some(summary("publish", "pending"))), + ]); + let mut completed = HashSet::new(); + let mut completed_at_ms = HashMap::new(); + assert!(!record_completion_evidence( + &task, + &state, + &mut completed, + &mut completed_at_ms + )); + assert!(!completed.contains("release")); + + state.insert("prepare".into(), Some(summary("prepare", "completed"))); + assert!(record_completion_evidence( + &task, + &state, + &mut completed, + &mut completed_at_ms + )); + assert!(completed.contains("release")); + + completed.clear(); + state.insert("prepare".into(), Some(summary("prepare", "archived"))); + assert!(dependency_failed(&task, &task.nodes[1], &state, &completed)); + } + + #[test] + fn relative_timer_anchors_to_dependency_completion() { + let workflow = Workflow::new("wf-relative") + .add("prepare", env("task:prepare"), Vec::::new()) + .add_timer_after("wait", Duration::from_millis(250), ["prepare"]) + .unwrap(); + let batch = workflow.prepare().unwrap(); + let task = CoordinatorTask::decode(&batch[0].payload).unwrap(); + let timer = &task.nodes[1]; + assert_eq!( + dependency_completion_anchor(timer, &HashMap::from([("prepare".into(), 1_000)])) + .unwrap() + + timer.delay_ms.unwrap(), + 1_250 + ); + } + + #[test] + fn child_workflow_is_an_explicit_pending_node() { + let batch = Workflow::new("parent") + .add_child("billing", "billing-child", Vec::::new()) + .add("finish", env("task:finish"), ["billing"]) + .prepare() + .unwrap(); + assert_eq!(batch[1].kind, ChildWorkflowTask::TYPE); + assert!(batch[1].pending); + let child = ChildWorkflowTask::decode(&batch[1].payload).unwrap(); + assert_eq!(child.parent_workflow_id, "parent"); + assert_eq!(child.child_workflow_id, "billing-child"); + let coordinator = CoordinatorTask::decode(&batch[0].payload).unwrap(); + assert_eq!(coordinator.nodes[0].kind, NodeType::ChildWorkflow); + assert_eq!( + coordinator.nodes[0].child_workflow_id.as_deref(), + Some("billing-child") + ); + } + #[test] fn retained_completion_survives_a_missing_job_row() { let task = CoordinatorTask { @@ -511,14 +3200,20 @@ mod tests { name: "prepare".into(), job_id: "prepare".into(), deps: Vec::new(), + kind: NodeType::Task, + signal: None, + wake_at_ms: None, + delay_ms: None, + child_workflow_id: None, + condition: None, }], + failed_subgraph_retry: false, + retry_policy: None, }; let completed = completed_set(&task, &["prepare".into()]); - assert_eq!( - effective_state(None, "prepare", &completed), - Some("completed") - ); - assert_eq!(effective_state(None, "unknown", &completed), None); + let node = &task.nodes[0]; + assert_eq!(effective_state(node, None, &completed), Some("completed")); + assert_eq!(effective_state(node, None, &HashSet::new()), None); } #[test] @@ -536,6 +3231,116 @@ mod tests { assert!(cycle.to_string().contains("cycle")); } + #[test] + fn revisioned_graft_prepares_one_atomic_receipt_and_pending_tasks() { + let graft = WorkflowGraft::new("wf-graft", 1) + .add("after", graft_env("after"), ["root"]) + .prepare() + .unwrap(); + assert_eq!(graft[0].id, "wf-graft:graft:2"); + assert_eq!(graft[1].id, "wf-graft:g2:after"); + assert!(graft.iter().all(|job| job.pending)); + let receipt = GraftTask::decode(&graft[0].payload).unwrap(); + assert_eq!(receipt.workflow_id, "wf-graft"); + assert_eq!(receipt.expected_revision, 1); + assert_eq!(receipt.nodes[0].deps, ["root"]); + + let cycle = WorkflowGraft::new("wf-graft", 1) + .add("a", graft_env("a"), ["b"]) + .add("b", graft_env("b"), ["a"]) + .prepare() + .unwrap_err(); + assert!(cycle.to_string().contains("cycle")); + } + + #[test] + fn failed_subgraph_retry_is_explicit_in_the_coordinator_payload() { + let batch = Workflow::new("wf-retry") + .failed_subgraph_retry() + .add("prepare", graft_env("prepare"), Vec::::new()) + .add("finish", graft_env("finish"), ["prepare"]) + .prepare() + .unwrap(); + let coordinator = CoordinatorTask::decode(&batch[0].payload).unwrap(); + assert!(coordinator.failed_subgraph_retry); + assert!(batch[1..].iter().all(|job| job.pending)); + } + + #[test] + fn automatic_retry_policy_and_cel_condition_are_validated() { + let batch = Workflow::new("wf-auto") + .automatic_retry(3, Duration::from_millis(25)) + .unwrap() + .add("prepare", env("task:prepare"), Vec::::new()) + .add_condition( + "ready", + "completed.prepare && states.prepare == 'completed' && generation == 1u", + ["prepare"], + ) + .prepare() + .unwrap(); + let coordinator = CoordinatorTask::decode(&batch[0].payload).unwrap(); + assert_eq!( + coordinator.retry_policy, + Some(WorkflowRetryPolicy { + max_generations: 3, + backoff_ms: 25, + }) + ); + let mut cursor = WorkflowCursor::default(); + cursor.completed.push("prepare".into()); + let completed = completed_set(&coordinator, &cursor.completed); + let states = HashMap::from([( + "prepare".into(), + Some(summary("wf-auto:prepare", "completed")), + )]); + assert!( + evaluate_condition( + &coordinator.nodes[1], + &cursor, + &coordinator, + &states, + &completed, + ) + .unwrap() + ); + + assert!( + Workflow::new("bad-cel") + .add_condition("ready", "completed[", Vec::::new()) + .prepare() + .is_err() + ); + } + + #[test] + fn atomic_bundle_rejects_cross_workflow_cycles() { + let parent = Workflow::new("parent").add_child("child", "child", Vec::::new()); + let child = Workflow::new("child").add("work", env("task:child"), Vec::::new()); + let batch = prepare_bundle(vec![parent, child]).unwrap(); + assert_eq!(batch.len(), 4); + + let left = Workflow::new("left").add_child("right", "right", Vec::::new()); + let right = Workflow::new("right").add_child("left", "left", Vec::::new()); + assert!( + prepare_bundle(vec![left, right]) + .unwrap_err() + .to_string() + .contains("cycle") + ); + } + + #[test] + fn workflow_history_is_bounded_and_monotonic() { + let mut cursor = WorkflowCursor::default(); + for index in 0..(MAX_WORKFLOW_EVENTS + 7) { + record_event(&mut cursor, "tick", Some(index.to_string()), None).unwrap(); + } + assert_eq!(cursor.events.len(), MAX_WORKFLOW_EVENTS); + assert_eq!(cursor.events.first().unwrap().sequence, 8); + assert_eq!(cursor.events.last().unwrap().sequence, 263); + } + #[test] fn workflow_and_coordinator_resource_bounds_are_enforced() { let mut workflow = Workflow::new("too-large"); @@ -561,8 +3366,16 @@ mod tests { name: format!("node-{index}"), job_id: format!("job-{index}"), deps: Vec::new(), + kind: NodeType::Task, + signal: None, + wake_at_ms: None, + delay_ms: None, + child_workflow_id: None, + condition: None, }) .collect(), + failed_subgraph_retry: false, + retry_policy: None, }; assert!(validate_coordinator(&forged).is_err()); } diff --git a/crates/headgate-workflow/tests/live.rs b/crates/headgate-workflow/tests/live.rs index 6f1d6f2..8953896 100644 --- a/crates/headgate-workflow/tests/live.rs +++ b/crates/headgate-workflow/tests/live.rs @@ -1,12 +1,19 @@ use std::{ - sync::{Arc, Mutex}, + sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }, time::Duration, }; use headgate::{CodecError, Envelope, JobCtx, Registry, Store, Task, WorkerConfig, testing}; use headgate_core::Inspect; use headgate_postgres::PgStore; -use headgate_workflow::{Workflow, register_coordinator}; +use headgate_redis::{RedisStore, RedisStoreOptions}; +use headgate_workflow::{ + SignalEmission, Workflow, WorkflowGraft, emit_signal, emit_signal_with, list_signals, + register_coordinator, request_failed_subgraph_retry, workflow_events, +}; #[derive(Clone)] struct Step(String); @@ -31,6 +38,157 @@ fn envelope(queue: &str, step: &str) -> Envelope { } } +async fn run_experimental_matrix_cell(store: Arc, backend: &str) +where + S: Store + Inspect + Send + Sync + 'static, +{ + let suffix = format!( + "{}-{}-{}", + std::process::id(), + backend, + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + let workflow_id = format!("workflow-matrix-{suffix}"); + let queue = format!("workflow-matrix-{suffix}"); + let mut unstable = envelope(&queue, "unstable"); + unstable.max_attempts = 1; + let batch = Workflow::new(&workflow_id) + .coordinator_queue(&queue) + .automatic_retry(2, Duration::from_millis(2)) + .unwrap() + .add("prepare", envelope(&queue, "prepare"), Vec::::new()) + .add("unstable", unstable, ["prepare"]) + .add_condition( + "ready", + "completed.unstable && states.unstable == 'completed'", + ["unstable"], + ) + .add_timer_after("pause", Duration::from_millis(2), ["ready"]) + .unwrap() + .add_signal("approval", "approved", ["pause"]) + .add("finish", envelope(&queue, "finish"), ["approval"]) + .prepare() + .unwrap(); + store.enqueue(&batch).await.unwrap(); + let signal = SignalEmission { + signal: "approved".into(), + idempotency_key: format!("matrix-approval:{workflow_id}"), + payload: serde_json::json!({"approved": true, "backend": backend}), + source: serde_json::json!({"emitter": "workflow-matrix"}), + }; + let receipt = emit_signal_with(store.as_ref(), &workflow_id, signal.clone()) + .await + .unwrap(); + assert_eq!(receipt.matched, 1); + assert!(receipt.inserted); + let replay = emit_signal_with(store.as_ref(), &workflow_id, signal) + .await + .unwrap(); + assert!(!replay.inserted); + assert_eq!(replay.emission, receipt.emission); + let signals = list_signals(store.as_ref(), &workflow_id, None, 100) + .await + .unwrap(); + assert_eq!(signals, [receipt.emission]); + + let failures = Arc::new(AtomicUsize::new(1)); + let order = Arc::new(Mutex::new(Vec::new())); + let mut registry = Registry::new(); + register_coordinator( + &mut registry, + store.clone() as Arc, + Duration::from_millis(2), + ) + .unwrap(); + let seen = order.clone(); + let remaining = failures.clone(); + registry + .register::(move |_ctx: JobCtx, step: Step| { + let seen = seen.clone(); + let remaining = remaining.clone(); + async move { + seen.lock().unwrap().push(step.0.clone()); + if step.0 == "unstable" && remaining.fetch_sub(1, Ordering::SeqCst) == 1 { + return Err::<(), headgate::JobError>("planned workflow failure".into()); + } + Ok(()) + } + }) + .unwrap(); + let registry = Arc::new(registry); + let cfg = WorkerConfig { + queues: vec![queue], + ..Default::default() + }; + for _ in 0..100 { + let _ = testing::drain(&store, ®istry, &cfg, 32).await; + if store + .get_job(&format!("{workflow_id}:coordinator"), false) + .await + .unwrap() + .is_some_and(|job| job.state == "completed") + { + break; + } + tokio::time::sleep(Duration::from_millis(3)).await; + } + assert_eq!( + store + .get_job(&format!("{workflow_id}:coordinator"), false) + .await + .unwrap() + .unwrap() + .state, + "completed" + ); + assert_eq!( + order.lock().unwrap().as_slice(), + &["prepare", "unstable", "unstable", "finish"] + ); + let events = workflow_events(store.as_ref(), &workflow_id).await.unwrap(); + assert!( + events + .iter() + .any(|event| event.event == "automatic_retry_scheduled") + ); + assert!( + events + .iter() + .any(|event| event.event == "workflow_succeeded") + ); +} + +#[tokio::test] +async fn workflow_experiments_postgres_matrix_cell() { + let Ok(conninfo) = std::env::var("HG_TEST_PG") else { + eprintln!("HG_TEST_PG not set; skipping Rust/Postgres workflow matrix cell"); + return; + }; + run_experimental_matrix_cell(Arc::new(PgStore::connect(&conninfo, 4).unwrap()), "pg").await; +} + +#[tokio::test] +async fn workflow_experiments_redis_matrix_cell() { + let Ok(url) = std::env::var("HG_TEST_REDIS") else { + eprintln!("HG_TEST_REDIS not set; skipping Rust/Redis workflow matrix cell"); + return; + }; + let client = redis::Client::open(url).unwrap(); + let conn = client.get_connection_manager().await.unwrap(); + let prefix = format!( + "workflow-matrix-rust-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + let store = RedisStore::with_options(conn, prefix, RedisStoreOptions::default()); + run_experimental_matrix_cell(Arc::new(store), "redis").await; +} + #[tokio::test] async fn live_postgres_dag_promotes_fan_out_then_fan_in() { let Ok(conninfo) = std::env::var("HG_TEST_PG") else { @@ -102,3 +260,354 @@ async fn live_postgres_dag_promotes_fan_out_then_fan_in() { assert!(order[1..3].contains(&"left".to_string())); assert!(order[1..3].contains(&"right".to_string())); } + +#[tokio::test] +async fn live_postgres_buffers_and_idempotently_replays_workflow_signal() { + let Ok(conninfo) = std::env::var("HG_TEST_PG") else { + eprintln!("HG_TEST_PG not set; skipping workflow signal proof"); + return; + }; + let store = Arc::new(PgStore::connect(&conninfo, 4).expect("connect")); + let suffix = std::process::id(); + let workflow_id = format!("workflow-signal-live-{suffix}"); + let queue = format!("workflow-signal-live-{suffix}"); + let batch = Workflow::new(&workflow_id) + .coordinator_queue(&queue) + .add("prepare", envelope(&queue, "prepare"), Vec::::new()) + .add_signal("approval", "approved", ["prepare"]) + .add("publish", envelope(&queue, "publish"), ["approval"]) + .prepare() + .unwrap(); + store.enqueue(&batch).await.unwrap(); + + let first = emit_signal(store.as_ref(), &workflow_id, "approved") + .await + .unwrap(); + assert_eq!(first.matched, 1); + assert_eq!(first.promoted, 1); + + let order = Arc::new(Mutex::new(Vec::new())); + let mut registry = Registry::new(); + register_coordinator( + &mut registry, + store.clone() as Arc, + Duration::from_millis(2), + ) + .unwrap(); + let seen = order.clone(); + registry + .register::(move |_ctx: JobCtx, step: Step| { + let seen = seen.clone(); + async move { + seen.lock().unwrap().push(step.0); + Ok(()) + } + }) + .unwrap(); + let registry = Arc::new(registry); + let cfg = WorkerConfig { + queues: vec![queue], + ..Default::default() + }; + + for _ in 0..30 { + let _ = testing::drain(&store, ®istry, &cfg, 16).await; + let state = store + .get_job(&format!("{workflow_id}:coordinator"), false) + .await + .unwrap() + .unwrap() + .state; + if state == "completed" { + break; + } + tokio::time::sleep(Duration::from_millis(3)).await; + } + assert_eq!( + order.lock().unwrap().as_slice(), + &["prepare".to_string(), "publish".to_string()] + ); + let repeated = emit_signal(store.as_ref(), &workflow_id, "approved") + .await + .unwrap(); + assert_eq!(repeated.matched, 1); + assert_eq!(repeated.promoted, 0); +} + +#[tokio::test] +async fn live_postgres_parent_waits_for_child_workflow() { + let Ok(conninfo) = std::env::var("HG_TEST_PG") else { + eprintln!("HG_TEST_PG not set; skipping nested workflow proof"); + return; + }; + let store = Arc::new(PgStore::connect(&conninfo, 4).expect("connect")); + let suffix = std::process::id(); + let child_id = format!("workflow-child-live-{suffix}"); + let parent_id = format!("workflow-parent-live-{suffix}"); + let queue = format!("workflow-nested-live-{suffix}"); + let child = Workflow::new(&child_id) + .coordinator_queue(&queue) + .add( + "child-work", + envelope(&queue, "child-work"), + Vec::::new(), + ) + .prepare() + .unwrap(); + let parent = Workflow::new(&parent_id) + .coordinator_queue(&queue) + .add_child("child", &child_id, Vec::::new()) + .add( + "parent-finish", + envelope(&queue, "parent-finish"), + ["child"], + ) + .prepare() + .unwrap(); + store.enqueue(&child).await.unwrap(); + store.enqueue(&parent).await.unwrap(); + + let order = Arc::new(Mutex::new(Vec::new())); + let mut registry = Registry::new(); + register_coordinator( + &mut registry, + store.clone() as Arc, + Duration::from_millis(2), + ) + .unwrap(); + let seen = order.clone(); + registry + .register::(move |_ctx: JobCtx, step: Step| { + let seen = seen.clone(); + async move { + seen.lock().unwrap().push(step.0); + Ok(()) + } + }) + .unwrap(); + let registry = Arc::new(registry); + let cfg = WorkerConfig { + queues: vec![queue], + ..Default::default() + }; + for _ in 0..40 { + let _ = testing::drain(&store, ®istry, &cfg, 16).await; + let state = store + .get_job(&format!("{parent_id}:coordinator"), false) + .await + .unwrap() + .unwrap() + .state; + if state == "completed" { + break; + } + tokio::time::sleep(Duration::from_millis(3)).await; + } + assert_eq!( + order.lock().unwrap().as_slice(), + &["child-work".to_string(), "parent-finish".to_string()] + ); +} + +#[tokio::test] +async fn live_postgres_accepts_one_revisioned_workflow_graft_atomically() { + let Ok(conninfo) = std::env::var("HG_TEST_PG") else { + eprintln!("HG_TEST_PG not set; skipping workflow graft proof"); + return; + }; + let store = Arc::new(PgStore::connect(&conninfo, 4).expect("connect")); + let suffix = std::process::id(); + let workflow_id = format!("workflow-graft-live-{suffix}"); + let queue = format!("workflow-graft-live-{suffix}"); + let base = Workflow::new(&workflow_id) + .coordinator_queue(&queue) + .add("root", envelope(&queue, "root"), Vec::::new()) + .prepare() + .unwrap(); + let graft = WorkflowGraft::new(&workflow_id, 1) + .queue(&queue) + .add("after", envelope(&queue, "after"), ["root"]) + .prepare() + .unwrap(); + store.enqueue(&base).await.unwrap(); + store.enqueue(&graft).await.unwrap(); + + let conflict = WorkflowGraft::new(&workflow_id, 1) + .queue(&queue) + .add("loser", envelope(&queue, "loser"), ["root"]) + .prepare() + .unwrap(); + assert!(store.enqueue(&conflict).await.is_err()); + assert!( + store + .get_job(&format!("{workflow_id}:g2:loser"), false) + .await + .unwrap() + .is_none() + ); + + let order = Arc::new(Mutex::new(Vec::new())); + let mut registry = Registry::new(); + register_coordinator( + &mut registry, + store.clone() as Arc, + Duration::from_millis(2), + ) + .unwrap(); + let seen = order.clone(); + registry + .register::(move |_ctx: JobCtx, step: Step| { + let seen = seen.clone(); + async move { + seen.lock().unwrap().push(step.0); + Ok(()) + } + }) + .unwrap(); + let registry = Arc::new(registry); + let cfg = WorkerConfig { + queues: vec![queue], + ..Default::default() + }; + for _ in 0..40 { + let _ = testing::drain(&store, ®istry, &cfg, 16).await; + let state = store + .get_job(&format!("{workflow_id}:coordinator"), false) + .await + .unwrap() + .unwrap() + .state; + if state == "completed" { + break; + } + tokio::time::sleep(Duration::from_millis(3)).await; + } + assert_eq!( + order.lock().unwrap().as_slice(), + &["root".to_string(), "after".to_string()] + ); + assert_eq!( + store + .get_job(&format!("{workflow_id}:graft:2"), false) + .await + .unwrap() + .unwrap() + .state, + "completed" + ); +} + +#[tokio::test] +async fn live_postgres_retries_only_the_failed_workflow_subgraph() { + let Ok(conninfo) = std::env::var("HG_TEST_PG") else { + eprintln!("HG_TEST_PG not set; skipping workflow retry proof"); + return; + }; + let store = Arc::new(PgStore::connect(&conninfo, 4).expect("connect")); + let suffix = std::process::id(); + let workflow_id = format!("workflow-retry-live-{suffix}"); + let queue = format!("workflow-retry-live-{suffix}"); + let mut unstable = envelope(&queue, "unstable"); + unstable.max_attempts = 1; + let batch = Workflow::new(&workflow_id) + .coordinator_queue(&queue) + .failed_subgraph_retry() + .add("prepare", envelope(&queue, "prepare"), Vec::::new()) + .add("unstable", unstable, ["prepare"]) + .add("finish", envelope(&queue, "finish"), ["unstable"]) + .prepare() + .unwrap(); + store.enqueue(&batch).await.unwrap(); + + let order = Arc::new(Mutex::new(Vec::new())); + let unstable_attempts = Arc::new(AtomicUsize::new(0)); + let mut registry = Registry::new(); + register_coordinator( + &mut registry, + store.clone() as Arc, + Duration::from_millis(2), + ) + .unwrap(); + let seen = order.clone(); + let attempts = unstable_attempts.clone(); + registry + .register::(move |_ctx: JobCtx, step: Step| { + let seen = seen.clone(); + let attempts = attempts.clone(); + async move { + seen.lock().unwrap().push(step.0.clone()); + if step.0 == "unstable" && attempts.fetch_add(1, Ordering::SeqCst) == 0 { + return Err::<(), headgate::JobError>( + std::io::Error::other("planned workflow failure").into(), + ); + } + Ok(()) + } + }) + .unwrap(); + let registry = Arc::new(registry); + let cfg = WorkerConfig { + queues: vec![queue], + ..Default::default() + }; + + for _ in 0..40 { + let _ = testing::drain(&store, ®istry, &cfg, 16).await; + let state = store + .get_job(&format!("{workflow_id}:coordinator"), false) + .await + .unwrap() + .unwrap() + .state; + if state == "archived" { + break; + } + tokio::time::sleep(Duration::from_millis(3)).await; + } + assert_eq!( + store + .get_job(&format!("{workflow_id}:finish"), false) + .await + .unwrap() + .unwrap() + .state, + "pending" + ); + let receipt = request_failed_subgraph_retry(store.as_ref(), &workflow_id, 1) + .await + .unwrap(); + assert_eq!(receipt.revision, 2); + assert_eq!(receipt.generation, 2); + + for _ in 0..60 { + let _ = testing::drain(&store, ®istry, &cfg, 16).await; + let state = store + .get_job(&format!("{workflow_id}:coordinator"), false) + .await + .unwrap() + .unwrap() + .state; + if state == "completed" { + break; + } + tokio::time::sleep(Duration::from_millis(3)).await; + } + assert_eq!( + order.lock().unwrap().as_slice(), + &[ + "prepare".to_string(), + "unstable".to_string(), + "unstable".to_string(), + "finish".to_string(), + ] + ); + assert_eq!( + store + .get_job(&format!("{workflow_id}:retry:2"), false) + .await + .unwrap() + .unwrap() + .state, + "completed" + ); +} diff --git a/crates/headgate-workflow/tests/live_mysql.rs b/crates/headgate-workflow/tests/live_mysql.rs new file mode 100644 index 0000000..ff58d13 --- /dev/null +++ b/crates/headgate-workflow/tests/live_mysql.rs @@ -0,0 +1,136 @@ +use std::{ + sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use headgate::{CodecError, Envelope, JobCtx, Registry, Store, Task, WorkerConfig, testing}; +use headgate_core::Inspect; +use headgate_mysql::MysqlStore; +use headgate_workflow::{Workflow, emit_signal, register_coordinator, workflow_events}; + +#[derive(Clone)] +struct MatrixStep(String); + +impl Task for MatrixStep { + const TYPE: &'static str = "workflow:mysql-matrix-step"; + + fn encode(&self) -> Result, CodecError> { + Ok(self.0.as_bytes().to_vec()) + } + + fn decode(bytes: &[u8]) -> Result { + Ok(Self(String::from_utf8_lossy(bytes).into_owned())) + } +} + +fn envelope(queue: &str, step: &str) -> Envelope { + let task = MatrixStep(step.into()); + Envelope { + kind: MatrixStep::TYPE.into(), + payload: task.encode().unwrap(), + queue: queue.into(), + ..Default::default() + } +} + +#[tokio::test] +async fn workflow_experiments_mysql_matrix_cell() { + let Ok(url) = std::env::var("HG_TEST_MYSQL") else { + eprintln!("HG_TEST_MYSQL not set; skipping Rust/MySQL workflow matrix cell"); + return; + }; + let store = Arc::new(MysqlStore::connect(&url).unwrap()); + let suffix = format!( + "{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + let workflow_id = format!("workflow-matrix-mysql-{suffix}"); + let queue = format!("workflow-matrix-mysql-{suffix}"); + let mut unstable = envelope(&queue, "unstable"); + unstable.max_attempts = 1; + let batch = Workflow::new(&workflow_id) + .coordinator_queue(&queue) + .automatic_retry(2, Duration::from_millis(2)) + .unwrap() + .add("prepare", envelope(&queue, "prepare"), Vec::::new()) + .add("unstable", unstable, ["prepare"]) + .add_condition( + "ready", + "completed.unstable && states.unstable == 'completed'", + ["unstable"], + ) + .add_timer_after("pause", Duration::from_millis(2), ["ready"]) + .unwrap() + .add_signal("approval", "approved", ["pause"]) + .add("finish", envelope(&queue, "finish"), ["approval"]) + .prepare() + .unwrap(); + store.enqueue(&batch).await.unwrap(); + emit_signal(store.as_ref(), &workflow_id, "approved") + .await + .unwrap(); + + let remaining = Arc::new(AtomicUsize::new(1)); + let order = Arc::new(Mutex::new(Vec::new())); + let mut registry = Registry::new(); + register_coordinator( + &mut registry, + store.clone() as Arc, + Duration::from_millis(2), + ) + .unwrap(); + let seen = order.clone(); + let failures = remaining.clone(); + registry + .register::(move |_ctx: JobCtx, step: MatrixStep| { + let seen = seen.clone(); + let failures = failures.clone(); + async move { + seen.lock().unwrap().push(step.0.clone()); + if step.0 == "unstable" && failures.fetch_sub(1, Ordering::SeqCst) == 1 { + return Err::<(), headgate::JobError>("planned workflow failure".into()); + } + Ok(()) + } + }) + .unwrap(); + let registry = Arc::new(registry); + let cfg = WorkerConfig { + queues: vec![queue], + ..Default::default() + }; + for _ in 0..100 { + let _ = testing::drain(&store, ®istry, &cfg, 32).await; + if store + .get_job(&format!("{workflow_id}:coordinator"), false) + .await + .unwrap() + .is_some_and(|job| job.state == "completed") + { + break; + } + tokio::time::sleep(Duration::from_millis(3)).await; + } + assert_eq!( + order.lock().unwrap().as_slice(), + &["prepare", "unstable", "unstable", "finish"] + ); + let events = workflow_events(store.as_ref(), &workflow_id).await.unwrap(); + assert!( + events + .iter() + .any(|event| event.event == "automatic_retry_scheduled") + ); + assert!( + events + .iter() + .any(|event| event.event == "workflow_succeeded") + ); +} diff --git a/crates/headgate/src/isolated.rs b/crates/headgate/src/isolated.rs index 2e70f7a..c15c9ef 100644 --- a/crates/headgate/src/isolated.rs +++ b/crates/headgate/src/isolated.rs @@ -181,10 +181,10 @@ async fn execute_request( read_bounded(stderr, max), child.wait(), ); - if let Err(error) = write_result { - if error.kind() != std::io::ErrorKind::BrokenPipe { - return Err(Box::new(error)); - } + if let Err(error) = write_result + && error.kind() != std::io::ErrorKind::BrokenPipe + { + return Err(Box::new(error)); } let (stdout, stdout_overflow) = stdout_result?; let (stderr, stderr_overflow) = stderr_result?; diff --git a/crates/headgate/src/tracked.rs b/crates/headgate/src/tracked.rs index 8360e9b..75396bc 100644 --- a/crates/headgate/src/tracked.rs +++ b/crates/headgate/src/tracked.rs @@ -93,12 +93,12 @@ impl TaskTracker { format!("tracked task was cancelled unexpectedly: {error}").into(), )), }; - if first.is_none() { - if let Some(failure) = failure { - first = Some(failure); - aborting = true; - tasks.abort_all(); - } + if first.is_none() + && let Some(failure) = failure + { + first = Some(failure); + aborting = true; + tasks.abort_all(); } } match first { @@ -137,10 +137,10 @@ impl TaskTracker { impl Drop for TaskTracker { fn drop(&mut self) { - if let Ok(state) = self.state.get_mut() { - if let Some(tasks) = state.tasks.as_mut() { - tasks.abort_all(); - } + if let Ok(state) = self.state.get_mut() + && let Some(tasks) = state.tasks.as_mut() + { + tasks.abort_all(); } } } diff --git a/crates/headgate/src/worker.rs b/crates/headgate/src/worker.rs index 9b00d2a..0f33423 100644 --- a/crates/headgate/src/worker.rs +++ b/crates/headgate/src/worker.rs @@ -152,12 +152,12 @@ impl Worker { // typed dispatch startup validation: warn on kinds waiting in the store that no // registered handler (or alias) answers — before they fail one at a time. - if let Some(insp) = self.store.as_inspect() { - if let Ok(kinds) = insp.distinct_kinds(1_000).await { - for kind in kinds { - if self.registry.get(&kind).is_none() { - tracing::warn!(%kind, "jobs of this kind are waiting but no handler is registered"); - } + if let Some(insp) = self.store.as_inspect() + && let Ok(kinds) = insp.distinct_kinds(1_000).await + { + for kind in kinds { + if self.registry.get(&kind).is_none() { + tracing::warn!(%kind, "jobs of this kind are waiting but no handler is registered"); } } } @@ -486,10 +486,10 @@ fn finish_task( // Aborted (lease lost / shutdown) or panicked with catch_panics=false. In // both cases the reclaimer owns the job's fate — an uncaught panic IS a // crash and is counted as one. - if let Some(i) = inflight.remove(&join_err.id()) { - if join_err.is_panic() { - tracing::error!(job = %i.job_id, "handler panicked (catch_panics=false); job left to the reclaimer as a crash"); - } + if let Some(i) = inflight.remove(&join_err.id()) + && join_err.is_panic() + { + tracing::error!(job = %i.job_id, "handler panicked (catch_panics=false); job left to the reclaimer as a crash"); } } } @@ -502,6 +502,7 @@ fn finish_task( /// carries. it used to return `()`, which is why the "execute a worker" testing /// row had nothing behind it: a helper that runs one job but cannot say what happened to it /// is `drain` with extra steps. +#[allow(clippy::too_many_arguments)] // The shared execution boundary mirrors independently optional runtime policies. pub(crate) async fn process_one( store: Arc, registry: Arc, @@ -1571,8 +1572,7 @@ mod the_runtime_loop_without_a_database { let cap = Arc::new(Capture::default()); let tel: Arc = cap.clone(); run_duty(&store, "retention", &tel, &[]).await; - let v = cap.0.lock().unwrap().clone(); - v + cap.0.lock().unwrap().clone() } // ----------------------------------------------------------------------- diff --git a/crates/headgate/tests/runtime.rs b/crates/headgate/tests/runtime.rs index 9f9bbc2..9c012f9 100644 --- a/crates/headgate/tests/runtime.rs +++ b/crates/headgate/tests/runtime.rs @@ -1174,7 +1174,8 @@ async fn trace_context_and_the_autoscaling_signal_reach_the_facade() { const TP: &str = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; // What the handler SAW, so the ctx accessor is asserted from inside a real dispatch. - let seen: Arc)>>> = Default::default(); + type SeenTraceContexts = Arc)>>>; + let seen: SeenTraceContexts = Default::default(); let release = Arc::new(tokio::sync::Notify::new()); let mut reg = Registry::new(); diff --git a/examples/go/go.mod b/examples/go/go.mod index 772559e..423ed9f 100644 --- a/examples/go/go.mod +++ b/examples/go/go.mod @@ -11,7 +11,10 @@ require ( ) require ( + cel.dev/cel-go v0.32.0 // indirect + cel.dev/expr v0.25.1 // indirect filippo.io/edwards25519 v1.2.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/go-sql-driver/mysql v1.9.3 // indirect @@ -20,7 +23,12 @@ require ( github.com/jackc/pgx/v5 v5.9.2 // indirect github.com/mujhtech/headgate/go/headgatemigrate v0.1.7 // indirect github.com/redis/go-redis/v9 v9.7.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect golang.org/x/text v0.39.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect + google.golang.org/protobuf v1.36.12 // indirect ) replace github.com/mujhtech/headgate/go => ../../go diff --git a/examples/go/go.sum b/examples/go/go.sum index b43e5b1..d2ded8e 100644 --- a/examples/go/go.sum +++ b/examples/go/go.sum @@ -1,5 +1,11 @@ +cel.dev/cel-go v0.32.0 h1:irvpFKr5EuGPyxeME03ERh0rii1TX+BDAnB9eL3IvNk= +cel.dev/cel-go v0.32.0/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -13,6 +19,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -21,20 +29,44 @@ github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mujhtech/headgate/go/driver/headgatemysql v0.1.7 h1:hae/xf5o6uhkQYiWOPP2/ipsMD23da1IepWqy9RFNsw= +github.com/mujhtech/headgate/go/driver/headgatemysql v0.1.7/go.mod h1:FV8xIHDnuzKBppuBpGR1ieLh0gAZq1jiXUTg+Si8jwc= +github.com/mujhtech/headgate/go/driver/headgatepgx v0.1.7 h1:Ba5yywQX338FOgod53qpA5ATcPS4SR6Fa4hks+j0a28= +github.com/mujhtech/headgate/go/driver/headgatepgx v0.1.7/go.mod h1:RHpyo2uWKS/HCbZtnT7V4YmSWbE+oBmcXQSZ20tHjjw= +github.com/mujhtech/headgate/go/driver/headgateredis v0.1.7 h1:Y75BERPC+bNceM7qRB0irCNuiIBX5oAWsMBBBfhuChE= +github.com/mujhtech/headgate/go/driver/headgateredis v0.1.7/go.mod h1:pUOfSEhWxCsluKaCMe5ArLkZThzTZzPbKYWIZ+CKmB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.work b/go.work new file mode 100644 index 0000000..96968a2 --- /dev/null +++ b/go.work @@ -0,0 +1,16 @@ +go 1.25.0 + +use ( + ./go + ./go/driver/headgatemysql + ./go/driver/headgatepgx + ./go/driver/headgateredis + ./go/headgateapi + ./go/headgatecrypto + ./go/headgatectl + ./go/headgatemigrate + ./go/headgateotel + ./go/headgatetest + ./go/headgateui + ./go/headgateworkflow +) diff --git a/go/go.work.sum b/go.work.sum similarity index 88% rename from go/go.work.sum rename to go.work.sum index db9b3e8..6f87f59 100644 --- a/go/go.work.sum +++ b/go.work.sum @@ -1,6 +1,5 @@ filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= @@ -12,18 +11,12 @@ github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QII github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= @@ -72,35 +65,39 @@ go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu6 go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959/go.mod h1:LV7u5Oco+Z/g6XI7PqN+EUUUGGkEcmB1uj2ceI0fOVg= +golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= @@ -108,7 +105,10 @@ golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0 golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/go/driver/headgatemysql/go.mod b/go/driver/headgatemysql/go.mod index ff5012f..36c6e25 100644 --- a/go/driver/headgatemysql/go.mod +++ b/go/driver/headgatemysql/go.mod @@ -17,7 +17,7 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.9.2 // indirect github.com/redis/go-redis/v9 v9.7.3 // indirect - golang.org/x/text v0.39.0 // indirect + golang.org/x/text v0.40.0 // indirect ) replace github.com/mujhtech/headgate/go => ../.. diff --git a/go/driver/headgatemysql/go.sum b/go/driver/headgatemysql/go.sum index b43e5b1..4563525 100644 --- a/go/driver/headgatemysql/go.sum +++ b/go/driver/headgatemysql/go.sum @@ -30,10 +30,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/go/driver/headgatemysql/inspect.go b/go/driver/headgatemysql/inspect.go index 7334244..6d084f3 100644 --- a/go/driver/headgatemysql/inspect.go +++ b/go/driver/headgatemysql/inspect.go @@ -23,6 +23,7 @@ package headgatemysql // so "0 rows" unambiguously means "no such row". import ( + "bytes" "context" "database/sql" "encoding/json" @@ -782,14 +783,14 @@ func (s *MysqlStore) OperatorRetry(ctx context.Context, id string) error { if _, err := tx.ExecContext(ctx, ` INSERT INTO headgate_active_partition (queue, partition_key) SELECT queue, partition_key FROM headgate_job - WHERE ulid = ? AND state IN ('archived', 'cancelled') + WHERE ulid = ? AND state IN ('archived', 'cancelled', 'undecodable') ON DUPLICATE KEY UPDATE queue = VALUES(queue)`, id); err != nil { return 0, err } res, err := tx.ExecContext(ctx, ` UPDATE headgate_job SET state = 'available', scheduled_at_ms = `+nowMS+`, finalized_at_ms = NULL - WHERE ulid = ? AND state IN ('archived', 'cancelled')`, id) + WHERE ulid = ? AND state IN ('archived', 'cancelled', 'undecodable')`, id) if err != nil { return 0, err } @@ -812,7 +813,7 @@ func (s *MysqlStore) OperatorRetry(ctx context.Context, id string) error { if !found { return headgate.NotFoundf("job %s", id) } - return headgate.Invalidf("operator_retry is only defined from archived or cancelled; job %s is %s", id, st) + return headgate.Invalidf("operator_retry is only defined from archived, cancelled, or undecodable; job %s is %s", id, st) } func (s *MysqlStore) OperatorCancel(ctx context.Context, id string) error { @@ -838,7 +839,7 @@ func (s *MysqlStore) OperatorCancel(ctx context.Context, id string) error { UPDATE headgate_job SET state = 'cancelled', lease_id = NULL, lease_expires_at_ms = NULL, claimed_by = NULL, finalized_at_ms = `+nowMS+` - WHERE ulid = ? AND state IN ('pending', 'scheduled', 'available', 'running')`, id) + WHERE ulid = ? AND state IN ('pending', 'scheduled', 'available', 'running', 'retryable')`, id) if err != nil { return 0, err } @@ -1287,6 +1288,83 @@ func (s *MysqlStore) ListScheduleEvents(ctx context.Context, scheduleID string, return out, rows.Err() } +func (s *MysqlStore) AppendDurableEvent(ctx context.Context, event headgate.DurableEvent) (headgate.DurableEvent, bool, error) { + if err := validateDurableEvent(event); err != nil { + return headgate.DurableEvent{}, false, err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return headgate.DurableEvent{}, false, err + } + defer func() { _ = tx.Rollback() }() + if _, err = tx.ExecContext(ctx, `INSERT IGNORE INTO headgate_durable_event_scope(scope) VALUES(?)`, event.Scope); err != nil { + return headgate.DurableEvent{}, false, err + } + var locked string + if err = tx.QueryRowContext(ctx, `SELECT scope FROM headgate_durable_event_scope WHERE scope=? FOR UPDATE`, event.Scope).Scan(&locked); err != nil { + return headgate.DurableEvent{}, false, err + } + var existingID uint64 + err = tx.QueryRowContext(ctx, `SELECT id FROM headgate_durable_event WHERE scope=? AND idempotency_key=?`, event.Scope, event.IdempotencyKey).Scan(&existingID) + inserted := errors.Is(err, sql.ErrNoRows) + if err != nil && !inserted { + return headgate.DurableEvent{}, false, err + } + if inserted { + if _, err = tx.ExecContext(ctx, `INSERT INTO headgate_durable_event(scope,topic,idempotency_key,payload,source,recorded_at_ms) VALUES(?,?,?,?,?,`+nowMS+`)`, event.Scope, event.Topic, event.IdempotencyKey, event.Payload, event.Source); err != nil { + return headgate.DurableEvent{}, false, err + } + } + var topic string + var payload, source []byte + if err = tx.QueryRowContext(ctx, `SELECT id,topic,payload,source,recorded_at_ms FROM headgate_durable_event WHERE scope=? AND idempotency_key=?`, event.Scope, event.IdempotencyKey).Scan(&event.EventID, &topic, &payload, &source, &event.RecordedAtMs); err != nil { + return headgate.DurableEvent{}, false, err + } + if topic != event.Topic || !bytes.Equal(payload, event.Payload) || !bytes.Equal(source, event.Source) { + return headgate.DurableEvent{}, false, &headgate.InvalidError{Msg: "durable event idempotency key was reused with different content"} + } + if _, err = tx.ExecContext(ctx, `DELETE e FROM headgate_durable_event e LEFT JOIN (SELECT id FROM headgate_durable_event WHERE scope=? ORDER BY id DESC LIMIT ?) keep ON keep.id=e.id WHERE e.scope=? AND keep.id IS NULL`, event.Scope, headgate.DurableEventLimit, event.Scope); err != nil { + return headgate.DurableEvent{}, false, err + } + if err = tx.Commit(); err != nil { + return headgate.DurableEvent{}, false, err + } + return event, inserted, nil +} + +func (s *MysqlStore) ListDurableEvents(ctx context.Context, scope string, beforeEventID uint64, limit uint32) ([]headgate.DurableEvent, error) { + if err := headgate.ValidateDurableEventLimit(limit); err != nil { + return nil, err + } + rows, err := s.db.QueryContext(ctx, `SELECT id,scope,topic,idempotency_key,payload,source,recorded_at_ms FROM headgate_durable_event WHERE scope=? AND (?=0 OR id 512 || event.Topic == "" || len(event.Topic) > 255 || event.IdempotencyKey == "" || len(event.IdempotencyKey) > 255 { + return &headgate.InvalidError{Msg: "durable event scope, topic, or idempotency key is invalid"} + } + if len(event.Payload) > headgate.MaxDurableEventPayloadBytes || !json.Valid(event.Payload) { + return &headgate.InvalidError{Msg: "durable event payload must be valid JSON of at most 65536 bytes"} + } + if len(event.Source) > headgate.MaxDurableEventSourceBytes || !json.Valid(event.Source) { + return &headgate.InvalidError{Msg: "durable event source must be valid JSON of at most 16384 bytes"} + } + return nil +} + // ---------- worker registry + surveyed policy behavior control channel ---------- func (s *MysqlStore) HeartbeatWorker(ctx context.Context, w headgate.WorkerMeta) (string, error) { @@ -1592,6 +1670,25 @@ func (s *MysqlStore) PromoteJob(ctx context.Context, id string) error { return tx.Commit() } +func (s *MysqlStore) SchedulePendingJob(ctx context.Context, id string, atMs int64) error { + if atMs <= 0 { + return headgate.Invalidf("pending schedule timestamp must be positive") + } + result, err := s.db.ExecContext(ctx, `UPDATE headgate_job SET state='scheduled', scheduled_at_ms=? + WHERE ulid=? AND state='pending'`, atMs, id) + if err != nil { + return err + } + n, err := result.RowsAffected() + if err != nil { + return err + } + if n != 1 { + return headgate.Invalidf("workflow_schedule is defined only from pending") + } + return nil +} + func (s *MysqlStore) DeleteQueue(ctx context.Context, queue string, force bool) (string, error) { tx, err := s.db.BeginTx(ctx, nil) if err != nil { diff --git a/go/driver/headgatepgx/go.mod b/go/driver/headgatepgx/go.mod index 633fcd2..f65beae 100644 --- a/go/driver/headgatepgx/go.mod +++ b/go/driver/headgatepgx/go.mod @@ -18,8 +18,8 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/redis/go-redis/v9 v9.7.3 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/text v0.39.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/text v0.40.0 // indirect ) replace github.com/mujhtech/headgate/go => ../.. diff --git a/go/driver/headgatepgx/go.sum b/go/driver/headgatepgx/go.sum index b43e5b1..4563525 100644 --- a/go/driver/headgatepgx/go.sum +++ b/go/driver/headgatepgx/go.sum @@ -30,10 +30,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/go/driver/headgatepgx/inspect.go b/go/driver/headgatepgx/inspect.go index 69148fc..c139484 100644 --- a/go/driver/headgatepgx/inspect.go +++ b/go/driver/headgatepgx/inspect.go @@ -7,6 +7,7 @@ package headgatepgx // exactness. import ( + "bytes" "context" "encoding/json" "errors" @@ -746,7 +747,7 @@ func (s *PgxStore) OperatorRetry(ctx context.Context, id string) error { WITH upd AS ( UPDATE headgate_job SET state = 'available', scheduled_at_ms = `+nowMS+`, finalized_at_ms = NULL - WHERE ulid = $1 AND state IN ('archived', 'cancelled') + WHERE ulid = $1 AND state IN ('archived', 'cancelled', 'undecodable') RETURNING queue, partition_key ), -- tenant fairness/adaptive admission retry-now makes the row available; list its partition here. @@ -769,7 +770,7 @@ func (s *PgxStore) OperatorRetry(ctx context.Context, id string) error { if !found { return headgate.NotFoundf("job %s", id) } - return headgate.Invalidf("operator_retry is only defined from archived or cancelled; job %s is %s", id, st) + return headgate.Invalidf("operator_retry is only defined from archived, cancelled, or undecodable; job %s is %s", id, st) } func (s *PgxStore) OperatorCancel(ctx context.Context, id string) error { @@ -783,7 +784,7 @@ func (s *PgxStore) OperatorCancel(ctx context.Context, id string) error { WITH pick AS ( SELECT j.id, j.queue, j.partition_key, (j.state = 'running') AS was_running FROM headgate_job j - WHERE j.ulid = $1 AND j.state IN ('pending', 'scheduled', 'available', 'running') + WHERE j.ulid = $1 AND j.state IN ('pending', 'scheduled', 'available', 'running', 'retryable') FOR UPDATE ), upd AS ( @@ -1211,6 +1212,81 @@ func (s *PgxStore) ListScheduleEvents(ctx context.Context, scheduleID string, be return out, rows.Err() } +func (s *PgxStore) AppendDurableEvent(ctx context.Context, event headgate.DurableEvent) (headgate.DurableEvent, bool, error) { + if err := validateDurableEvent(event); err != nil { + return headgate.DurableEvent{}, false, err + } + tx, err := s.pool.Begin(ctx) + if err != nil { + return headgate.DurableEvent{}, false, err + } + defer func() { _ = tx.Rollback(ctx) }() + if _, err = tx.Exec(ctx, `INSERT INTO headgate_durable_event_scope(scope) VALUES($1) ON CONFLICT DO NOTHING`, event.Scope); err != nil { + return headgate.DurableEvent{}, false, err + } + var locked string + if err = tx.QueryRow(ctx, `SELECT scope FROM headgate_durable_event_scope WHERE scope=$1 FOR UPDATE`, event.Scope).Scan(&locked); err != nil { + return headgate.DurableEvent{}, false, err + } + var id uint64 + var recorded int64 + err = tx.QueryRow(ctx, `INSERT INTO headgate_durable_event(scope,topic,idempotency_key,payload,source,recorded_at_ms) VALUES($1,$2,$3,$4,$5,`+nowMS+`) ON CONFLICT(scope,idempotency_key) DO NOTHING RETURNING id,recorded_at_ms`, event.Scope, event.Topic, event.IdempotencyKey, event.Payload, event.Source).Scan(&id, &recorded) + inserted := err == nil + if errors.Is(err, pgx.ErrNoRows) { + var topic string + var payload, source []byte + if err = tx.QueryRow(ctx, `SELECT id,topic,payload,source,recorded_at_ms FROM headgate_durable_event WHERE scope=$1 AND idempotency_key=$2`, event.Scope, event.IdempotencyKey).Scan(&id, &topic, &payload, &source, &recorded); err != nil { + return headgate.DurableEvent{}, false, err + } + if topic != event.Topic || !bytes.Equal(payload, event.Payload) || !bytes.Equal(source, event.Source) { + return headgate.DurableEvent{}, false, &headgate.InvalidError{Msg: "durable event idempotency key was reused with different content"} + } + } else if err != nil { + return headgate.DurableEvent{}, false, err + } + if _, err = tx.Exec(ctx, `DELETE FROM headgate_durable_event WHERE scope=$1 AND id NOT IN (SELECT id FROM headgate_durable_event WHERE scope=$1 ORDER BY id DESC LIMIT $2)`, event.Scope, headgate.DurableEventLimit); err != nil { + return headgate.DurableEvent{}, false, err + } + if err = tx.Commit(ctx); err != nil { + return headgate.DurableEvent{}, false, err + } + event.EventID, event.RecordedAtMs = id, recorded + return event, inserted, nil +} + +func (s *PgxStore) ListDurableEvents(ctx context.Context, scope string, beforeEventID uint64, limit uint32) ([]headgate.DurableEvent, error) { + if err := headgate.ValidateDurableEventLimit(limit); err != nil { + return nil, err + } + rows, err := s.pool.Query(ctx, `SELECT id,scope,topic,idempotency_key,payload,source,recorded_at_ms FROM headgate_durable_event WHERE scope=$1 AND ($2=0 OR id<$2) ORDER BY id DESC LIMIT $3`, scope, beforeEventID, limit) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]headgate.DurableEvent, 0, limit) + for rows.Next() { + var event headgate.DurableEvent + if err := rows.Scan(&event.EventID, &event.Scope, &event.Topic, &event.IdempotencyKey, &event.Payload, &event.Source, &event.RecordedAtMs); err != nil { + return nil, err + } + out = append(out, event) + } + return out, rows.Err() +} + +func validateDurableEvent(event headgate.DurableEvent) error { + if event.Scope == "" || len(event.Scope) > 512 || event.Topic == "" || len(event.Topic) > 255 || event.IdempotencyKey == "" || len(event.IdempotencyKey) > 255 { + return &headgate.InvalidError{Msg: "durable event scope, topic, or idempotency key is invalid"} + } + if len(event.Payload) > headgate.MaxDurableEventPayloadBytes || !json.Valid(event.Payload) { + return &headgate.InvalidError{Msg: "durable event payload must be valid JSON of at most 65536 bytes"} + } + if len(event.Source) > headgate.MaxDurableEventSourceBytes || !json.Valid(event.Source) { + return &headgate.InvalidError{Msg: "durable event source must be valid JSON of at most 16384 bytes"} + } + return nil +} + // ---------- worker registry + surveyed policy behavior control channel ---------- func (s *PgxStore) HeartbeatWorker(ctx context.Context, w headgate.WorkerMeta) (string, error) { @@ -1487,6 +1563,21 @@ func (s *PgxStore) PromoteJob(ctx context.Context, id string) error { return nil } +func (s *PgxStore) SchedulePendingJob(ctx context.Context, id string, atMs int64) error { + if atMs <= 0 { + return headgate.Invalidf("pending schedule timestamp must be positive") + } + result, err := s.pool.Exec(ctx, `UPDATE headgate_job SET state='scheduled', scheduled_at_ms=$2 + WHERE ulid=$1 AND state='pending'`, id, atMs) + if err != nil { + return err + } + if result.RowsAffected() != 1 { + return headgate.Invalidf("workflow_schedule is defined only from pending") + } + return nil +} + func queueDeleteID(now int64, queue string) string { clean := strings.Map(func(r rune) rune { if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' { diff --git a/go/driver/headgateredis/go.mod b/go/driver/headgateredis/go.mod index 1743f3b..9a422cf 100644 --- a/go/driver/headgateredis/go.mod +++ b/go/driver/headgateredis/go.mod @@ -17,7 +17,7 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.9.2 // indirect github.com/mujhtech/headgate/go/headgatemigrate v0.1.7 // indirect - golang.org/x/text v0.39.0 // indirect + golang.org/x/text v0.40.0 // indirect ) replace github.com/mujhtech/headgate/go => ../.. diff --git a/go/driver/headgateredis/go.sum b/go/driver/headgateredis/go.sum index b43e5b1..4563525 100644 --- a/go/driver/headgateredis/go.sum +++ b/go/driver/headgateredis/go.sum @@ -30,10 +30,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/go/driver/headgateredis/inspect.go b/go/driver/headgateredis/inspect.go index 2e7aafd..9027701 100644 --- a/go/driver/headgateredis/inspect.go +++ b/go/driver/headgateredis/inspect.go @@ -9,6 +9,7 @@ package headgateredis // match the other backends word-for-word (the mutation-diff discipline). import ( + "bytes" "context" "encoding/json" "errors" @@ -972,7 +973,7 @@ func (s *RedisStore) OperatorRetry(ctx context.Context, id string) error { case "DUP": return &headgate.DuplicateError{ExistingID: second(res)} default: - return headgate.Invalidf("operator_retry is only defined from archived or cancelled; job %s is %s", + return headgate.Invalidf("operator_retry is only defined from archived, cancelled, or undecodable; job %s is %s", id, second(res)) } } @@ -1327,6 +1328,92 @@ func (s *RedisStore) ListScheduleEvents(ctx context.Context, scheduleID string, return out, nil } +func (s *RedisStore) AppendDurableEvent(ctx context.Context, event headgate.DurableEvent) (headgate.DurableEvent, bool, error) { + if err := validateDurableEvent(event); err != nil { + return headgate.DurableEvent{}, false, err + } + now, err := s.storeNowMs(ctx) + if err != nil { + return headgate.DurableEvent{}, false, err + } + const script = `local old=redis.call('HGET',KEYS[2],ARGV[2]) +if old then return {'0',old} end +local id=redis.call('INCR',KEYS[3]) +local value=cjson.encode({event_id=id,scope=ARGV[1],topic=ARGV[3],idempotency_key=ARGV[2],payload=cjson.decode(ARGV[4]),source=cjson.decode(ARGV[5]),recorded_at_ms=ARGV[6]}) +redis.call('HSET',KEYS[2],ARGV[2],value) +redis.call('ZADD',KEYS[1],id,value) +local excess=redis.call('ZCARD',KEYS[1])-tonumber(ARGV[7]) +if excess>0 then local gone=redis.call('ZRANGE',KEYS[1],0,excess-1) for _,v in ipairs(gone) do local e=cjson.decode(v) redis.call('HDEL',KEYS[2],e.idempotency_key) end redis.call('ZREMRANGEBYRANK',KEYS[1],0,excess-1) end +return {'1',value}` + values, err := redis.NewScript(script).Run(ctx, s.rdb, []string{s.key("durable-events", event.Scope), s.key("durable-event-idem", event.Scope), s.key("durable-event-seq")}, event.Scope, event.IdempotencyKey, event.Topic, string(event.Payload), string(event.Source), now, headgate.DurableEventLimit).StringSlice() + if err != nil { + return headgate.DurableEvent{}, false, err + } + if len(values) != 2 { + return headgate.DurableEvent{}, false, fmt.Errorf("headgate: unexpected durable event reply") + } + stored, err := decodeDurableEvent([]byte(values[1])) + if err != nil { + return headgate.DurableEvent{}, false, err + } + if stored.Topic != event.Topic || !bytes.Equal(stored.Payload, event.Payload) || !bytes.Equal(stored.Source, event.Source) { + return headgate.DurableEvent{}, false, &headgate.InvalidError{Msg: "durable event idempotency key was reused with different content"} + } + return stored, values[0] == "1", nil +} + +func (s *RedisStore) ListDurableEvents(ctx context.Context, scope string, beforeEventID uint64, limit uint32) ([]headgate.DurableEvent, error) { + if err := headgate.ValidateDurableEventLimit(limit); err != nil { + return nil, err + } + max := "+inf" + if beforeEventID != 0 { + max = "(" + strconv.FormatUint(beforeEventID, 10) + } + values, err := s.rdb.ZRevRangeByScore(ctx, s.key("durable-events", scope), &redis.ZRangeBy{Max: max, Min: "-inf", Offset: 0, Count: int64(limit)}).Result() + if err != nil { + return nil, err + } + out := make([]headgate.DurableEvent, 0, len(values)) + for _, value := range values { + event, err := decodeDurableEvent([]byte(value)) + if err != nil { + return nil, err + } + out = append(out, event) + } + return out, nil +} + +func validateDurableEvent(event headgate.DurableEvent) error { + if event.Scope == "" || len(event.Scope) > 512 || event.Topic == "" || len(event.Topic) > 255 || event.IdempotencyKey == "" || len(event.IdempotencyKey) > 255 { + return &headgate.InvalidError{Msg: "durable event scope, topic, or idempotency key is invalid"} + } + if len(event.Payload) > headgate.MaxDurableEventPayloadBytes || !json.Valid(event.Payload) { + return &headgate.InvalidError{Msg: "durable event payload must be valid JSON of at most 65536 bytes"} + } + if len(event.Source) > headgate.MaxDurableEventSourceBytes || !json.Valid(event.Source) { + return &headgate.InvalidError{Msg: "durable event source must be valid JSON of at most 16384 bytes"} + } + return nil +} + +func decodeDurableEvent(data []byte) (headgate.DurableEvent, error) { + var raw struct { + EventID uint64 `json:"event_id"` + Scope string `json:"scope"` + Topic string `json:"topic"` + IdempotencyKey string `json:"idempotency_key"` + Payload json.RawMessage `json:"payload"` + Source json.RawMessage `json:"source"` + RecordedAtMs int64 `json:"recorded_at_ms"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return headgate.DurableEvent{}, fmt.Errorf("decoding stored durable event: %w", err) + } + return headgate.DurableEvent{EventID: raw.EventID, Scope: raw.Scope, Topic: raw.Topic, IdempotencyKey: raw.IdempotencyKey, Payload: raw.Payload, Source: raw.Source, RecordedAtMs: raw.RecordedAtMs}, nil +} + func (s *RedisStore) HeartbeatWorker(ctx context.Context, w headgate.WorkerMeta) (string, error) { status := w.Status if status == "" { @@ -1671,6 +1758,29 @@ func (s *RedisStore) PromoteJob(ctx context.Context, id string) error { } } +func (s *RedisStore) SchedulePendingJob(ctx context.Context, id string, atMs int64) error { + if atMs <= 0 { + return headgate.Invalidf("pending schedule timestamp must be positive") + } + res, err := s.adminJobOp(ctx, "schedule_pending", id, atMs) + if err != nil { + return err + } + if len(res) == 0 { + return errors.New("invalid schedule_pending response") + } + switch res[0] { + case "OK": + return nil + case "NF": + return headgate.NotFoundf("job %s", id) + case "ERR": + return headgate.Invalidf("workflow_schedule is defined only from pending") + default: + return errors.New("invalid schedule_pending response") + } +} + func (s *RedisStore) DeleteQueue(ctx context.Context, queue string, force bool) (string, error) { now, err := s.storeNowMs(ctx) if err != nil { diff --git a/go/driver/headgateredis/lua/admin.lua b/go/driver/headgateredis/lua/admin.lua index fed1cca..2311bc7 100644 --- a/go/driver/headgateredis/lua/admin.lua +++ b/go/driver/headgateredis/lua/admin.lua @@ -95,7 +95,7 @@ local function do_cancel(id, h) if ret > 0 then redis.call('ZADD', p .. ':ret', now + ret, id) end end --- archived|cancelled -> available (operator_retry). +-- archived|cancelled|undecodable -> available (operator_retry). local function do_retry(id, h) local st, q, part, fp = h[1], h[2], h[3], h[4] local uk, uw = h[5], tonumber(h[6] or '0') or 0 @@ -141,7 +141,7 @@ if op == 'cancel' then local id = ARGV[2] local h = job_head(id) if not h[1] then return {'NF'} end - if h[1] ~= 'pending' and h[1] ~= 'scheduled' and h[1] ~= 'available' and h[1] ~= 'running' then + if h[1] ~= 'pending' and h[1] ~= 'scheduled' and h[1] ~= 'available' and h[1] ~= 'running' and h[1] ~= 'retryable' then return {'ERR', h[1]} end do_cancel(id, h) @@ -159,6 +159,17 @@ elseif op == 'promote' then redis.call('ZADD', p .. ':avail:' .. h[2] .. ':' .. h[3], now, id) redis.call('ZADD', p .. ':metricparts:' .. h[2], now, h[3]) return {'OK'} +elseif op == 'schedule_pending' then + local id, at = ARGV[2], tonumber(ARGV[3]) + local h = job_head(id) + if not h[1] then return {'NF'} end + if h[1] ~= 'pending' or not at or at <= 0 then return {'ERR', h[1]} end + redis.call('ZREM', idx(h[2], 'pending'), id) + redis.call('ZADD', idx(h[2], 'scheduled'), at, id) + redis.call('ZADD', p .. ':sched', at, id) + add_waiting(id, h[2], h[3], h[8], at) + redis.call('HSET', jk(id), 'state', 'scheduled', 'scheduled_at_ms', at) + return {'OK'} elseif op == 'queue_delete' then local q, force, opid = ARGV[2], ARGV[3] == '1', ARGV[4] @@ -187,7 +198,7 @@ elseif op == 'retry' then local id = ARGV[2] local h = job_head(id) if not h[1] then return {'NF'} end - if h[1] ~= 'archived' and h[1] ~= 'cancelled' then return {'ERR', h[1]} end + if h[1] ~= 'archived' and h[1] ~= 'cancelled' and h[1] ~= 'undecodable' then return {'ERR', h[1]} end local holder = do_retry(id, h) if holder then return {'DUP', holder} end return {'OK'} diff --git a/go/go.work b/go/go.work deleted file mode 100644 index dd1de5a..0000000 --- a/go/go.work +++ /dev/null @@ -1,16 +0,0 @@ -go 1.25.0 - -use ( - . - ./driver/headgatemysql - ./driver/headgatepgx - ./driver/headgateredis - ./headgateapi - ./headgatecrypto - ./headgatectl - ./headgatemigrate - ./headgateotel - ./headgatetest - ./headgateui - ./headgateworkflow -) diff --git a/go/headgate.go b/go/headgate.go index c361113..3dfa346 100644 --- a/go/headgate.go +++ b/go/headgate.go @@ -143,6 +143,30 @@ const ( OutcomeRateLimited = headgateshared.OutcomeRateLimited ) +const DurableEventLimit = uint32(100) +const MaxDurableEventPayloadBytes = 64 * 1024 +const MaxDurableEventSourceBytes = 16 * 1024 + +// DurableEvent is one bounded, store-timestamped fact attached to an application scope. +// Payload and Source contain valid JSON so every backend preserves the same value. +type DurableEvent struct { + EventID uint64 + Scope string + Topic string + IdempotencyKey string + Payload []byte + Source []byte + RecordedAtMs int64 +} + +// DurableEventStore is an optional inspection capability used by layered packages such +// as headgateworkflow. It stays separate from InspectStore so core inspection adapters +// do not falsely claim durable event support. +type DurableEventStore interface { + AppendDurableEvent(ctx context.Context, event DurableEvent) (stored DurableEvent, inserted bool, err error) + ListDurableEvents(ctx context.Context, scope string, beforeEventID uint64, limit uint32) ([]DurableEvent, error) +} + func ParseOutcome(value string) (Outcome, bool) { return headgateshared.ParseOutcome(value) } @@ -1027,6 +1051,14 @@ type CheckpointInspectStore interface { GetJobCheckpoint(ctx context.Context, id string) (*Checkpoint, error) } +// PendingScheduleStore is the narrow optional capability workflow-relative timers use +// to atomically move pending work to an absolute store-clock deadline. Keeping it +// separate from InspectStore does not force unrelated third-party inspection adapters +// to claim support they do not have. +type PendingScheduleStore interface { + SchedulePendingJob(ctx context.Context, id string, atMs int64) error +} + // Tx is a caller-owned store transaction. Drivers wrap their concrete handle and // recover it via Unwrap — the Go mirror of Rust's TxHandle::as_any (transactional API): the // compile-time path is typed, the dyn path downcasts, and a foreign handle is a hard @@ -1702,6 +1734,13 @@ func ValidateScheduleEventLimit(limit uint32) error { return nil } +func ValidateDurableEventLimit(limit uint32) error { + if limit == 0 || limit > DurableEventLimit { + return &InvalidError{Msg: "durable event limit must be between 1 and 100"} + } + return nil +} + // SaturationStrategy is the wire/storage spelling read by every atomic gate. type SaturationStrategy = headgateshared.SaturationStrategy diff --git a/go/headgateapi/api.go b/go/headgateapi/api.go index addfa8d..5cdbdef 100644 --- a/go/headgateapi/api.go +++ b/go/headgateapi/api.go @@ -23,6 +23,7 @@ import ( "time" headgate "github.com/mujhtech/headgate/go" + headgateworkflow "github.com/mujhtech/headgate/go/headgateworkflow" ) type api struct { @@ -174,10 +175,251 @@ func handler( mux.HandleFunc("GET /api/v1/workers", a.workers) mux.HandleFunc("GET /api/v1/cluster", a.cluster) mux.HandleFunc("POST /api/v1/workers/{worker_id}/signal", a.signalWorker) + mux.HandleFunc("GET /api/v1/workflows", a.workflowList) + mux.HandleFunc("GET /api/v1/workflows/{id}", a.workflowDetail) + mux.HandleFunc("GET /api/v1/workflows/{id}/events", a.workflowEvents) + mux.HandleFunc("GET /api/v1/workflows/{id}/nodes/{node}", a.workflowNode) + mux.HandleFunc("GET /api/v1/workflows/{id}/nodes/{node}/dependencies", a.workflowDependencies) + mux.HandleFunc("GET /api/v1/workflows/{id}/nodes/{node}/dependents", a.workflowDependents) + mux.HandleFunc("POST /api/v1/workflows/{id}/signals", a.workflowSignal) + mux.HandleFunc("GET /api/v1/workflows/{id}/signals", a.workflowSignals) + mux.HandleFunc("POST /api/v1/workflows/{id}/grafts", a.workflowGraft) + mux.HandleFunc("POST /api/v1/workflows/{id}/retry", a.workflowRetry) + mux.HandleFunc("POST /api/v1/workflows/{id}/cancel", a.workflowCancel) mux.HandleFunc("GET /api/v1/events", a.events) return routeParity(mux) } +func workflowErr(w http.ResponseWriter, err error) { + message := err.Error() + status := http.StatusBadRequest + if strings.Contains(message, "was not found") { + status = http.StatusNotFound + } else if strings.Contains(message, "revision conflict") { + status = http.StatusConflict + } + errJSON(w, status, message) +} + +func (a *api) workflowEvents(w http.ResponseWriter, r *http.Request) { + events, err := headgateworkflow.WorkflowEvents(r.Context(), a.store, r.PathValue("id")) + if err != nil { + workflowErr(w, err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"events": events}) +} + +func (a *api) workflowDetail(w http.ResponseWriter, r *http.Request) { + snapshot, err := headgateworkflow.InspectWorkflow(r.Context(), a.store, r.PathValue("id")) + if err != nil { + workflowErr(w, err) + return + } + writeJSON(w, http.StatusOK, snapshot) +} + +func (a *api) workflowList(w http.ResponseWriter, r *http.Request) { + limit, ok := queryUint32(w, r, "limit", 50) + if !ok { + return + } + if r.URL.Query().Has("cursor") && r.URL.Query().Get("cursor") == "" { + errJSON(w, http.StatusBadRequest, "bad cursor") + return + } + page, err := headgateworkflow.ListWorkflows(r.Context(), a.store, r.URL.Query().Get("cursor"), limit) + if err != nil { + workflowErr(w, err) + return + } + var cursor any + if page.NextCursor != "" { + cursor = page.NextCursor + } + writeJSON(w, http.StatusOK, map[string]any{"workflows": page.Workflows, "next_cursor": cursor}) +} + +func (a *api) workflowNode(w http.ResponseWriter, r *http.Request) { + node, err := headgateworkflow.GetWorkflowNode(r.Context(), a.store, r.PathValue("id"), r.PathValue("node")) + if err != nil { + workflowErr(w, err) + return + } + writeJSON(w, http.StatusOK, node) +} + +func (a *api) workflowDependencies(w http.ResponseWriter, r *http.Request) { + dependencies, err := headgateworkflow.WorkflowDependencies(r.Context(), a.store, r.PathValue("id"), r.PathValue("node")) + if err != nil { + workflowErr(w, err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"dependencies": dependencies}) +} + +func (a *api) workflowDependents(w http.ResponseWriter, r *http.Request) { + dependents, err := headgateworkflow.WorkflowDependents(r.Context(), a.store, r.PathValue("id"), r.PathValue("node")) + if err != nil { + workflowErr(w, err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"dependents": dependents}) +} + +func (a *api) workflowSignal(w http.ResponseWriter, r *http.Request) { + var body struct { + Signal *string `json:"signal"` + Payload json.RawMessage `json:"payload"` + Source json.RawMessage `json:"source"` + } + raw, ok := decodeJSON(w, r, &body) + if !ok || !requireFields(w, raw, "signal") { + return + } + receipt, err := headgateworkflow.EmitSignalWith(r.Context(), a.store, r.PathValue("id"), headgateworkflow.SignalEmission{ + Signal: *body.Signal, IdempotencyKey: r.Header.Get("Idempotency-Key"), Payload: body.Payload, Source: body.Source, + }) + if err != nil { + workflowErr(w, err) + return + } + writeJSON(w, http.StatusOK, receipt) +} + +func (a *api) workflowSignals(w http.ResponseWriter, r *http.Request) { + limit, ok := queryUint32(w, r, "limit", 100) + if !ok { + return + } + before, ok := queryUint64(w, r, "cursor", 0) + if !ok { + return + } + signals, err := headgateworkflow.ListSignals(r.Context(), a.store, r.PathValue("id"), before, limit) + if err != nil { + workflowErr(w, err) + return + } + var next any + if len(signals) == int(limit) { + next = signals[len(signals)-1].ID + } + writeJSON(w, http.StatusOK, map[string]any{"signals": signals, "next_cursor": next}) +} + +func (a *api) workflowRetry(w http.ResponseWriter, r *http.Request) { + var body struct { + ExpectedRevision *uint64 `json:"expected_revision"` + Recoveries []struct { + Node string `json:"node"` + Payload *string `json:"payload"` + SchemaVersion uint32 `json:"schema_version"` + ReleaseQuarantine bool `json:"release_quarantine"` + } `json:"recoveries"` + } + raw, ok := decodeJSON(w, r, &body) + if !ok || !requireFields(w, raw, "expected_revision") { + return + } + recoveries := make([]headgateworkflow.WorkflowRecovery, 0, len(body.Recoveries)) + for _, recovery := range body.Recoveries { + var payload []byte + if recovery.Payload != nil { + var err error + payload, err = base64.StdEncoding.DecodeString(*recovery.Payload) + if err != nil { + errJSON(w, http.StatusBadRequest, "recovery payload must be base64") + return + } + } + recoveries = append(recoveries, headgateworkflow.WorkflowRecovery{ + Node: recovery.Node, Payload: payload, SchemaVersion: recovery.SchemaVersion, + ReleaseQuarantine: recovery.ReleaseQuarantine, + }) + } + receipt, err := headgateworkflow.RequestFailedSubgraphRetryWithRecovery( + r.Context(), a.store, r.PathValue("id"), *body.ExpectedRevision, recoveries, + ) + if err != nil { + workflowErr(w, err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"revision": receipt.Revision, "generation": receipt.Generation}) +} + +func (a *api) workflowCancel(w http.ResponseWriter, r *http.Request) { + var body struct { + PropagateChildren *bool `json:"propagate_children"` + } + if _, ok := decodeJSON(w, r, &body); !ok { + return + } + propagate := true + if body.PropagateChildren != nil { + propagate = *body.PropagateChildren + } + receipt, err := headgateworkflow.CancelWorkflow(r.Context(), a.store, r.PathValue("id"), propagate) + if err != nil { + workflowErr(w, err) + return + } + writeJSON(w, http.StatusOK, receipt) +} + +type workflowGraftBody struct { + ExpectedRevision *uint64 `json:"expected_revision"` + Queue string `json:"queue"` + Tasks []workflowGraftTask `json:"tasks"` +} + +type workflowGraftTask struct { + Name string `json:"name"` + Deps []string `json:"deps"` + Kind string `json:"kind"` + Payload string `json:"payload"` + ID string `json:"id"` + Queue string `json:"queue"` + SchemaVersion uint32 `json:"schema_version"` +} + +func (a *api) workflowGraft(w http.ResponseWriter, r *http.Request) { + var body workflowGraftBody + raw, ok := decodeJSON(w, r, &body) + if !ok || !requireFields(w, raw, "expected_revision", "tasks") { + return + } + graft := headgateworkflow.NewGraft(r.PathValue("id"), *body.ExpectedRevision) + if body.Queue != "" { + graft.Queue(body.Queue) + } + for _, task := range body.Tasks { + payload, err := base64.StdEncoding.DecodeString(task.Payload) + if err != nil { + errJSON(w, http.StatusBadRequest, "payload must be base64") + return + } + version := task.SchemaVersion + if version == 0 { + version = 1 + } + graft.Add(task.Name, headgate.Envelope{ + ID: task.ID, Kind: task.Kind, Payload: payload, Queue: task.Queue, + SchemaVersion: version, Fingerprint: headgate.Fingerprint(task.Kind, payload), + }, task.Deps...) + } + batch, err := graft.Prepare() + if err != nil { + workflowErr(w, err) + return + } + if err := a.producer.EnqueueWithSource(r.Context(), headgate.EnqueueSourceHTTP, batch); err != nil { + enqueueClientErr(w, err) + return + } + writeJSON(w, http.StatusAccepted, map[string]any{"receipt_id": batch[0].ID}) +} + // routeParity makes an unrouted path or a wrong method answer the way axum's Router // does, and runs the Idempotency-Key check only AFTER a route has matched. // diff --git a/go/headgateapi/api_test.go b/go/headgateapi/api_test.go index e3a3702..22eb8a2 100644 --- a/go/headgateapi/api_test.go +++ b/go/headgateapi/api_test.go @@ -50,6 +50,93 @@ type outputAPIStore struct { type queuePageStore struct{ errStore } +type workflowAPIStore struct{ errStore } + +type workflowSignalAPIStore struct { + workflowAPIStore + events []headgate.DurableEvent +} + +func (s *workflowSignalAPIStore) GetJob(_ context.Context, id string, includePayload bool) (*headgate.JobSummary, error) { + jobs := map[string]headgate.JobSummary{ + "wf:coordinator": {ID: "wf:coordinator", Kind: "headgate:workflow", State: "running"}, + "wf:approval": {ID: "wf:approval", Kind: "headgate:workflow-signal", State: "pending"}, + } + job, ok := jobs[id] + if !ok { + return nil, nil + } + if includePayload && id == "wf:coordinator" { + job.Payload = []byte(`{"workflow_id":"wf","nodes":[{"name":"approval","job_id":"wf:approval","deps":[],"kind":"signal","signal":"approved"}]}`) + } + return &job, nil +} + +func (s *workflowSignalAPIStore) PromoteJob(context.Context, string) error { return nil } + +func (s *workflowSignalAPIStore) AppendDurableEvent(_ context.Context, event headgate.DurableEvent) (headgate.DurableEvent, bool, error) { + for _, existing := range s.events { + if existing.IdempotencyKey == event.IdempotencyKey { + if existing.Topic != event.Topic || string(existing.Payload) != string(event.Payload) || string(existing.Source) != string(event.Source) { + return headgate.DurableEvent{}, false, headgate.Invalidf("durable event idempotency key was reused with different content") + } + return existing, false, nil + } + } + event.EventID = uint64(len(s.events) + 1) + event.RecordedAtMs = int64(event.EventID) + s.events = append([]headgate.DurableEvent{event}, s.events...) + return event, true, nil +} + +func (s *workflowSignalAPIStore) ListDurableEvents(_ context.Context, _ string, before uint64, limit uint32) ([]headgate.DurableEvent, error) { + out := make([]headgate.DurableEvent, 0, limit) + for _, event := range s.events { + if before == 0 || event.EventID < before { + out = append(out, event) + if len(out) == int(limit) { + break + } + } + } + return out, nil +} + +func (s *workflowAPIStore) GetJob(_ context.Context, id string, includePayload bool) (*headgate.JobSummary, error) { + jobs := map[string]headgate.JobSummary{ + "wf:coordinator": {ID: "wf:coordinator", Kind: "headgate:workflow", State: "running"}, + "wf:prepare": {ID: "wf:prepare", Kind: "task:prepare", State: "completed"}, + "wf:publish": {ID: "wf:publish", Kind: "task:publish", State: "pending"}, + } + job, ok := jobs[id] + if !ok { + return nil, nil + } + if includePayload && id == "wf:coordinator" { + job.Payload = []byte(`{"workflow_id":"wf","nodes":[{"name":"prepare","job_id":"wf:prepare","deps":[],"kind":"task"},{"name":"publish","job_id":"wf:publish","deps":["prepare"],"kind":"task"}]}`) + } + return &job, nil +} + +func (s *workflowAPIStore) GetJobCheckpoint(_ context.Context, id string) (*headgate.Checkpoint, error) { + if id != "wf:coordinator" { + return nil, nil + } + return &headgate.Checkpoint{ + CursorStep: "headgate:workflow-state", + Cursor: []byte(`{"revision":2,"generation":1,"completed":["prepare"],"completed_at_ms":{"prepare":42}}`), + }, nil +} + +func (s *workflowAPIStore) ListJobs(_ context.Context, filter headgate.JobFilter, _ string, _ uint32) (headgate.JobPage, error) { + if filter.Kind == nil || *filter.Kind != "headgate:workflow" { + return headgate.JobPage{}, errors.New("workflow list did not use the coordinator kind") + } + return headgate.JobPage{Jobs: []headgate.JobSummary{{ + ID: "wf:coordinator", Kind: "headgate:workflow", State: "running", + }}}, nil +} + func (s *queuePageStore) QueueStats(context.Context) ([]headgate.QueueStatsView, error) { stats := make([]headgate.QueueStatsView, 205) for i := range stats { @@ -706,6 +793,127 @@ func TestRouteParity(t *testing.T) { } } +func TestWorkflowMutationRoutesRequireIdempotencyKey(t *testing.T) { + h := Handler(&errStore{err: errors.New("unused")}) + for _, path := range []string{ + "/api/v1/workflows/wf/signals", + "/api/v1/workflows/wf/grafts", + "/api/v1/workflows/wf/retry", + "/api/v1/workflows/wf/cancel", + } { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{}`)) + r.Header.Set("Content-Type", "application/json") + h.ServeHTTP(w, r) + if w.Code != http.StatusBadRequest || !strings.Contains(w.Body.String(), "Idempotency-Key") { + t.Fatalf("POST %s without key = %d %s", path, w.Code, w.Body.String()) + } + } +} + +func TestWorkflowSignalRoutePersistsPayloadSourceAndReplay(t *testing.T) { + store := &workflowSignalAPIStore{} + h := Handler(store) + body := `{"signal":"approved","payload":{"approved":true,"reviewer":"Ada"},"source":{"emitter":"admin-console","actor":"operator-42"}}` + + post := func() map[string]any { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/api/v1/workflows/wf/signals", strings.NewReader(body)) + r.Header.Set("Content-Type", "application/json") + r.Header.Set("Idempotency-Key", "workflow-signal-1") + h.ServeHTTP(w, r) + if w.Code != http.StatusOK { + t.Fatalf("POST signal = %d %s", w.Code, w.Body.String()) + } + var response map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + return response + } + + first := post() + if first["inserted"] != true { + t.Fatalf("first signal receipt = %#v", first) + } + emission := first["emission"].(map[string]any) + if emission["idempotency_key"] != "workflow-signal-1" || emission["payload"].(map[string]any)["reviewer"] != "Ada" || emission["source"].(map[string]any)["actor"] != "operator-42" { + t.Fatalf("signal emission = %#v", emission) + } + + replay := post() + if replay["inserted"] != false || !reflect.DeepEqual(replay["emission"], first["emission"]) { + t.Fatalf("signal replay = %#v", replay) + } + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/v1/workflows/wf/signals?limit=100", nil)) + if w.Code != http.StatusOK { + t.Fatalf("GET signal history = %d %s", w.Code, w.Body.String()) + } + var history map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &history); err != nil { + t.Fatal(err) + } + signals := history["signals"].([]any) + if len(signals) != 1 || !reflect.DeepEqual(signals[0], first["emission"]) { + t.Fatalf("signal history = %#v", history) + } +} + +func TestWorkflowInspectionRoutesAreReadOnlyAndBoundedToKnownNodes(t *testing.T) { + h := Handler(&errStore{}) + for _, path := range []string{ + "/api/v1/workflows/missing", + "/api/v1/workflows/missing/nodes/task", + "/api/v1/workflows/missing/nodes/task/dependencies", + "/api/v1/workflows/missing/nodes/task/dependents", + } { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != http.StatusNotFound { + t.Fatalf("GET %s = %d %s, want 404", path, w.Code, w.Body.String()) + } + } +} + +func TestWorkflowInspectionRoutesReturnTopologyWithoutPayloads(t *testing.T) { + h := Handler(&workflowAPIStore{}) + for _, test := range []struct { + path, field, name string + }{ + {"/api/v1/workflows", "workflows", "wf"}, + {"/api/v1/workflows/wf", "nodes", "prepare"}, + {"/api/v1/workflows/wf/nodes/publish", "", "publish"}, + {"/api/v1/workflows/wf/nodes/publish/dependencies", "dependencies", "prepare"}, + {"/api/v1/workflows/wf/nodes/prepare/dependents", "dependents", "publish"}, + } { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, test.path, nil)) + if w.Code != http.StatusOK || strings.Contains(w.Body.String(), "Payload") { + t.Fatalf("GET %s = %d %s", test.path, w.Code, w.Body.String()) + } + var body map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if test.field == "" { + if body["name"] != test.name { + t.Fatalf("GET %s name = %#v", test.path, body["name"]) + } + continue + } + items, ok := body[test.field].([]any) + itemKey := "name" + if test.field == "workflows" { + itemKey = "workflow_id" + } + if !ok || len(items) == 0 || items[0].(map[string]any)[itemKey] != test.name { + t.Fatalf("GET %s %s = %#v", test.path, test.field, body[test.field]) + } + } +} + // TestRequiredFieldsRejected is the teeth on the tier-1 data-corruption fixes. Each of // these bodies used to reach the store and mutate a job. func TestRequiredFieldsRejected(t *testing.T) { diff --git a/go/headgateapi/go.mod b/go/headgateapi/go.mod index 64ed3c5..8e19947 100644 --- a/go/headgateapi/go.mod +++ b/go/headgateapi/go.mod @@ -4,8 +4,12 @@ go 1.25.0 require github.com/mujhtech/headgate/go v0.1.7 +require github.com/mujhtech/headgate/go/headgateworkflow v0.1.7 + replace github.com/mujhtech/headgate/go => ../ +replace github.com/mujhtech/headgate/go/headgateworkflow => ../headgateworkflow + require github.com/mujhtech/headgate/go/driver/headgatepgx v0.1.7 require github.com/mujhtech/headgate/go/driver/headgateredis v0.1.7 @@ -15,7 +19,10 @@ require github.com/mujhtech/headgate/go/driver/headgatemysql v0.1.7 require github.com/mujhtech/headgate/go/headgateui v0.1.7 require ( + cel.dev/cel-go v0.32.0 // indirect + cel.dev/expr v0.25.1 // indirect filippo.io/edwards25519 v1.2.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/go-sql-driver/mysql v1.9.3 // indirect @@ -23,9 +30,15 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.9.2 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/kr/text v0.2.0 // indirect github.com/redis/go-redis/v9 v9.7.3 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/text v0.39.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/text v0.40.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect + google.golang.org/protobuf v1.36.12 // indirect ) replace github.com/mujhtech/headgate/go/driver/headgatepgx => ../driver/headgatepgx diff --git a/go/headgateapi/go.sum b/go/headgateapi/go.sum index b43e5b1..62da5e0 100644 --- a/go/headgateapi/go.sum +++ b/go/headgateapi/go.sum @@ -1,11 +1,18 @@ +cel.dev/cel-go v0.32.0 h1:irvpFKr5EuGPyxeME03ERh0rii1TX+BDAnB9eL3IvNk= +cel.dev/cel-go v0.32.0/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -13,6 +20,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -21,20 +30,38 @@ github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/headgatecrypto/go.mod b/go/headgatecrypto/go.mod index c10a462..ebd22f5 100644 --- a/go/headgatecrypto/go.mod +++ b/go/headgatecrypto/go.mod @@ -4,7 +4,10 @@ go 1.25.0 require github.com/mujhtech/headgate/go v0.1.7 -require github.com/mujhtech/headgate/go/headgatetest v0.1.7 // indirect +require ( + github.com/mujhtech/headgate/go/headgatetest v0.1.7 // indirect + golang.org/x/text v0.40.0 // indirect +) replace github.com/mujhtech/headgate/go => .. diff --git a/go/headgatecrypto/go.sum b/go/headgatecrypto/go.sum index 55d9045..9ccf788 100644 --- a/go/headgatecrypto/go.sum +++ b/go/headgatecrypto/go.sum @@ -14,5 +14,4 @@ github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= diff --git a/go/headgatemigrate/go.mod b/go/headgatemigrate/go.mod index 517b90d..df96e19 100644 --- a/go/headgatemigrate/go.mod +++ b/go/headgatemigrate/go.mod @@ -12,7 +12,7 @@ require ( filippo.io/edwards25519 v1.2.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - golang.org/x/text v0.39.0 // indirect + golang.org/x/text v0.40.0 // indirect ) replace github.com/mujhtech/headgate/go => .. diff --git a/go/headgatemigrate/go.sum b/go/headgatemigrate/go.sum index a4d75fc..581c13b 100644 --- a/go/headgatemigrate/go.sum +++ b/go/headgatemigrate/go.sum @@ -20,10 +20,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/go/headgatemigrate/live_mysql_test.go b/go/headgatemigrate/live_mysql_test.go index d01e6cc..f74e116 100644 --- a/go/headgatemigrate/live_mysql_test.go +++ b/go/headgatemigrate/live_mysql_test.go @@ -78,19 +78,19 @@ SELECT count(*) FROM information_schema.schemata WHERE schema_name = ?`, databas } result, err := MigrateMySQL(ctx, db, Up, Options{}) - if err != nil || len(result.Steps) != 12 || result.Steps[0].Migration.Version != 1 || result.Steps[11].Migration.Version != 12 { + if err != nil || len(result.Steps) != 13 || result.Steps[0].Migration.Version != 1 || result.Steps[12].Migration.Version != 13 { t.Fatalf("fresh up = %#v, %v", result, err) } validation, err := ValidateMySQL(ctx, db) - if err != nil || !validation.OK() || validation.CurrentVersion != 12 { + if err != nil || !validation.OK() || validation.CurrentVersion != 13 { t.Fatalf("fresh validation = %#v, %v", validation, err) } dry, err := MigrateMySQL(ctx, db, Down, Options{DryRun: true}) - if err != nil || !dry.DryRun || len(dry.Steps) != 12 { + if err != nil || !dry.DryRun || len(dry.Steps) != 13 { t.Fatalf("down dry-run = %#v, %v", dry, err) } downResult, err := MigrateMySQL(ctx, db, Down, Options{}) - if err != nil || len(downResult.Steps) != 12 { + if err != nil || len(downResult.Steps) != 13 { t.Fatalf("down = %#v, %v", downResult, err) } var jobExists, historyRows int @@ -125,7 +125,7 @@ UPDATE headgate_schema_migration SET checksum = 'tampered' WHERE version = 1`); t.Fatalf("unversioned up error = %v", err) } adopted, err := AdoptMySQL(ctx, db) - if err != nil || len(adopted) != 12 || adopted[11].Version != 12 { + if err != nil || len(adopted) != 13 || adopted[12].Version != 13 { t.Fatalf("adopted = %#v, %v", adopted, err) } validation, err = ValidateMySQL(ctx, db) @@ -259,7 +259,7 @@ SELECT count(*) FROM information_schema.schemata WHERE schema_name = ?`, databas } select { case migrated := <-done: - if migrated.err != nil || len(migrated.result.Steps) != 12 { + if migrated.err != nil || len(migrated.result.Steps) != 13 { t.Fatalf("configured migration = %#v, %v", migrated.result, migrated.err) } case <-time.After(20 * time.Second): diff --git a/go/headgatemigrate/live_postgres_test.go b/go/headgatemigrate/live_postgres_test.go index 4301d14..5503d69 100644 --- a/go/headgatemigrate/live_postgres_test.go +++ b/go/headgatemigrate/live_postgres_test.go @@ -50,19 +50,19 @@ SELECT count(*) FROM information_schema.schemata WHERE schema_name = $1`, schema } result, err := MigratePostgres(ctx, conn, Up, Options{}) - if err != nil || len(result.Steps) != 12 || result.Steps[0].Migration.Version != 1 || result.Steps[11].Migration.Version != 12 { + if err != nil || len(result.Steps) != 13 || result.Steps[0].Migration.Version != 1 || result.Steps[12].Migration.Version != 13 { t.Fatalf("fresh up = %#v, %v", result, err) } validation, err := ValidatePostgres(ctx, conn) - if err != nil || !validation.OK() || validation.CurrentVersion != 12 { + if err != nil || !validation.OK() || validation.CurrentVersion != 13 { t.Fatalf("fresh validation = %#v, %v", validation, err) } dry, err := MigratePostgres(ctx, conn, Down, Options{DryRun: true}) - if err != nil || !dry.DryRun || len(dry.Steps) != 12 { + if err != nil || !dry.DryRun || len(dry.Steps) != 13 { t.Fatalf("down dry-run = %#v, %v", dry, err) } downResult, err := MigratePostgres(ctx, conn, Down, Options{}) - if err != nil || len(downResult.Steps) != 12 { + if err != nil || len(downResult.Steps) != 13 { t.Fatalf("down = %#v, %v", downResult, err) } var jobExists bool @@ -95,7 +95,7 @@ UPDATE headgate_schema_migration SET checksum = 'tampered' WHERE version = 1`); t.Fatalf("unversioned up error = %v", err) } adopted, err := AdoptPostgres(ctx, conn) - if err != nil || len(adopted) != 12 || adopted[11].Version != 12 { + if err != nil || len(adopted) != 13 || adopted[12].Version != 13 { t.Fatalf("adopted = %#v, %v", adopted, err) } validation, err = ValidatePostgres(ctx, conn) diff --git a/go/headgatemigrate/migrate.go b/go/headgatemigrate/migrate.go index 2995892..3cf315c 100644 --- a/go/headgatemigrate/migrate.go +++ b/go/headgatemigrate/migrate.go @@ -108,6 +108,12 @@ var postgresWorkerControlStateUp string //go:embed migrations/postgres/0012_worker_control_state.down.sql var postgresWorkerControlStateDown string +//go:embed migrations/postgres/0013_durable_events.up.sql +var postgresDurableEventsUp string + +//go:embed migrations/postgres/0013_durable_events.down.sql +var postgresDurableEventsDown string + //go:embed migrations/mysql/0001_init.up.sql var mysqlInitialUp string @@ -180,6 +186,12 @@ var mysqlWorkerControlStateUp string //go:embed migrations/mysql/0012_worker_control_state.down.sql var mysqlWorkerControlStateDown string +//go:embed migrations/mysql/0013_durable_events.up.sql +var mysqlDurableEventsUp string + +//go:embed migrations/mysql/0013_durable_events.down.sql +var mysqlDurableEventsDown string + var byBackend = map[Backend][]Migration{ Postgres: { {Version: 1, Name: "initial_schema", UpSQL: postgresInitialUp, DownSQL: postgresInitialDown, OnlineSafe: false}, @@ -194,6 +206,7 @@ var byBackend = map[Backend][]Migration{ {Version: 10, Name: "sticky_routing", UpSQL: postgresStickyRoutingUp, DownSQL: postgresStickyRoutingDown, OnlineSafe: false}, {Version: 11, Name: "partitioned_archive", UpSQL: postgresPartitionedArchiveUp, DownSQL: postgresPartitionedArchiveDown, OnlineSafe: true}, {Version: 12, Name: "worker_control_state", UpSQL: postgresWorkerControlStateUp, DownSQL: postgresWorkerControlStateDown, OnlineSafe: true}, + {Version: 13, Name: "durable_events", UpSQL: postgresDurableEventsUp, DownSQL: postgresDurableEventsDown, OnlineSafe: true}, }, MySQL: { {Version: 1, Name: "initial_schema", UpSQL: mysqlInitialUp, DownSQL: mysqlInitialDown, OnlineSafe: false}, @@ -208,6 +221,7 @@ var byBackend = map[Backend][]Migration{ {Version: 10, Name: "sticky_routing", UpSQL: mysqlStickyRoutingUp, DownSQL: mysqlStickyRoutingDown, OnlineSafe: false}, {Version: 11, Name: "partitioned_archive", UpSQL: mysqlPartitionedArchiveUp, DownSQL: mysqlPartitionedArchiveDown, OnlineSafe: false}, {Version: 12, Name: "worker_control_state", UpSQL: mysqlWorkerControlStateUp, DownSQL: mysqlWorkerControlStateDown, OnlineSafe: false}, + {Version: 13, Name: "durable_events", UpSQL: mysqlDurableEventsUp, DownSQL: mysqlDurableEventsDown, OnlineSafe: true}, }, } diff --git a/go/headgatemigrate/migrate_test.go b/go/headgatemigrate/migrate_test.go index 0256ff1..e9af07a 100644 --- a/go/headgatemigrate/migrate_test.go +++ b/go/headgatemigrate/migrate_test.go @@ -13,16 +13,16 @@ func applied(backend Backend, version int) AppliedMigration { func TestPlanUpDownAndCurrentNoop(t *testing.T) { steps, err := Plan(Postgres, nil, Up, Options{}) - if err != nil || len(steps) != 12 || steps[0].Migration.Version != 1 || steps[11].Migration.Version != 12 { + if err != nil || len(steps) != 13 || steps[0].Migration.Version != 1 || steps[12].Migration.Version != 13 { t.Fatalf("up plan = %#v, %v", steps, err) } - current := []AppliedMigration{applied(Postgres, 1), applied(Postgres, 2), applied(Postgres, 3), applied(Postgres, 4), applied(Postgres, 5), applied(Postgres, 6), applied(Postgres, 7), applied(Postgres, 8), applied(Postgres, 9), applied(Postgres, 10), applied(Postgres, 11), applied(Postgres, 12)} + current := []AppliedMigration{applied(Postgres, 1), applied(Postgres, 2), applied(Postgres, 3), applied(Postgres, 4), applied(Postgres, 5), applied(Postgres, 6), applied(Postgres, 7), applied(Postgres, 8), applied(Postgres, 9), applied(Postgres, 10), applied(Postgres, 11), applied(Postgres, 12), applied(Postgres, 13)} steps, err = Plan(Postgres, current, Up, Options{}) if err != nil || len(steps) != 0 { t.Fatalf("current plan = %#v, %v", steps, err) } steps, err = Plan(Postgres, current, Down, Options{}) - if err != nil || len(steps) != 12 || steps[0].Direction != Down || steps[0].Migration.Version != 12 { + if err != nil || len(steps) != 13 || steps[0].Direction != Down || steps[0].Migration.Version != 13 { t.Fatalf("down plan = %#v, %v", steps, err) } } @@ -38,15 +38,15 @@ func TestChecksumAndHistoryGapFailPlanning(t *testing.T) { t.Fatalf("error = %T %v", err, err) } } - future := AppliedMigration{Version: 12, Name: "future", Checksum: "x", AppliedAtMS: 1} + future := AppliedMigration{Version: 13, Name: "future", Checksum: "x", AppliedAtMS: 1} if err := ValidateHistory(Postgres, []AppliedMigration{future}); err == nil { t.Fatal("history gap accepted") } } func TestTargetsAndMaxStepsAreBounded(t *testing.T) { - thirteen := 13 - if _, err := Plan(Postgres, nil, Up, Options{TargetVersion: &thirteen}); err == nil { + fourteen := 14 + if _, err := Plan(Postgres, nil, Up, Options{TargetVersion: &fourteen}); err == nil { t.Fatal("future target accepted") } zero := 0 diff --git a/go/headgatemigrate/migrations/mysql/0013_durable_events.down.sql b/go/headgatemigrate/migrations/mysql/0013_durable_events.down.sql new file mode 100644 index 0000000..106c075 --- /dev/null +++ b/go/headgatemigrate/migrations/mysql/0013_durable_events.down.sql @@ -0,0 +1,2 @@ +DROP TABLE headgate_durable_event; +DROP TABLE headgate_durable_event_scope; diff --git a/go/headgatemigrate/migrations/mysql/0013_durable_events.up.sql b/go/headgatemigrate/migrations/mysql/0013_durable_events.up.sql new file mode 100644 index 0000000..4eceb1d --- /dev/null +++ b/go/headgatemigrate/migrations/mysql/0013_durable_events.up.sql @@ -0,0 +1,17 @@ +CREATE TABLE headgate_durable_event_scope ( + scope VARCHAR(512) NOT NULL PRIMARY KEY +); + +CREATE TABLE headgate_durable_event ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + scope VARCHAR(512) NOT NULL, + topic VARCHAR(255) NOT NULL, + idempotency_key VARCHAR(255) NOT NULL, + payload LONGBLOB NOT NULL, + source LONGBLOB NOT NULL, + recorded_at_ms BIGINT NOT NULL, + UNIQUE KEY headgate_durable_event_idempotency (scope, idempotency_key), + KEY headgate_durable_event_recent (scope, id DESC), + CONSTRAINT headgate_durable_event_scope_fk FOREIGN KEY (scope) + REFERENCES headgate_durable_event_scope(scope) ON DELETE CASCADE +); diff --git a/go/headgatemigrate/migrations/postgres/0013_durable_events.down.sql b/go/headgatemigrate/migrations/postgres/0013_durable_events.down.sql new file mode 100644 index 0000000..106c075 --- /dev/null +++ b/go/headgatemigrate/migrations/postgres/0013_durable_events.down.sql @@ -0,0 +1,2 @@ +DROP TABLE headgate_durable_event; +DROP TABLE headgate_durable_event_scope; diff --git a/go/headgatemigrate/migrations/postgres/0013_durable_events.up.sql b/go/headgatemigrate/migrations/postgres/0013_durable_events.up.sql new file mode 100644 index 0000000..308f8d2 --- /dev/null +++ b/go/headgatemigrate/migrations/postgres/0013_durable_events.up.sql @@ -0,0 +1,17 @@ +CREATE TABLE headgate_durable_event_scope ( + scope text PRIMARY KEY +); + +CREATE TABLE headgate_durable_event ( + id bigserial PRIMARY KEY, + scope text NOT NULL REFERENCES headgate_durable_event_scope(scope) ON DELETE CASCADE, + topic text NOT NULL, + idempotency_key text NOT NULL, + payload bytea NOT NULL, + source bytea NOT NULL, + recorded_at_ms bigint NOT NULL, + UNIQUE (scope, idempotency_key) +); + +CREATE INDEX headgate_durable_event_recent + ON headgate_durable_event (scope, id DESC); diff --git a/go/headgateotel/go.mod b/go/headgateotel/go.mod index 77b1e13..8ca4591 100644 --- a/go/headgateotel/go.mod +++ b/go/headgateotel/go.mod @@ -20,7 +20,10 @@ require ( go.opentelemetry.io/auto/sdk v1.2.1 // indirect ) -require golang.org/x/sys v0.41.0 // indirect +require ( + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.40.0 // indirect +) replace github.com/mujhtech/headgate/go => .. diff --git a/go/headgateotel/go.sum b/go/headgateotel/go.sum index 48258e5..b5c8f76 100644 --- a/go/headgateotel/go.sum +++ b/go/headgateotel/go.sum @@ -43,9 +43,7 @@ go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/headgatetest/go.mod b/go/headgatetest/go.mod index 57b5fbe..e1498f0 100644 --- a/go/headgatetest/go.mod +++ b/go/headgatetest/go.mod @@ -16,7 +16,7 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - golang.org/x/text v0.39.0 // indirect + golang.org/x/text v0.40.0 // indirect ) replace github.com/mujhtech/headgate/go => .. diff --git a/go/headgatetest/go.sum b/go/headgatetest/go.sum index b43e5b1..4563525 100644 --- a/go/headgatetest/go.sum +++ b/go/headgatetest/go.sum @@ -30,10 +30,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/go/headgateworkflow/experimental/reducer.go b/go/headgateworkflow/experimental/reducer.go deleted file mode 100644 index 7f2a60b..0000000 --- a/go/headgateworkflow/experimental/reducer.go +++ /dev/null @@ -1,418 +0,0 @@ -// Package experimental settles dynamic workflow semantics before durable adapters and -// control APIs make the contract permanent. It is not a persistence implementation. -package experimental - -import ( - "errors" - "fmt" - "sort" -) - -type NodeKind string - -const ( - Task NodeKind = "task" - Signal NodeKind = "signal" - Timer NodeKind = "timer" - ChildWorkflow NodeKind = "child_workflow" -) - -type NodeSpec struct { - Name string - Deps []string - Kind NodeKind - Signal string - WakeAtMs int64 - WorkflowID string -} - -func TaskNode(name string, deps ...string) NodeSpec { - return NodeSpec{Name: name, Deps: clone(deps), Kind: Task} -} - -func SignalNode(name, signal string, deps ...string) NodeSpec { - return NodeSpec{Name: name, Deps: clone(deps), Kind: Signal, Signal: signal} -} - -func TimerNode(name string, wakeAtMs int64, deps ...string) NodeSpec { - return NodeSpec{Name: name, Deps: clone(deps), Kind: Timer, WakeAtMs: wakeAtMs} -} - -func ChildNode(name, workflowID string, deps ...string) NodeSpec { - return NodeSpec{Name: name, Deps: clone(deps), Kind: ChildWorkflow, WorkflowID: workflowID} -} - -type NodeState string - -const ( - Waiting NodeState = "waiting" - Active NodeState = "active" - Succeeded NodeState = "succeeded" - Failed NodeState = "failed" - Blocked NodeState = "blocked" -) - -type RunStatus string - -const ( - Running RunStatus = "running" - RunSucceeded RunStatus = "succeeded" - RunFailed RunStatus = "failed" -) - -type RuntimeNode struct { - Spec NodeSpec - State NodeState -} - -type Run struct { - Revision uint64 - Generation uint32 - Status RunStatus - StoreNowMs int64 - Nodes map[string]*RuntimeNode - Signals map[string]struct{} -} - -type ActionType string - -const ( - DispatchTask ActionType = "dispatch_task" - WaitForSignal ActionType = "wait_for_signal" - ArmTimer ActionType = "arm_timer" - StartChildWorkflow ActionType = "start_child_workflow" - WorkflowSucceeded ActionType = "workflow_succeeded" - WorkflowFailed ActionType = "workflow_failed" -) - -type Action struct { - Type ActionType - Name string - Signal string - WakeAtMs int64 - WorkflowID string - Generation uint32 -} - -func NewRun(nodes []NodeSpec, storeNowMs int64) (*Run, []Action, error) { - if err := validateGraph(nodes); err != nil { - return nil, nil, err - } - run := &Run{ - Revision: 1, Generation: 1, Status: Running, StoreNowMs: storeNowMs, - Nodes: make(map[string]*RuntimeNode, len(nodes)), Signals: map[string]struct{}{}, - } - for _, raw := range nodes { - spec := cloneSpec(raw) - run.Nodes[spec.Name] = &RuntimeNode{Spec: spec, State: Waiting} - } - return run, run.reconcile(), nil -} - -func (r *Run) ReceiveSignal(signal string) ([]Action, error) { - if signal == "" { - return nil, errors.New("signal name must not be empty") - } - known := false - for _, node := range r.Nodes { - if node.Spec.Kind == Signal && node.Spec.Signal == signal { - known = true - break - } - } - if !known { - return nil, fmt.Errorf("unknown signal `%s`", signal) - } - r.Signals[signal] = struct{}{} - for _, node := range r.Nodes { - if node.State == Active && node.Spec.Kind == Signal && node.Spec.Signal == signal { - node.State = Succeeded - } - } - return r.reconcile(), nil -} - -func (r *Run) AdvanceStoreTime(nowMs int64) ([]Action, error) { - if nowMs < r.StoreNowMs { - return nil, errors.New("store time must not move backwards") - } - r.StoreNowMs = nowMs - for _, node := range r.Nodes { - if node.State == Active && node.Spec.Kind == Timer && node.Spec.WakeAtMs <= nowMs { - node.State = Succeeded - } - } - return r.reconcile(), nil -} - -func (r *Run) SucceedNode(name string) ([]Action, error) { - if err := r.settleNode(name, true); err != nil { - return nil, err - } - return r.reconcile(), nil -} - -func (r *Run) FailNode(name string) ([]Action, error) { - if err := r.settleNode(name, false); err != nil { - return nil, err - } - r.blockDescendants(name) - r.Status = RunFailed - return []Action{{Type: WorkflowFailed, Name: name, Generation: r.Generation}}, nil -} - -func (r *Run) Graft(expectedRevision uint64, nodes ...NodeSpec) ([]Action, error) { - if err := r.requireRevision(expectedRevision); err != nil { - return nil, err - } - if r.Status != Running { - return nil, errors.New("nodes may only be grafted onto a running workflow") - } - if len(nodes) == 0 { - return nil, errors.New("graft must contain at least one node") - } - combined := make([]NodeSpec, 0, len(r.Nodes)+len(nodes)) - for _, node := range r.Nodes { - combined = append(combined, cloneSpec(node.Spec)) - } - for _, node := range nodes { - if _, exists := r.Nodes[node.Name]; exists { - return nil, fmt.Errorf("graft repeats existing node `%s`", node.Name) - } - combined = append(combined, cloneSpec(node)) - } - if err := validateGraph(combined); err != nil { - return nil, err - } - for _, raw := range nodes { - spec := cloneSpec(raw) - r.Nodes[spec.Name] = &RuntimeNode{Spec: spec, State: Waiting} - } - r.Revision++ - return r.reconcile(), nil -} - -func (r *Run) RetryFailedSubgraph(expectedRevision uint64) ([]Action, error) { - if err := r.requireRevision(expectedRevision); err != nil { - return nil, err - } - if r.Status != RunFailed { - return nil, errors.New("only a failed workflow may be retried") - } - if r.Generation == ^uint32(0) { - return nil, errors.New("workflow generation overflow") - } - for _, node := range r.Nodes { - if node.State == Failed || node.State == Blocked { - node.State = Waiting - } - } - r.Generation++ - r.Revision++ - r.Status = Running - return r.reconcile(), nil -} - -func (r *Run) requireRevision(expected uint64) error { - if expected != r.Revision { - return fmt.Errorf("revision conflict: expected %d, current %d", expected, r.Revision) - } - return nil -} - -func (r *Run) settleNode(name string, success bool) error { - node := r.Nodes[name] - if node == nil { - return fmt.Errorf("unknown node `%s`", name) - } - if node.State != Active { - return fmt.Errorf("node `%s` is not active", name) - } - if node.Spec.Kind != Task && node.Spec.Kind != ChildWorkflow { - return fmt.Errorf("node `%s` is settled by its signal or timer", name) - } - if success { - node.State = Succeeded - } else { - node.State = Failed - } - return nil -} - -func (r *Run) blockDescendants(failed string) { - queue := []string{failed} - for len(queue) > 0 { - parent := queue[0] - queue = queue[1:] - children := make([]string, 0) - for name, node := range r.Nodes { - if contains(node.Spec.Deps, parent) { - children = append(children, name) - } - } - sort.Strings(children) - for _, child := range children { - node := r.Nodes[child] - if node.State == Waiting || node.State == Active { - node.State = Blocked - queue = append(queue, child) - } - } - } -} - -func (r *Run) reconcile() []Action { - if r.Status != Running { - return nil - } - actions := make([]Action, 0) - for { - ready := make([]string, 0) - for name, node := range r.Nodes { - if node.State != Waiting { - continue - } - complete := true - for _, dep := range node.Spec.Deps { - if r.Nodes[dep] == nil || r.Nodes[dep].State != Succeeded { - complete = false - break - } - } - if complete { - ready = append(ready, name) - } - } - sort.Strings(ready) - if len(ready) == 0 { - break - } - completedVirtual := false - for _, name := range ready { - node := r.Nodes[name] - switch node.Spec.Kind { - case Task: - node.State = Active - actions = append(actions, Action{Type: DispatchTask, Name: name, Generation: r.Generation}) - case Signal: - if _, received := r.Signals[node.Spec.Signal]; received { - node.State = Succeeded - completedVirtual = true - } else { - node.State = Active - actions = append(actions, Action{Type: WaitForSignal, Name: name, Signal: node.Spec.Signal}) - } - case Timer: - if node.Spec.WakeAtMs <= r.StoreNowMs { - node.State = Succeeded - completedVirtual = true - } else { - node.State = Active - actions = append(actions, Action{Type: ArmTimer, Name: name, WakeAtMs: node.Spec.WakeAtMs}) - } - case ChildWorkflow: - node.State = Active - actions = append(actions, Action{ - Type: StartChildWorkflow, Name: name, WorkflowID: node.Spec.WorkflowID, Generation: r.Generation, - }) - } - } - if !completedVirtual { - break - } - } - allSucceeded := true - for _, node := range r.Nodes { - if node.State != Succeeded { - allSucceeded = false - break - } - } - if allSucceeded { - r.Status = RunSucceeded - actions = append(actions, Action{Type: WorkflowSucceeded, Generation: r.Generation}) - } - return actions -} - -func validateGraph(nodes []NodeSpec) error { - if len(nodes) == 0 { - return errors.New("workflow must contain at least one node") - } - names := make(map[string]struct{}, len(nodes)) - for _, node := range nodes { - if node.Name == "" { - return errors.New("node names must be non-empty and unique") - } - if _, exists := names[node.Name]; exists { - return errors.New("node names must be non-empty and unique") - } - names[node.Name] = struct{}{} - if node.Kind == Signal && node.Signal == "" { - return fmt.Errorf("signal node `%s` has an empty signal", node.Name) - } - if node.Kind == ChildWorkflow && node.WorkflowID == "" { - return fmt.Errorf("child node `%s` has an empty workflow id", node.Name) - } - if node.Kind != Task && node.Kind != Signal && node.Kind != Timer && node.Kind != ChildWorkflow { - return fmt.Errorf("node `%s` has unknown kind `%s`", node.Name, node.Kind) - } - } - degree := make(map[string]int, len(nodes)) - outgoing := make(map[string][]string) - for _, node := range nodes { - seen := map[string]struct{}{} - for _, dep := range node.Deps { - if _, exists := names[dep]; !exists { - return fmt.Errorf("node `%s` depends on missing node `%s`", node.Name, dep) - } - if _, exists := seen[dep]; exists { - return fmt.Errorf("node `%s` repeats dependency `%s`", node.Name, dep) - } - seen[dep] = struct{}{} - degree[node.Name]++ - outgoing[dep] = append(outgoing[dep], node.Name) - } - } - ready := make([]string, 0) - for name := range names { - if degree[name] == 0 { - ready = append(ready, name) - } - } - sort.Strings(ready) - visited := 0 - for len(ready) > 0 { - name := ready[0] - ready = ready[1:] - visited++ - children := outgoing[name] - sort.Strings(children) - for _, child := range children { - degree[child]-- - if degree[child] == 0 { - ready = append(ready, child) - } - } - } - if visited != len(nodes) { - return errors.New("workflow graph contains a cycle") - } - return nil -} - -func contains(values []string, want string) bool { - for _, value := range values { - if value == want { - return true - } - } - return false -} - -func clone(values []string) []string { return append([]string(nil), values...) } - -func cloneSpec(spec NodeSpec) NodeSpec { - spec.Deps = clone(spec.Deps) - return spec -} diff --git a/go/headgateworkflow/experimental/reducer_test.go b/go/headgateworkflow/experimental/reducer_test.go deleted file mode 100644 index 5ec6934..0000000 --- a/go/headgateworkflow/experimental/reducer_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package experimental - -import ( - "strings" - "testing" -) - -func actionNames(actions []Action) []string { - result := make([]string, 0) - for _, action := range actions { - if action.Type == DispatchTask || action.Type == StartChildWorkflow { - result = append(result, action.Name) - } - } - return result -} - -func TestSignalsAndStoreTimeTimersUnlockInDependencyOrder(t *testing.T) { - run, first, err := NewRun([]NodeSpec{ - TaskNode("prepare"), - SignalNode("approval", "approved", "prepare"), - TimerNode("release", 1_500, "approval"), - TaskNode("publish", "release"), - }, 1_000) - if err != nil || len(first) != 1 || first[0].Name != "prepare" { - t.Fatalf("new run = %#v, %v", first, err) - } - if _, err := run.ReceiveSignal("typo"); err == nil || !strings.Contains(err.Error(), "unknown signal") { - t.Fatalf("unknown signal = %v", err) - } - if actions, err := run.ReceiveSignal("approved"); err != nil || len(actions) != 0 { - t.Fatalf("early signal = %#v, %v", actions, err) - } - wait, err := run.SucceedNode("prepare") - if err != nil || len(wait) != 1 || wait[0].Type != ArmTimer || wait[0].WakeAtMs != 1_500 { - t.Fatalf("timer arm = %#v, %v", wait, err) - } - if actions, err := run.AdvanceStoreTime(1_499); err != nil || len(actions) != 0 { - t.Fatalf("early time = %#v, %v", actions, err) - } - actions, err := run.AdvanceStoreTime(1_500) - if err != nil || len(actionNames(actions)) != 1 || actionNames(actions)[0] != "publish" { - t.Fatalf("timer fire = %#v, %v", actions, err) - } -} - -func TestGraftIsAdditiveRevisionCheckedAndCycleSafe(t *testing.T) { - run, _, err := NewRun([]NodeSpec{TaskNode("root")}, 0) - if err != nil { - t.Fatal(err) - } - if actions, err := run.Graft(1, TaskNode("grafted", "root")); err != nil || len(actions) != 0 { - t.Fatalf("graft = %#v, %v", actions, err) - } - if run.Revision != 2 { - t.Fatalf("revision = %d", run.Revision) - } - if _, err := run.Graft(1, TaskNode("stale", "root")); err == nil || !strings.Contains(err.Error(), "revision conflict") { - t.Fatalf("stale graft = %v", err) - } - if _, err := run.Graft(2, TaskNode("a", "b"), TaskNode("b", "a")); err == nil || !strings.Contains(err.Error(), "cycle") { - t.Fatalf("cyclic graft = %v", err) - } -} - -func TestNestedFailureRetriesOnlyFailedSubgraph(t *testing.T) { - run, first, err := NewRun([]NodeSpec{ - TaskNode("extract"), - ChildNode("child", "child-workflow", "extract"), - TaskNode("finish", "child"), - }, 0) - if err != nil || len(first) != 1 || first[0].Name != "extract" { - t.Fatalf("new run = %#v, %v", first, err) - } - child, err := run.SucceedNode("extract") - if err != nil || len(child) != 1 || child[0].Type != StartChildWorkflow { - t.Fatalf("child start = %#v, %v", child, err) - } - failed, err := run.FailNode("child") - if err != nil || len(failed) != 1 || failed[0].Type != WorkflowFailed { - t.Fatalf("child failure = %#v, %v", failed, err) - } - if run.Nodes["extract"].State != Succeeded || run.Nodes["finish"].State != Blocked { - t.Fatalf("states = extract %s, finish %s", run.Nodes["extract"].State, run.Nodes["finish"].State) - } - retried, err := run.RetryFailedSubgraph(1) - if err != nil || len(retried) != 1 || retried[0].Name != "child" || retried[0].Generation != 2 { - t.Fatalf("retry = %#v, %v", retried, err) - } - if run.Generation != 2 || run.Nodes["extract"].State != Succeeded { - t.Fatalf("generation = %d, extract = %s", run.Generation, run.Nodes["extract"].State) - } -} diff --git a/go/headgateworkflow/go.mod b/go/headgateworkflow/go.mod index 8a574e0..525d7e5 100644 --- a/go/headgateworkflow/go.mod +++ b/go/headgateworkflow/go.mod @@ -2,12 +2,54 @@ module github.com/mujhtech/headgate/go/headgateworkflow go 1.25.0 -require github.com/mujhtech/headgate/go v0.1.7 +require ( + cel.dev/cel-go v0.32.0 + github.com/mujhtech/headgate/go v0.1.7 +) -require github.com/mujhtech/headgate/go/headgatetest v0.1.7 // indirect +require ( + github.com/mujhtech/headgate/go/driver/headgatemysql v0.1.7 + github.com/mujhtech/headgate/go/driver/headgatepgx v0.1.7 + github.com/mujhtech/headgate/go/driver/headgateredis v0.1.7 +) + +require github.com/rogpeppe/go-internal v1.16.0 // indirect + +require ( + cel.dev/expr v0.25.1 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/kr/pretty v0.3.1 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect + golang.org/x/sync v0.22.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) + +require ( + filippo.io/edwards25519 v1.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/go-sql-driver/mysql v1.9.3 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.9.2 // indirect + github.com/mujhtech/headgate/go/headgatemigrate v0.1.7 // indirect + github.com/mujhtech/headgate/go/headgatetest v0.1.7 + github.com/redis/go-redis/v9 v9.7.3 // indirect + golang.org/x/text v0.40.0 // indirect +) replace github.com/mujhtech/headgate/go => .. replace github.com/mujhtech/headgate/go/headgatetest => ../headgatetest replace github.com/mujhtech/headgate/go/headgatemigrate => ../headgatemigrate + +replace github.com/mujhtech/headgate/go/driver/headgatemysql => ../driver/headgatemysql + +replace github.com/mujhtech/headgate/go/driver/headgatepgx => ../driver/headgatepgx + +replace github.com/mujhtech/headgate/go/driver/headgateredis => ../driver/headgateredis diff --git a/go/headgateworkflow/go.sum b/go/headgateworkflow/go.sum index 55d9045..355181e 100644 --- a/go/headgateworkflow/go.sum +++ b/go/headgateworkflow/go.sum @@ -1,18 +1,69 @@ +cel.dev/cel-go v0.32.0 h1:irvpFKr5EuGPyxeME03ERh0rii1TX+BDAnB9eL3IvNk= +cel.dev/cel-go v0.32.0/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= -golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= -golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/headgateworkflow/live_matrix_test.go b/go/headgateworkflow/live_matrix_test.go new file mode 100644 index 0000000..e541397 --- /dev/null +++ b/go/headgateworkflow/live_matrix_test.go @@ -0,0 +1,163 @@ +package headgateworkflow + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "reflect" + "strconv" + "sync" + "testing" + "time" + + headgate "github.com/mujhtech/headgate/go" + "github.com/mujhtech/headgate/go/driver/headgatepgx" + "github.com/mujhtech/headgate/go/driver/headgateredis" +) + +type matrixStep struct { + Name string `json:"name"` +} + +func (matrixStep) Kind() string { return "workflow:matrix-step" } + +func matrixEnvelope(queue, name string) headgate.Envelope { + payload := []byte(`{"name":"` + name + `"}`) + return headgate.Envelope{ + Kind: matrixStep{}.Kind(), Payload: payload, Queue: queue, + Fingerprint: headgate.Fingerprint(matrixStep{}.Kind(), payload), + } +} + +func runWorkflowMatrixCell(t *testing.T, store headgate.InspectStore, backend string) { + t.Helper() + ctx := context.Background() + suffix := strconv.Itoa(os.Getpid()) + "-" + backend + "-" + strconv.FormatInt(time.Now().UnixNano(), 10) + workflowID := "workflow-matrix-go-" + suffix + queue := "workflow-matrix-go-" + suffix + unstable := matrixEnvelope(queue, "unstable") + unstable.MaxAttempts = 1 + w := New(workflowID).CoordinatorQueue(queue) + if err := w.AutomaticRetry(2, 2*time.Millisecond); err != nil { + t.Fatal(err) + } + w.Add("prepare", matrixEnvelope(queue, "prepare")) + w.Add("unstable", unstable, "prepare") + w.AddCondition("ready", `completed.unstable && states.unstable == "completed"`, "unstable") + if err := w.AddTimerAfter("pause", 2*time.Millisecond, "ready"); err != nil { + t.Fatal(err) + } + w.AddSignal("approval", "approved", "pause") + w.Add("finish", matrixEnvelope(queue, "finish"), "approval") + batch, err := w.Prepare() + if err != nil { + t.Fatal(err) + } + if err := store.Enqueue(ctx, batch); err != nil { + t.Fatal(err) + } + emission := SignalEmission{ + Signal: "approved", IdempotencyKey: "matrix-approval:" + workflowID, + Payload: json.RawMessage(`{"approved":true,"backend":"` + backend + `"}`), + Source: json.RawMessage(`{"emitter":"workflow-matrix"}`), + } + receipt, err := EmitSignalWith(ctx, store, workflowID, emission) + if err != nil || receipt.Matched != 1 || !receipt.Inserted { + t.Fatalf("early signal = %#v, %v", receipt, err) + } + replay, err := EmitSignalWith(ctx, store, workflowID, emission) + if err != nil || replay.Inserted || !reflect.DeepEqual(replay.Emission, receipt.Emission) { + t.Fatalf("signal replay = %#v, %v", replay, err) + } + signals, err := ListSignals(ctx, store, workflowID, 0, 100) + if err != nil || len(signals) != 1 || !reflect.DeepEqual(signals[0], receipt.Emission) { + t.Fatalf("signal history = %#v, %v", signals, err) + } + + reg := headgate.NewRegistry() + if err := RegisterCoordinator(reg, store, 2*time.Millisecond); err != nil { + t.Fatal(err) + } + remaining := 1 + var mu sync.Mutex + order := make([]string, 0, 4) + if err := headgate.RegisterFunc[matrixStep](reg, func(_ context.Context, job *headgate.Job[matrixStep]) error { + mu.Lock() + defer mu.Unlock() + order = append(order, job.Args.Name) + if job.Args.Name == "unstable" && remaining > 0 { + remaining-- + return errors.New("planned workflow failure") + } + return nil + }); err != nil { + t.Fatal(err) + } + runner := headgate.NewRunner(store, reg, headgate.Config{ + Queues: map[string]headgate.QueueConfig{queue: {MaxWorkers: 8}}, + LeaseDuration: 30 * time.Second, + }) + for range 100 { + if _, err := runner.Drain(ctx, 32); err != nil { + t.Fatal(err) + } + coordinator, err := store.GetJob(ctx, workflowID+":coordinator", false) + if err != nil { + t.Fatal(err) + } + if coordinator != nil && coordinator.State == "completed" { + break + } + time.Sleep(3 * time.Millisecond) + } + coordinator, err := store.GetJob(ctx, workflowID+":coordinator", false) + if err != nil || coordinator == nil || coordinator.State != "completed" { + t.Fatalf("coordinator = %#v, %v", coordinator, err) + } + mu.Lock() + gotOrder := fmt.Sprint(order) + mu.Unlock() + if gotOrder != "[prepare unstable unstable finish]" { + t.Fatalf("execution order = %s", gotOrder) + } + events, err := WorkflowEvents(ctx, store, workflowID) + if err != nil { + t.Fatal(err) + } + var retried, succeeded bool + for _, event := range events { + retried = retried || event.Event == "automatic_retry_scheduled" + succeeded = succeeded || event.Event == "workflow_succeeded" + } + if !retried || !succeeded { + t.Fatalf("history lacks retry/success: %#v", events) + } +} + +func TestWorkflowExperimentsPostgresMatrixCell(t *testing.T) { + conn := os.Getenv("HG_TEST_PG") + if conn == "" { + t.Skip("HG_TEST_PG not set") + } + store, err := headgatepgx.Connect(t.Context(), conn) + if err != nil { + t.Fatal(err) + } + defer store.Close() + runWorkflowMatrixCell(t, store, "pg") +} + +func TestWorkflowExperimentsRedisMatrixCell(t *testing.T) { + url := os.Getenv("HG_TEST_REDIS") + if url == "" { + t.Skip("HG_TEST_REDIS not set") + } + store, err := headgateredis.Connect(url, "workflow-matrix-go-"+strconv.FormatInt(time.Now().UnixNano(), 10)) + if err != nil { + t.Fatal(err) + } + defer store.Close() + runWorkflowMatrixCell(t, store, "redis") +} diff --git a/go/headgateworkflow/live_mysql_test.go b/go/headgateworkflow/live_mysql_test.go new file mode 100644 index 0000000..d7c6f8e --- /dev/null +++ b/go/headgateworkflow/live_mysql_test.go @@ -0,0 +1,20 @@ +package headgateworkflow + +import ( + "os" + "testing" + + "github.com/mujhtech/headgate/go/driver/headgatemysql" +) + +func TestWorkflowExperimentsMySQLMatrixCell(t *testing.T) { + url := os.Getenv("HG_TEST_MYSQL") + if url == "" { + t.Skip("HG_TEST_MYSQL not set") + } + store, err := headgatemysql.Connect(url) + if err != nil { + t.Fatal(err) + } + runWorkflowMatrixCell(t, store, "mysql") +} diff --git a/go/headgateworkflow/workflow.go b/go/headgateworkflow/workflow.go index 0f92247..b4464ea 100644 --- a/go/headgateworkflow/workflow.go +++ b/go/headgateworkflow/workflow.go @@ -3,39 +3,285 @@ package headgateworkflow import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "math" + "strings" "sync" "time" + "cel.dev/cel-go/cel" headgate "github.com/mujhtech/headgate/go" ) const ( - CoordinatorKind = "headgate:workflow" - defaultRetention = int64((7 * 24 * time.Hour) / time.Millisecond) - maxWorkflowNodes = headgate.MaxEnqueueBatchSize - 1 - maxWorkflowEdges = 10_000 - workflowWorkers = 16 + CoordinatorKind = "headgate:workflow" + defaultRetention = int64((7 * 24 * time.Hour) / time.Millisecond) + maxWorkflowNodes = headgate.MaxEnqueueBatchSize - 1 + maxWorkflowEdges = 10_000 + maxWorkflowEvents = 256 + workflowWorkers = 16 + maxSignalPayload = 64 * 1024 + maxSignalSource = 16 * 1024 ) type draftNode struct { - name string - env headgate.Envelope - deps []string + name string + kind workflowNodeKind + env headgate.Envelope + signal string + wakeAtMs int64 + delayMs int64 + childWorkflowID string + condition string + deps []string } +type workflowNodeKind string + +const ( + workflowTask workflowNodeKind = "task" + workflowSignal workflowNodeKind = "signal" + workflowTimer workflowNodeKind = "timer" + workflowChild workflowNodeKind = "child_workflow" + workflowCondition workflowNodeKind = "condition" +) + // Workflow is a validated DAG builder. Prepare returns one atomic enqueue batch: the // durable coordinator followed by every child in pending state. type Workflow struct { - id string + id string + nodes []draftNode + coordinatorQueue string + retentionMs int64 + failedSubgraphRetry bool + retryPolicy *WorkflowRetryPolicy +} + +type WorkflowRetryPolicy struct { + MaxGenerations uint32 `json:"max_generations"` + BackoffMs int64 `json:"backoff_ms"` +} + +// WorkflowGraft is a revision-checked set of ordinary tasks to add to a running +// workflow. Prepare returns one atomic batch containing the graft receipt and tasks. +type WorkflowGraft struct { + workflowID string + expectedRevision uint64 nodes []draftNode - coordinatorQueue string + queue string retentionMs int64 } +// PrepareBundle validates the complete child graph and returns one atomic enqueue +// batch. Every child link must name another member of the bundle. +func PrepareBundle(workflows ...*Workflow) ([]headgate.Envelope, error) { + if len(workflows) == 0 { + return nil, errors.New("headgate workflow: bundle must contain at least one workflow") + } + ids := make(map[string]struct{}, len(workflows)) + for _, workflow := range workflows { + if workflow == nil || workflow.id == "" { + return nil, errors.New("headgate workflow: bundle ids must be non-empty and unique") + } + if _, exists := ids[workflow.id]; exists { + return nil, errors.New("headgate workflow: bundle ids must be non-empty and unique") + } + ids[workflow.id] = struct{}{} + } + degree := make(map[string]int, len(workflows)) + outgoing := make(map[string][]string) + for _, workflow := range workflows { + children := make(map[string]struct{}) + for _, node := range workflow.nodes { + if node.kind != workflowChild { + continue + } + if _, exists := ids[node.childWorkflowID]; !exists { + return nil, fmt.Errorf("headgate workflow: atomic bundle is missing child %q", node.childWorkflowID) + } + if _, duplicate := children[node.childWorkflowID]; !duplicate { + children[node.childWorkflowID] = struct{}{} + degree[node.childWorkflowID]++ + outgoing[workflow.id] = append(outgoing[workflow.id], node.childWorkflowID) + } + } + } + ready := make([]string, 0) + for id := range ids { + if degree[id] == 0 { + ready = append(ready, id) + } + } + visited := 0 + for len(ready) != 0 { + id := ready[0] + ready = ready[1:] + visited++ + for _, child := range outgoing[id] { + degree[child]-- + if degree[child] == 0 { + ready = append(ready, child) + } + } + } + if visited != len(workflows) { + return nil, errors.New("headgate workflow: cross-workflow child graph contains a cycle") + } + batch := make([]headgate.Envelope, 0) + for _, workflow := range workflows { + prepared, err := workflow.Prepare() + if err != nil { + return nil, err + } + batch = append(batch, prepared...) + if len(batch) > headgate.MaxEnqueueBatchSize { + return nil, fmt.Errorf("headgate workflow: bundle must contain at most %d jobs", headgate.MaxEnqueueBatchSize) + } + } + return batch, nil +} + +func NewGraft(workflowID string, expectedRevision uint64) *WorkflowGraft { + return &WorkflowGraft{ + workflowID: workflowID, expectedRevision: expectedRevision, + queue: "headgate-workflow", retentionMs: defaultRetention, + } +} + +func (g *WorkflowGraft) Queue(queue string) *WorkflowGraft { + g.queue = queue + return g +} + +func (g *WorkflowGraft) Retention(d time.Duration) error { + if d < time.Millisecond { + return errors.New("headgate workflow: graft retention must be at least 1ms") + } + g.retentionMs = d.Milliseconds() + return nil +} + +func (g *WorkflowGraft) Add(name string, env headgate.Envelope, deps ...string) *WorkflowGraft { + g.nodes = append(g.nodes, draftNode{name: name, kind: workflowTask, env: env, deps: append([]string{}, deps...)}) + return g +} + +func (g *WorkflowGraft) Prepare() ([]headgate.Envelope, error) { + if g.workflowID == "" { + return nil, errors.New("headgate workflow: workflow id must not be empty") + } + if g.expectedRevision == 0 { + return nil, errors.New("headgate workflow: graft expected revision must be at least 1") + } + if len(g.nodes) == 0 || len(g.nodes) > maxWorkflowNodes { + return nil, fmt.Errorf("headgate workflow: graft must contain 1-%d tasks", maxWorkflowNodes) + } + nextRevision := g.expectedRevision + 1 + if nextRevision == 0 { + return nil, errors.New("headgate workflow: graft revision would overflow") + } + names := make(map[string]struct{}, len(g.nodes)) + specs := make([]nodeSpec, 0, len(g.nodes)) + children := make([]headgate.Envelope, 0, len(g.nodes)) + for _, node := range g.nodes { + if node.name == "" || len(node.name) > 128 { + return nil, errors.New("headgate workflow: graft task names must be non-empty and at most 128 bytes") + } + if _, exists := names[node.name]; exists { + return nil, errors.New("headgate workflow: graft task names must be unique") + } + names[node.name] = struct{}{} + env := node.env + if env.ID == "" { + env.ID = fmt.Sprintf("%s:g%d:%s", g.workflowID, nextRevision, node.name) + } + env.Pending = true + env.ScheduledAtMs = 0 + if env.RetentionMs < g.retentionMs { + env.RetentionMs = g.retentionMs + } + if env.Fingerprint == "" { + env.Fingerprint = headgate.Fingerprint(env.Kind, env.Payload) + } + specs = append(specs, nodeSpec{Name: node.name, JobID: env.ID, Deps: node.deps, Kind: workflowTask}) + children = append(children, env) + } + if err := validateGraftNodes(specs); err != nil { + return nil, err + } + receiptArgs := GraftArgs{WorkflowID: g.workflowID, ExpectedRevision: g.expectedRevision, Nodes: specs} + payload, err := json.Marshal(receiptArgs) + if err != nil { + return nil, err + } + receipt := headgate.Envelope{ + ID: graftReceiptID(g.workflowID, nextRevision), Kind: GraftKind, SchemaVersion: 1, + Payload: payload, Queue: g.queue, Pending: true, RetentionMs: g.retentionMs, + Fingerprint: headgate.Fingerprint(GraftKind, payload), + } + batch := append([]headgate.Envelope{receipt}, children...) + if err := headgate.ValidateEnqueue(batch); err != nil { + return nil, err + } + return batch, nil +} + +func validateGraftNodes(nodes []nodeSpec) error { + names := make(map[string]struct{}, len(nodes)) + for _, node := range nodes { + names[node.Name] = struct{}{} + } + degree := make(map[string]int, len(nodes)) + outgoing := make(map[string][]string) + edges := 0 + for _, node := range nodes { + seen := make(map[string]struct{}, len(node.Deps)) + for _, dep := range node.Deps { + edges++ + if _, exists := seen[dep]; exists { + return fmt.Errorf("headgate workflow: graft task %q repeats dependency %q", node.Name, dep) + } + seen[dep] = struct{}{} + if dep == node.Name { + return fmt.Errorf("headgate workflow: graft task %q depends on itself", node.Name) + } + if _, local := names[dep]; local { + degree[node.Name]++ + outgoing[dep] = append(outgoing[dep], node.Name) + } + } + } + if edges > maxWorkflowEdges { + return fmt.Errorf("headgate workflow: graft must contain at most %d dependency edges", maxWorkflowEdges) + } + ready := make([]string, 0, len(nodes)) + for name := range names { + if degree[name] == 0 { + ready = append(ready, name) + } + } + visited := 0 + for len(ready) > 0 { + name := ready[0] + ready = ready[1:] + visited++ + for _, child := range outgoing[name] { + degree[child]-- + if degree[child] == 0 { + ready = append(ready, child) + } + } + } + if visited != len(nodes) { + return errors.New("headgate workflow: graft dependency graph contains a cycle") + } + return nil +} + func New(id string) *Workflow { return &Workflow{id: id, coordinatorQueue: "headgate-workflow", retentionMs: defaultRetention} } @@ -53,8 +299,67 @@ func (w *Workflow) Retention(d time.Duration) error { return nil } +// EnableFailedSubgraphRetry retains blocked pending jobs so a failed generation can be +// reopened without rerunning successful ancestors. +func (w *Workflow) EnableFailedSubgraphRetry() *Workflow { + w.failedSubgraphRetry = true + return w +} + +// AutomaticRetry enables failed-subgraph retry after a store-timed backoff. The +// generation limit includes the initial run. +func (w *Workflow) AutomaticRetry(maxGenerations uint32, backoff time.Duration) error { + if maxGenerations < 2 || backoff < time.Millisecond { + return errors.New("headgate workflow: automatic retry requires at least 2 generations and 1ms backoff") + } + w.failedSubgraphRetry = true + w.retryPolicy = &WorkflowRetryPolicy{MaxGenerations: maxGenerations, BackoffMs: backoff.Milliseconds()} + return nil +} + func (w *Workflow) Add(name string, env headgate.Envelope, deps ...string) *Workflow { - w.nodes = append(w.nodes, draftNode{name: name, env: env, deps: append([]string{}, deps...)}) + w.nodes = append(w.nodes, draftNode{name: name, kind: workflowTask, env: env, deps: append([]string{}, deps...)}) + return w +} + +// AddSignal adds a durable, buffered workflow signal node. Emission may happen before +// its dependencies complete; the coordinator consumes it only when the node is eligible. +func (w *Workflow) AddSignal(name, signal string, deps ...string) *Workflow { + w.nodes = append(w.nodes, draftNode{name: name, kind: workflowSignal, signal: signal, deps: append([]string{}, deps...)}) + return w +} + +// AddTimerAt adds an absolute store-time timer. The ordinary scheduled-job promoter +// supplies the clock; worker clock skew cannot fire the timer early or late. +func (w *Workflow) AddTimerAt(name string, wakeAtMs int64, deps ...string) *Workflow { + w.nodes = append(w.nodes, draftNode{name: name, kind: workflowTimer, wakeAtMs: wakeAtMs, deps: append([]string{}, deps...)}) + return w +} + +// AddTimerAfter adds a relative timer anchored to the latest dependency finalization +// timestamp, which the coordinator records before scheduling the internal job. +func (w *Workflow) AddTimerAfter(name string, delay time.Duration, deps ...string) error { + if delay < time.Millisecond { + return errors.New("headgate workflow: timer delay must be at least 1ms") + } + w.nodes = append(w.nodes, draftNode{name: name, kind: workflowTimer, delayMs: delay.Milliseconds(), deps: append([]string{}, deps...)}) + return nil +} + +// AddChild adds an explicit child-workflow link. The child is enqueued separately; +// this node mirrors its coordinator's terminal state into the parent. +func (w *Workflow) AddChild(name, workflowID string, deps ...string) *Workflow { + w.nodes = append(w.nodes, draftNode{name: name, kind: workflowChild, childWorkflowID: workflowID, deps: append([]string{}, deps...)}) + return w +} + +// AddCondition waits until a CEL expression over revision, generation, completed, +// and states evaluates to true. +func (w *Workflow) AddCondition(name, expression string, deps ...string) *Workflow { + w.nodes = append(w.nodes, draftNode{ + name: name, kind: workflowCondition, condition: expression, + deps: append([]string{}, deps...), + }) return w } @@ -72,21 +377,75 @@ func (w *Workflow) Prepare() ([]headgate.Envelope, error) { children := make([]headgate.Envelope, 0, len(w.nodes)) for _, node := range w.nodes { env := node.env + if node.kind == workflowSignal { + payload, err := json.Marshal(SignalArgs{WorkflowID: w.id, Signal: node.signal}) + if err != nil { + return nil, err + } + env = headgate.Envelope{ + Kind: SignalKind, SchemaVersion: 1, Payload: payload, + Queue: w.coordinatorQueue, Fingerprint: headgate.Fingerprint(SignalKind, payload), + } + } + if node.kind == workflowTimer { + payload, err := json.Marshal(TimerArgs{WorkflowID: w.id, WakeAtMs: node.wakeAtMs, DelayMs: node.delayMs}) + if err != nil { + return nil, err + } + env = headgate.Envelope{ + Kind: TimerKind, SchemaVersion: 1, Payload: payload, ScheduledAtMs: node.wakeAtMs, + Queue: w.coordinatorQueue, Fingerprint: headgate.Fingerprint(TimerKind, payload), + } + } + if node.kind == workflowChild { + if node.childWorkflowID == w.id { + return nil, errors.New("headgate workflow: workflow cannot contain itself as a child") + } + payload, err := json.Marshal(ChildWorkflowArgs{ParentWorkflowID: w.id, ChildWorkflowID: node.childWorkflowID}) + if err != nil { + return nil, err + } + env = headgate.Envelope{ + Kind: ChildWorkflowKind, SchemaVersion: 1, Payload: payload, + Queue: w.coordinatorQueue, Fingerprint: headgate.Fingerprint(ChildWorkflowKind, payload), + } + } + if node.kind == workflowCondition { + payload, err := json.Marshal(ConditionArgs{WorkflowID: w.id, Expression: node.condition}) + if err != nil { + return nil, err + } + env = headgate.Envelope{ + Kind: ConditionKind, SchemaVersion: 1, Payload: payload, + Queue: w.coordinatorQueue, Fingerprint: headgate.Fingerprint(ConditionKind, payload), + } + } if env.ID == "" { env.ID = w.id + ":" + node.name } if env.RetentionMs < w.retentionMs { env.RetentionMs = w.retentionMs } - env.Pending = true - env.ScheduledAtMs = 0 + if node.kind == workflowTimer && node.wakeAtMs > 0 { + env.Pending = false + } else { + env.Pending = true + env.ScheduledAtMs = 0 + } if env.Fingerprint == "" { env.Fingerprint = headgate.Fingerprint(env.Kind, env.Payload) } - specs = append(specs, nodeSpec{Name: node.name, JobID: env.ID, Deps: node.deps}) + specs = append(specs, nodeSpec{ + Name: node.name, JobID: env.ID, Deps: node.deps, Kind: node.kind, + Signal: node.signal, WakeAtMs: node.wakeAtMs, DelayMs: node.delayMs, + ChildWorkflowID: node.childWorkflowID, Condition: node.condition, + }) children = append(children, env) } - task := CoordinatorArgs{WorkflowID: w.id, Nodes: specs} + task := CoordinatorArgs{ + WorkflowID: w.id, Nodes: specs, FailedSubgraphRetry: w.failedSubgraphRetry, + RetryPolicy: w.retryPolicy, + } payload, err := json.Marshal(task) if err != nil { return nil, err @@ -96,141 +455,1530 @@ func (w *Workflow) Prepare() ([]headgate.Envelope, error) { Payload: payload, Queue: w.coordinatorQueue, RetentionMs: w.retentionMs, Fingerprint: headgate.Fingerprint(CoordinatorKind, payload), } - batch := append([]headgate.Envelope{coordinator}, children...) - if err := headgate.ValidateEnqueue(batch); err != nil { - return nil, err + batch := append([]headgate.Envelope{coordinator}, children...) + if err := headgate.ValidateEnqueue(batch); err != nil { + return nil, err + } + return batch, nil +} + +func validateGraph(nodes []draftNode) error { + if len(nodes) > maxWorkflowNodes { + return fmt.Errorf("headgate workflow: must contain at most %d tasks", maxWorkflowNodes) + } + names := make(map[string]struct{}, len(nodes)) + edges := 0 + for _, node := range nodes { + if node.name == "" { + return errors.New("headgate workflow: task names must not be empty") + } + if _, exists := names[node.name]; exists { + return fmt.Errorf("headgate workflow: task name %q is repeated", node.name) + } + if len(node.name) > 128 { + return fmt.Errorf("headgate workflow: task name %q exceeds 128 bytes", node.name) + } + if node.kind == workflowSignal && node.signal == "" { + return fmt.Errorf("headgate workflow: signal node %q has an empty signal", node.name) + } + if node.kind == workflowTimer && !validTimerSchedule(node.wakeAtMs, node.delayMs) { + return fmt.Errorf("headgate workflow: timer node %q must have exactly one positive schedule", node.name) + } + if node.kind == workflowTimer && node.delayMs > 0 && len(node.deps) == 0 { + return fmt.Errorf("headgate workflow: relative timer %q requires at least one dependency", node.name) + } + if node.kind == workflowChild && node.childWorkflowID == "" { + return fmt.Errorf("headgate workflow: child node %q has an empty workflow id", node.name) + } + if node.kind == workflowCondition { + if err := validateCondition(node.condition); err != nil { + return fmt.Errorf("headgate workflow: condition node %q: %w", node.name, err) + } + } + edges += len(node.deps) + names[node.name] = struct{}{} + } + if edges > maxWorkflowEdges { + return fmt.Errorf("headgate workflow: must contain at most %d dependency edges", maxWorkflowEdges) + } + degree := make(map[string]int, len(nodes)) + outgoing := make(map[string][]string) + for _, node := range nodes { + seen := map[string]struct{}{} + for _, dep := range node.deps { + if _, exists := names[dep]; !exists { + return fmt.Errorf("headgate workflow: task %q depends on missing task %q", node.name, dep) + } + if _, exists := seen[dep]; exists { + return fmt.Errorf("headgate workflow: task %q repeats dependency %q", node.name, dep) + } + seen[dep] = struct{}{} + degree[node.name]++ + outgoing[dep] = append(outgoing[dep], node.name) + } + } + ready := make([]string, 0, len(nodes)) + for name := range names { + if degree[name] == 0 { + ready = append(ready, name) + } + } + visited := 0 + for len(ready) > 0 { + name := ready[0] + ready = ready[1:] + visited++ + for _, child := range outgoing[name] { + degree[child]-- + if degree[child] == 0 { + ready = append(ready, child) + } + } + } + if visited != len(nodes) { + return errors.New("headgate workflow: dependency graph contains a cycle") + } + return nil +} + +type nodeSpec struct { + Name string `json:"name"` + JobID string `json:"job_id"` + Deps []string `json:"deps"` + Kind workflowNodeKind `json:"kind,omitempty"` + Signal string `json:"signal,omitempty"` + WakeAtMs int64 `json:"wake_at_ms,omitempty"` + DelayMs int64 `json:"delay_ms,omitempty"` + ChildWorkflowID string `json:"child_workflow_id,omitempty"` + Condition string `json:"condition,omitempty"` +} + +type CoordinatorArgs struct { + WorkflowID string `json:"workflow_id"` + Nodes []nodeSpec `json:"nodes"` + FailedSubgraphRetry bool `json:"failed_subgraph_retry,omitempty"` + RetryPolicy *WorkflowRetryPolicy `json:"retry_policy,omitempty"` +} + +func (CoordinatorArgs) Kind() string { return CoordinatorKind } + +const SignalKind = "headgate:workflow-signal" +const TimerKind = "headgate:workflow-timer" +const ChildWorkflowKind = "headgate:workflow-child" +const GraftKind = "headgate:workflow-graft" +const RetryKind = "headgate:workflow-retry" +const ConditionKind = "headgate:workflow-condition" + +type SignalArgs struct { + WorkflowID string `json:"workflow_id"` + Signal string `json:"signal"` +} + +func (SignalArgs) Kind() string { return SignalKind } + +type TimerArgs struct { + WorkflowID string `json:"workflow_id"` + WakeAtMs int64 `json:"wake_at_ms"` + DelayMs int64 `json:"delay_ms"` +} + +func (TimerArgs) Kind() string { return TimerKind } + +type ChildWorkflowArgs struct { + ParentWorkflowID string `json:"parent_workflow_id"` + ChildWorkflowID string `json:"child_workflow_id"` +} + +type ConditionArgs struct { + WorkflowID string `json:"workflow_id"` + Expression string `json:"expression"` +} + +func (ConditionArgs) Kind() string { return ConditionKind } + +func (ChildWorkflowArgs) Kind() string { return ChildWorkflowKind } + +type GraftArgs struct { + WorkflowID string `json:"workflow_id"` + ExpectedRevision uint64 `json:"expected_revision"` + Nodes []nodeSpec `json:"nodes"` +} + +func (GraftArgs) Kind() string { return GraftKind } + +type RetryArgs struct { + WorkflowID string `json:"workflow_id"` + ExpectedRevision uint64 `json:"expected_revision"` +} + +func (RetryArgs) Kind() string { return RetryKind } + +func graftReceiptID(workflowID string, revision uint64) string { + return fmt.Sprintf("%s:graft:%d", workflowID, revision) +} + +func retryReceiptID(workflowID string, revision uint64) string { + return fmt.Sprintf("%s:retry:%d", workflowID, revision) +} + +type SignalReceipt struct { + Matched int `json:"matched"` + Promoted int `json:"promoted"` + Inserted bool `json:"inserted"` + Emission WorkflowSignal `json:"emission"` +} + +type SignalEmission struct { + Signal string + IdempotencyKey string + Payload json.RawMessage + Source json.RawMessage +} + +type WorkflowSignal struct { + ID uint64 `json:"id"` + Signal string `json:"signal"` + IdempotencyKey string `json:"idempotency_key"` + Payload json.RawMessage `json:"payload"` + Source json.RawMessage `json:"source"` + RecordedAtMs int64 `json:"recorded_at_ms"` +} + +type RetryReceipt struct { + Revision uint64 + Generation uint32 +} + +type WorkflowRecovery struct { + Node string + Payload []byte + SchemaVersion uint32 + ReleaseQuarantine bool +} + +type CancelReceipt struct { + Workflows int `json:"workflows"` + Jobs int `json:"jobs"` +} + +// CancelWorkflow cancels the workflow and optionally all linked children. Traversal +// and point reads are bounded by the workflow node limit. +func CancelWorkflow( + ctx context.Context, + inspect headgate.InspectStore, + workflowID string, + propagateChildren bool, +) (CancelReceipt, error) { + if workflowID == "" { + return CancelReceipt{}, errors.New("headgate workflow: workflow id must not be empty") + } + pending := []string{workflowID} + visited := make(map[string]struct{}) + receipt := CancelReceipt{} + for len(pending) != 0 { + current := pending[0] + pending = pending[1:] + if _, exists := visited[current]; exists { + continue + } + visited[current] = struct{}{} + if len(visited) > maxWorkflowNodes { + return CancelReceipt{}, errors.New("headgate workflow: cancellation exceeds the bounded nested-workflow limit") + } + coordinatorID := current + ":coordinator" + coordinator, err := inspect.GetJob(ctx, coordinatorID, true) + if err != nil { + return CancelReceipt{}, err + } + if coordinator == nil { + return CancelReceipt{}, fmt.Errorf("headgate workflow: workflow %q was not found", current) + } + var args CoordinatorArgs + if err := json.Unmarshal(coordinator.Payload, &args); err != nil { + return CancelReceipt{}, fmt.Errorf("headgate workflow: invalid coordinator: %w", err) + } + if propagateChildren { + for _, node := range args.Nodes { + if node.ChildWorkflowID != "" { + pending = append(pending, node.ChildWorkflowID) + } + } + } + ids := make([]string, 0, len(args.Nodes)+1) + for _, node := range args.Nodes { + ids = append(ids, node.JobID) + } + ids = append(ids, coordinatorID) + for _, id := range ids { + job, err := inspect.GetJob(ctx, id, false) + if err != nil { + return CancelReceipt{}, err + } + if job != nil && cancellableWorkflowState(job.State) { + if err := inspect.OperatorCancel(ctx, id); err != nil { + return CancelReceipt{}, err + } + receipt.Jobs++ + } + } + } + receipt.Workflows = len(visited) + return receipt, nil +} + +func cancellableWorkflowState(state string) bool { + switch state { + case "pending", "scheduled", "available", "running", "retryable": + return true + default: + return false + } +} + +type WorkflowEvent struct { + Sequence uint64 `json:"sequence"` + Event string `json:"event"` + Node string `json:"node,omitempty"` + Revision uint64 `json:"revision"` + Generation uint32 `json:"generation"` + AtMs *int64 `json:"at_ms,omitempty"` +} + +// WorkflowNodeKind identifies the durable role a node plays in a workflow graph. +type WorkflowNodeKind string + +const ( + WorkflowNodeTask WorkflowNodeKind = "task" + WorkflowNodeSignal WorkflowNodeKind = "signal" + WorkflowNodeTimer WorkflowNodeKind = "timer" + WorkflowNodeChildWorkflow WorkflowNodeKind = "child_workflow" + WorkflowNodeCondition WorkflowNodeKind = "condition" +) + +// WorkflowNode is one node in an inspected graph. Dependencies and Dependents contain +// node names; JobID identifies the underlying Headgate job. +type WorkflowNode struct { + Name string `json:"name"` + JobID string `json:"job_id"` + Kind WorkflowNodeKind `json:"kind"` + JobKind string `json:"job_kind"` + State string `json:"state"` + Dependencies []string `json:"dependencies"` + Dependents []string `json:"dependents"` + Signal string `json:"signal,omitempty"` + WakeAtMs *int64 `json:"wake_at_ms,omitempty"` + DelayMs *int64 `json:"delay_ms,omitempty"` + ChildWorkflowID string `json:"child_workflow_id,omitempty"` + Condition string `json:"condition,omitempty"` + CompletedAtMs *int64 `json:"completed_at_ms,omitempty"` +} + +// WorkflowSnapshot is a bounded point-in-time view of the complete accepted graph, +// including additive grafts accepted in later revisions. +type WorkflowSnapshot struct { + WorkflowID string `json:"workflow_id"` + CoordinatorJobID string `json:"coordinator_job_id"` + CoordinatorState string `json:"coordinator_state"` + Revision uint64 `json:"revision"` + Generation uint32 `json:"generation"` + Failed bool `json:"failed"` + FailedSubgraphRetry bool `json:"failed_subgraph_retry"` + RetryPolicy *WorkflowRetryPolicy `json:"retry_policy,omitempty"` + Nodes []WorkflowNode `json:"nodes"` +} + +// WorkflowSummary is one coordinator entry returned by ListWorkflows. +type WorkflowSummary struct { + WorkflowID string `json:"workflow_id"` + CoordinatorJobID string `json:"coordinator_job_id"` + State string `json:"state"` + EnqueuedAtMs int64 `json:"enqueued_at_ms"` + ScheduledAtMs int64 `json:"scheduled_at_ms"` + FinalizedAtMs *int64 `json:"finalized_at_ms,omitempty"` +} + +// WorkflowPage is one bounded page of workflow coordinators. +type WorkflowPage struct { + Workflows []WorkflowSummary `json:"workflows"` + NextCursor string `json:"next_cursor,omitempty"` +} + +// ListWorkflows lists workflow coordinators without loading every graph. Use +// InspectWorkflow for a selected execution that needs node-level detail. +func ListWorkflows(ctx context.Context, inspect headgate.InspectStore, cursor string, limit uint32) (WorkflowPage, error) { + if limit == 0 || limit > 200 { + return WorkflowPage{}, errors.New("headgate workflow: list limit must be between 1 and 200") + } + page, err := inspect.ListJobs(ctx, headgate.JobFilter{Kind: headgate.Ptr(CoordinatorKind)}, cursor, limit) + if err != nil { + return WorkflowPage{}, err + } + workflows := make([]WorkflowSummary, 0, len(page.Jobs)) + for _, job := range page.Jobs { + workflowID := strings.TrimSuffix(job.ID, ":coordinator") + workflows = append(workflows, WorkflowSummary{ + WorkflowID: workflowID, CoordinatorJobID: job.ID, State: job.State, + EnqueuedAtMs: job.EnqueuedAtMs, ScheduledAtMs: job.ScheduledAtMs, + FinalizedAtMs: job.FinalizedAtMs, + }) + } + return WorkflowPage{Workflows: workflows, NextCursor: page.NextCursor}, nil +} + +// Node returns a graph node by its workflow-local name. +func (s *WorkflowSnapshot) Node(name string) *WorkflowNode { + for i := range s.Nodes { + if s.Nodes[i].Name == name { + return &s.Nodes[i] + } + } + return nil +} + +// Dependencies returns the named node's immediate prerequisites. +func (s *WorkflowSnapshot) Dependencies(name string) ([]WorkflowNode, bool) { + node := s.Node(name) + if node == nil { + return nil, false + } + result := make([]WorkflowNode, 0, len(node.Dependencies)) + for _, dependency := range node.Dependencies { + if found := s.Node(dependency); found != nil { + result = append(result, *found) + } + } + return result, true +} + +// Dependents returns the nodes that immediately depend on the named node. +func (s *WorkflowSnapshot) Dependents(name string) ([]WorkflowNode, bool) { + node := s.Node(name) + if node == nil { + return nil, false + } + result := make([]WorkflowNode, 0, len(node.Dependents)) + for _, dependent := range node.Dependents { + if found := s.Node(dependent); found != nil { + result = append(result, *found) + } + } + return result, true +} + +// InspectWorkflow returns graph topology and live execution state without exposing +// application task payloads. +func InspectWorkflow(ctx context.Context, inspect headgate.InspectStore, workflowID string) (WorkflowSnapshot, error) { + if workflowID == "" { + return WorkflowSnapshot{}, errors.New("headgate workflow: workflow id must not be empty") + } + coordinatorID := workflowID + ":coordinator" + coordinator, err := inspect.GetJob(ctx, coordinatorID, true) + if err != nil { + return WorkflowSnapshot{}, err + } + if coordinator == nil { + return WorkflowSnapshot{}, fmt.Errorf("headgate workflow: workflow %q was not found", workflowID) + } + var base CoordinatorArgs + if err := json.Unmarshal(coordinator.Payload, &base); err != nil { + return WorkflowSnapshot{}, fmt.Errorf("headgate workflow: invalid coordinator: %w", err) + } + cursor, err := loadWorkflowCursor(ctx, inspect, coordinatorID, coordinator.State) + if err != nil { + return WorkflowSnapshot{}, err + } + effective := effectiveWorkflow(base, cursor) + dependents := make(map[string][]string, len(effective.Nodes)) + for _, node := range effective.Nodes { + for _, dependency := range node.Deps { + dependents[dependency] = append(dependents[dependency], node.Name) + } + } + completed := make(map[string]struct{}, len(cursor.Completed)) + for _, name := range cursor.Completed { + completed[name] = struct{}{} + } + nodes := make([]WorkflowNode, len(effective.Nodes)) + semaphore := make(chan struct{}, workflowWorkers) + var reads sync.WaitGroup + var errorMu sync.Mutex + var firstError error + for index, node := range effective.Nodes { + semaphore <- struct{}{} + reads.Add(1) + go func() { + defer reads.Done() + defer func() { <-semaphore }() + job, err := inspect.GetJob(ctx, node.JobID, false) + if err != nil { + errorMu.Lock() + if firstError == nil { + firstError = err + } + errorMu.Unlock() + return + } + state, jobKind := "missing", "" + if job != nil { + state, jobKind = job.State, job.Kind + } else if _, ok := completed[node.Name]; ok { + state = "completed" + } + var wakeAtMs, delayMs, completedAtMs *int64 + if node.Kind == workflowTimer { + if node.WakeAtMs != 0 { + value := node.WakeAtMs + wakeAtMs = &value + } + if node.DelayMs != 0 { + value := node.DelayMs + delayMs = &value + } + } + if value, ok := cursor.CompletedAtMs[node.Name]; ok { + completedAtMs = &value + } + nodes[index] = WorkflowNode{ + Name: node.Name, JobID: node.JobID, Kind: publicWorkflowNodeKind(node.Kind), + JobKind: jobKind, State: state, Dependencies: workflowNodeNames(node.Deps), + Dependents: workflowNodeNames(dependents[node.Name]), Signal: node.Signal, + WakeAtMs: wakeAtMs, DelayMs: delayMs, ChildWorkflowID: node.ChildWorkflowID, + Condition: node.Condition, CompletedAtMs: completedAtMs, + } + }() + } + reads.Wait() + if firstError != nil { + return WorkflowSnapshot{}, firstError + } + return WorkflowSnapshot{ + WorkflowID: workflowID, CoordinatorJobID: coordinatorID, CoordinatorState: coordinator.State, + Revision: cursor.Revision, Generation: cursor.Generation, Failed: cursor.Failed, + FailedSubgraphRetry: base.FailedSubgraphRetry, RetryPolicy: base.RetryPolicy, Nodes: nodes, + }, nil +} + +func workflowNodeNames(names []string) []string { + result := make([]string, len(names)) + copy(result, names) + return result +} + +func publicWorkflowNodeKind(kind workflowNodeKind) WorkflowNodeKind { + switch kind { + case workflowSignal: + return WorkflowNodeSignal + case workflowTimer: + return WorkflowNodeTimer + case workflowChild: + return WorkflowNodeChildWorkflow + case workflowCondition: + return WorkflowNodeCondition + default: + return WorkflowNodeTask + } +} + +// GetWorkflowNode returns one node by its workflow-local name. +func GetWorkflowNode(ctx context.Context, inspect headgate.InspectStore, workflowID, node string) (WorkflowNode, error) { + snapshot, err := InspectWorkflow(ctx, inspect, workflowID) + if err != nil { + return WorkflowNode{}, err + } + found := snapshot.Node(node) + if found == nil { + return WorkflowNode{}, fmt.Errorf("headgate workflow: workflow node %q was not found", node) + } + return *found, nil +} + +// WorkflowDependencies returns a node's immediate prerequisites. +func WorkflowDependencies(ctx context.Context, inspect headgate.InspectStore, workflowID, node string) ([]WorkflowNode, error) { + snapshot, err := InspectWorkflow(ctx, inspect, workflowID) + if err != nil { + return nil, err + } + dependencies, ok := snapshot.Dependencies(node) + if !ok { + return nil, fmt.Errorf("headgate workflow: workflow node %q was not found", node) + } + return dependencies, nil +} + +// WorkflowDependents returns nodes that immediately depend on the named node. +func WorkflowDependents(ctx context.Context, inspect headgate.InspectStore, workflowID, node string) ([]WorkflowNode, error) { + snapshot, err := InspectWorkflow(ctx, inspect, workflowID) + if err != nil { + return nil, err + } + dependents, ok := snapshot.Dependents(node) + if !ok { + return nil, fmt.Errorf("headgate workflow: workflow node %q was not found", node) + } + return dependents, nil +} + +func loadWorkflowCursor(ctx context.Context, inspect headgate.InspectStore, coordinatorID, coordinatorState string) (workflowCursor, error) { + cursor := workflowCursor{Revision: 1, Generation: 1} + checkpointStore, ok := inspect.(headgate.CheckpointInspectStore) + if !ok { + return workflowCursor{}, errors.New("headgate workflow: inspection requires checkpoint inspection support") + } + checkpoint, err := checkpointStore.GetJobCheckpoint(ctx, coordinatorID) + if err != nil { + return workflowCursor{}, err + } + if checkpoint == nil { + return cursor, nil + } + if checkpoint.CursorStep != "" && checkpoint.CursorStep != "headgate:workflow-state" { + return workflowCursor{}, errors.New("headgate workflow: coordinator has no workflow-state checkpoint") + } + bytes := checkpoint.Cursor + if len(bytes) == 0 { + if outputStore, ok := inspect.(interface { + GetJobOutput(context.Context, string) (*headgate.JobOutput, error) + }); ok { + output, err := outputStore.GetJobOutput(ctx, coordinatorID) + if err != nil { + return workflowCursor{}, err + } + if output != nil { + bytes = output.Bytes + } + } + } + if len(bytes) == 0 { + if terminalWorkflowState(coordinatorState) { + return workflowCursor{}, errors.New("headgate workflow: terminal workflow has no durable coordinator output") + } + return cursor, nil + } + if err := json.Unmarshal(bytes, &cursor); err != nil { + return workflowCursor{}, fmt.Errorf("headgate workflow: invalid cursor: %w", err) + } + cursor.normalize() + return cursor, nil +} + +func terminalWorkflowState(state string) bool { + switch state { + case "completed", "archived", "cancelled", "quarantined", "undecodable": + return true + default: + return false + } +} + +// WorkflowEvents returns the bounded durable event history from the coordinator's +// fenced checkpoint. +func WorkflowEvents(ctx context.Context, inspect headgate.InspectStore, workflowID string) ([]WorkflowEvent, error) { + checkpointStore, ok := inspect.(headgate.CheckpointInspectStore) + if !ok { + return nil, errors.New("headgate workflow: history requires checkpoint inspection support") + } + checkpoint, err := checkpointStore.GetJobCheckpoint(ctx, workflowID+":coordinator") + if err != nil { + return nil, err + } + if checkpoint == nil { + return nil, fmt.Errorf("headgate workflow: workflow %q was not found", workflowID) + } + if checkpoint.CursorStep != "" && checkpoint.CursorStep != "headgate:workflow-state" { + return nil, errors.New("headgate workflow: coordinator has no workflow-state checkpoint") + } + bytes := checkpoint.Cursor + if len(bytes) == 0 { + outputStore, ok := inspect.(interface { + GetJobOutput(context.Context, string) (*headgate.JobOutput, error) + }) + if !ok { + return nil, errors.New("headgate workflow: history requires output inspection support") + } + output, err := outputStore.GetJobOutput(ctx, workflowID+":coordinator") + if err != nil { + return nil, err + } + if output == nil { + return nil, errors.New("headgate workflow: workflow has no durable history") + } + bytes = output.Bytes + } + var cursor workflowCursor + if err := json.Unmarshal(bytes, &cursor); err != nil { + return nil, fmt.Errorf("headgate workflow: invalid cursor: %w", err) + } + return append([]WorkflowEvent(nil), cursor.Events...), nil +} + +// RequestFailedSubgraphRetry durably enqueues the retry receipt before reopening the +// archived coordinator. Successful ancestors remain completed. +func RequestFailedSubgraphRetry( + ctx context.Context, + inspect headgate.InspectStore, + workflowID string, + expectedRevision uint64, +) (RetryReceipt, error) { + return RequestFailedSubgraphRetryWithRecovery(ctx, inspect, workflowID, expectedRevision, nil) +} + +func RequestFailedSubgraphRetryWithRecovery( + ctx context.Context, + inspect headgate.InspectStore, + workflowID string, + expectedRevision uint64, + recoveries []WorkflowRecovery, +) (RetryReceipt, error) { + if workflowID == "" || expectedRevision == 0 { + return RetryReceipt{}, errors.New("headgate workflow: workflow id and expected revision must be set") + } + coordinatorID := workflowID + ":coordinator" + coordinator, err := inspect.GetJob(ctx, coordinatorID, true) + if err != nil { + return RetryReceipt{}, err + } + if coordinator == nil { + return RetryReceipt{}, fmt.Errorf("headgate workflow: workflow %q was not found", workflowID) + } + var args CoordinatorArgs + if err := json.Unmarshal(coordinator.Payload, &args); err != nil { + return RetryReceipt{}, fmt.Errorf("headgate workflow: invalid coordinator: %w", err) + } + if !args.FailedSubgraphRetry { + return RetryReceipt{}, errors.New("headgate workflow: failed-subgraph retry was not enabled") + } + if coordinator.State != "archived" { + return RetryReceipt{}, fmt.Errorf("headgate workflow: retry requires an archived coordinator, found %q", coordinator.State) + } + nodes := make(map[string]nodeSpec, len(args.Nodes)) + for _, node := range args.Nodes { + nodes[node.Name] = node + } + seenRecovery := make(map[string]struct{}, len(recoveries)) + for _, recovery := range recoveries { + if _, duplicate := seenRecovery[recovery.Node]; duplicate { + return RetryReceipt{}, fmt.Errorf("headgate workflow: recovery repeats node %q", recovery.Node) + } + seenRecovery[recovery.Node] = struct{}{} + node, exists := nodes[recovery.Node] + if !exists { + return RetryReceipt{}, fmt.Errorf("headgate workflow: recovery names unknown node %q", recovery.Node) + } + job, err := inspect.GetJob(ctx, node.JobID, true) + if err != nil { + return RetryReceipt{}, err + } + if job == nil { + return RetryReceipt{}, fmt.Errorf("headgate workflow: node %q is missing", node.JobID) + } + switch job.State { + case "quarantined": + if !recovery.ReleaseQuarantine { + return RetryReceipt{}, fmt.Errorf("headgate workflow: node %q requires explicit quarantine release", recovery.Node) + } + if _, err := inspect.QuarantineRelease(ctx, job.Fingerprint); err != nil { + return RetryReceipt{}, err + } + case "undecodable": + if recovery.Payload == nil || recovery.SchemaVersion == 0 { + return RetryReceipt{}, fmt.Errorf("headgate workflow: undecodable node %q requires payload and schema_version", recovery.Node) + } + if err := inspect.EditPayload(ctx, node.JobID, recovery.Payload, recovery.SchemaVersion, + headgate.Fingerprint(job.Kind, recovery.Payload)); err != nil { + return RetryReceipt{}, err + } + if err := inspect.OperatorRetry(ctx, node.JobID); err != nil { + return RetryReceipt{}, err + } + case "archived", "cancelled": + case "available": + // A retry request may be replayed after recovery completed but before + // the coordinator was reopened. + default: + return RetryReceipt{}, fmt.Errorf("headgate workflow: node %q does not require recovery from %q", recovery.Node, job.State) + } + } + for _, node := range args.Nodes { + job, err := inspect.GetJob(ctx, node.JobID, false) + if err != nil { + return RetryReceipt{}, err + } + if job != nil && (job.State == "quarantined" || job.State == "undecodable") { + return RetryReceipt{}, fmt.Errorf( + "headgate workflow: node %q requires recovery from %q", node.Name, job.State, + ) + } + } + checkpointStore, ok := inspect.(headgate.CheckpointInspectStore) + if !ok { + return RetryReceipt{}, errors.New("headgate workflow: retry requires checkpoint inspection support") + } + checkpoint, err := checkpointStore.GetJobCheckpoint(ctx, coordinatorID) + if err != nil { + return RetryReceipt{}, err + } + if checkpoint == nil || checkpoint.CursorStep != "headgate:workflow-state" || len(checkpoint.Cursor) == 0 { + return RetryReceipt{}, errors.New("headgate workflow: coordinator workflow-state checkpoint is missing") + } + var cursor workflowCursor + if err := json.Unmarshal(checkpoint.Cursor, &cursor); err != nil { + return RetryReceipt{}, fmt.Errorf("headgate workflow: invalid coordinator cursor: %w", err) + } + cursor.normalize() + if !cursor.Failed || cursor.Revision != expectedRevision { + return RetryReceipt{}, fmt.Errorf("headgate workflow: retry revision conflict: expected %d, current %d", expectedRevision, cursor.Revision) + } + if cursor.Revision == ^uint64(0) || cursor.Generation == ^uint32(0) { + return RetryReceipt{}, errors.New("headgate workflow: retry revision or generation would overflow") + } + nextRevision := cursor.Revision + 1 + retry := RetryArgs{WorkflowID: workflowID, ExpectedRevision: expectedRevision} + payload, err := json.Marshal(retry) + if err != nil { + return RetryReceipt{}, err + } + receipt := headgate.Envelope{ + ID: retryReceiptID(workflowID, nextRevision), Kind: RetryKind, SchemaVersion: 1, + Payload: payload, Queue: coordinator.Queue, Pending: true, RetentionMs: defaultRetention, + Fingerprint: headgate.Fingerprint(RetryKind, payload), + } + if err := inspect.Enqueue(ctx, []headgate.Envelope{receipt}); err != nil { + return RetryReceipt{}, err + } + if err := inspect.OperatorRetry(ctx, coordinatorID); err != nil { + current, readErr := inspect.GetJob(ctx, coordinatorID, false) + if readErr != nil { + return RetryReceipt{}, readErr + } + if current == nil || (current.State != "available" && current.State != "running") { + return RetryReceipt{}, err + } + } + return RetryReceipt{Revision: nextRevision, Generation: cursor.Generation + 1}, nil +} + +// EmitSignal durably emits a named signal for an existing workflow. Repeating an +// emission after its signal jobs become available, running, or completed succeeds. +func EmitSignal(ctx context.Context, inspect headgate.InspectStore, workflowID, signal string) (SignalReceipt, error) { + return EmitSignalWith(ctx, inspect, workflowID, SignalEmission{ + Signal: signal, IdempotencyKey: "legacy:" + signal, Payload: json.RawMessage("null"), Source: json.RawMessage("{}"), + }) +} + +// EmitSignalWith records the payload and emitter metadata before releasing matching +// signal nodes. A replay with the same key returns the original emission and retries +// promotion; reusing a key with different content is rejected. +func EmitSignalWith(ctx context.Context, inspect headgate.InspectStore, workflowID string, emission SignalEmission) (SignalReceipt, error) { + signal := emission.Signal + if workflowID == "" || signal == "" { + return SignalReceipt{}, errors.New("headgate workflow: workflow id and signal must not be empty") + } + if emission.IdempotencyKey == "" { + return SignalReceipt{}, errors.New("headgate workflow: signal idempotency key must not be empty") + } + if len(emission.Payload) == 0 { + emission.Payload = json.RawMessage("null") + } + if len(emission.Source) == 0 { + emission.Source = json.RawMessage("{}") + } + payload, err := canonicalSignalJSON(emission.Payload) + if err != nil { + return SignalReceipt{}, errors.New("headgate workflow: signal payload and source must be valid JSON") + } + source, err := canonicalSignalJSON(emission.Source) + if err != nil { + return SignalReceipt{}, errors.New("headgate workflow: signal payload and source must be valid JSON") + } + emission.Payload, emission.Source = payload, source + if len(emission.Payload) > maxSignalPayload { + return SignalReceipt{}, errors.New("headgate workflow: signal payload must be at most 65536 bytes") + } + if len(emission.Source) > maxSignalSource { + return SignalReceipt{}, errors.New("headgate workflow: signal source must be at most 16384 bytes") + } + events, ok := inspect.(headgate.DurableEventStore) + if !ok { + return SignalReceipt{}, errors.New("headgate workflow: durable signal history is not supported by this backend") + } + coordinator, err := inspect.GetJob(ctx, workflowID+":coordinator", true) + if err != nil { + return SignalReceipt{}, err + } + if coordinator == nil { + return SignalReceipt{}, fmt.Errorf("headgate workflow: workflow %q was not found", workflowID) + } + var args CoordinatorArgs + if err := json.Unmarshal(coordinator.Payload, &args); err != nil { + return SignalReceipt{}, fmt.Errorf("headgate workflow: invalid coordinator: %w", err) + } + jobs := make([]string, 0) + for _, node := range args.Nodes { + if node.Kind == workflowSignal && node.Signal == signal { + jobs = append(jobs, node.JobID) + } + } + if len(jobs) == 0 { + return SignalReceipt{}, fmt.Errorf("headgate workflow: workflow %q has no signal %q", workflowID, signal) + } + stored, inserted, err := events.AppendDurableEvent(ctx, headgate.DurableEvent{ + Scope: workflowSignalScope(workflowID), Topic: signal, IdempotencyKey: emission.IdempotencyKey, + Payload: emission.Payload, Source: emission.Source, + }) + if err != nil { + return SignalReceipt{}, err + } + receipt := SignalReceipt{Matched: len(jobs), Inserted: inserted, Emission: publicWorkflowSignal(stored)} + for _, jobID := range jobs { + job, err := inspect.GetJob(ctx, jobID, false) + if err != nil { + return SignalReceipt{}, err + } + if job == nil { + return SignalReceipt{}, fmt.Errorf("headgate workflow: signal job %q was not found", jobID) + } + switch job.State { + case "pending": + if err := inspect.PromoteJob(ctx, jobID); err != nil { + current, readErr := inspect.GetJob(ctx, jobID, false) + if readErr != nil { + return SignalReceipt{}, readErr + } + if current == nil || !signalReceivedState(current.State) { + return SignalReceipt{}, err + } + } else { + receipt.Promoted++ + } + case "available", "running", "completed": + default: + return SignalReceipt{}, fmt.Errorf("headgate workflow: signal job %q cannot be emitted from state %q", jobID, job.State) + } + } + return receipt, nil +} + +func canonicalSignalJSON(raw json.RawMessage) (json.RawMessage, error) { + if !json.Valid(raw) { + return nil, errors.New("invalid JSON") + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + var value any + if err := decoder.Decode(&value); err != nil { + return nil, err + } + return json.Marshal(value) +} + +func ListSignals(ctx context.Context, inspect headgate.InspectStore, workflowID string, beforeID uint64, limit uint32) ([]WorkflowSignal, error) { + if workflowID == "" { + return nil, errors.New("headgate workflow: workflow id must not be empty") + } + events, ok := inspect.(headgate.DurableEventStore) + if !ok { + return nil, errors.New("headgate workflow: durable signal history is not supported by this backend") + } + stored, err := events.ListDurableEvents(ctx, workflowSignalScope(workflowID), beforeID, limit) + if err != nil { + return nil, err + } + out := make([]WorkflowSignal, len(stored)) + for i, event := range stored { + out[i] = publicWorkflowSignal(event) + } + return out, nil +} + +func workflowSignalScope(workflowID string) string { return "workflow:" + workflowID + ":signals" } +func publicWorkflowSignal(event headgate.DurableEvent) WorkflowSignal { + return WorkflowSignal{ID: event.EventID, Signal: event.Topic, IdempotencyKey: event.IdempotencyKey, Payload: event.Payload, Source: event.Source, RecordedAtMs: event.RecordedAtMs} +} + +func signalReceivedState(state string) bool { + return state == "available" || state == "running" || state == "completed" +} + +// RegisterCoordinator installs the durable dependency resolver. Each tick performs one +// bounded point read per node; it never scans queue depth. +func RegisterCoordinator(registry *headgate.Registry, inspect headgate.InspectStore, poll time.Duration) error { + if poll < time.Millisecond { + return errors.New("headgate workflow: poll interval must be at least 1ms") + } + if err := registerVirtualHandlers(registry); err != nil { + return err + } + if err := headgate.RegisterFunc[ChildWorkflowArgs](registry, func(ctx context.Context, job *headgate.Job[ChildWorkflowArgs]) error { + if job.Args.ChildWorkflowID == "" || job.Args.ChildWorkflowID == job.Args.ParentWorkflowID { + return errors.New("headgate workflow: invalid child workflow link") + } + child, err := inspect.GetJob(ctx, job.Args.ChildWorkflowID+":coordinator", false) + if err != nil { + return err + } + if child == nil { + return fmt.Errorf("headgate workflow: child workflow %q was not found", job.Args.ChildWorkflowID) + } + switch child.State { + case "completed": + return nil + case "archived", "cancelled", "quarantined", "undecodable": + return headgate.ErrSkipJob + default: + return headgate.Snooze(poll) + } + }); err != nil { + return err + } + return headgate.RegisterFunc[CoordinatorArgs](registry, func(ctx context.Context, job *headgate.Job[CoordinatorArgs]) error { + return headgate.StepCursor(ctx, "headgate:workflow-state", func(ctx context.Context, cursor workflowCursor) error { + cursor.normalize() + if len(cursor.Events) == 0 { + if err := cursor.recordEvent("workflow_started", "", nil); err != nil { + return err + } + if err := persistWorkflowCursor(ctx, cursor); err != nil { + return err + } + } + if cursor.AutomaticRetryPending { + if err := enqueueAutomaticRetry(ctx, inspect, job.Args, &cursor, job.Queue); err != nil { + return err + } + } + if result, handled, err := reconcileRetry(ctx, inspect, job.Args, &cursor, func(cursor workflowCursor) error { + return persistWorkflowCursor(ctx, cursor) + }); err != nil { + return err + } else if handled { + if result == tickFailed { + return headgate.ErrSkipJob + } + return headgate.Snooze(poll) + } + if result, handled, err := reconcileGraft(ctx, inspect, job.Args, &cursor, func(cursor workflowCursor) error { + return persistWorkflowCursor(ctx, cursor) + }); err != nil { + return err + } else if handled { + if result == tickFailed { + return headgate.ErrSkipJob + } + return headgate.Snooze(poll) + } + effective := effectiveWorkflow(job.Args, cursor) + result, err := tickWithCursor(ctx, inspect, effective, &cursor, func(cursor workflowCursor) error { + return persistWorkflowCursor(ctx, cursor) + }) + if err != nil { + return err + } + switch result { + case tickWaiting: + return headgate.Snooze(poll) + case tickFailed: + if job.Args.FailedSubgraphRetry { + cursor.Failed = true + if job.Args.RetryPolicy != nil && cursor.Generation < job.Args.RetryPolicy.MaxGenerations { + cursor.AutomaticRetryPending = true + if err := cursor.recordEvent("automatic_retry_scheduled", "", nil); err != nil { + return err + } + } else if err := cursor.recordEvent("workflow_failed", "", nil); err != nil { + return err + } + if err := persistWorkflowCursor(ctx, cursor); err != nil { + return err + } + } else { + if err := cursor.recordEvent("workflow_failed", "", nil); err != nil { + return err + } + if err := persistWorkflowCursor(ctx, cursor); err != nil { + return err + } + } + if cursor.AutomaticRetryPending { + return headgate.Snooze(time.Duration(job.Args.RetryPolicy.BackoffMs) * time.Millisecond) + } + return headgate.ErrSkipJob + default: + if err := cursor.recordEvent("workflow_succeeded", "", nil); err != nil { + return err + } + if err := persistWorkflowCursor(ctx, cursor); err != nil { + return err + } + return nil + } + }) + }) +} + +func registerVirtualHandlers(registry *headgate.Registry) error { + if err := headgate.RegisterFunc[SignalArgs](registry, func(context.Context, *headgate.Job[SignalArgs]) error { return nil }); err != nil { + return err + } + if err := headgate.RegisterFunc[TimerArgs](registry, func(context.Context, *headgate.Job[TimerArgs]) error { + return nil + }); err != nil { + return err + } + if err := headgate.RegisterFunc[ConditionArgs](registry, func(context.Context, *headgate.Job[ConditionArgs]) error { + return nil + }); err != nil { + return err + } + if err := headgate.RegisterFunc[GraftArgs](registry, func(context.Context, *headgate.Job[GraftArgs]) error { return nil }); err != nil { + return err + } + if err := headgate.RegisterFunc[RetryArgs](registry, func(context.Context, *headgate.Job[RetryArgs]) error { return nil }); err != nil { + return err + } + return nil +} + +type workflowCursor struct { + Revision uint64 `json:"revision"` + Completed []string `json:"completed"` + CompletedAtMs map[string]int64 `json:"completed_at_ms,omitempty"` + Grafts []nodeSpec `json:"grafts,omitempty"` + PendingGraftReceipt string `json:"pending_graft_receipt,omitempty"` + Generation uint32 `json:"generation"` + Failed bool `json:"failed,omitempty"` + PendingRetryReceipt string `json:"pending_retry_receipt,omitempty"` + AutomaticRetryPending bool `json:"automatic_retry_pending,omitempty"` + Events []WorkflowEvent `json:"events,omitempty"` +} + +func persistWorkflowCursor(ctx context.Context, cursor workflowCursor) error { + bytes, err := json.Marshal(cursor) + if err != nil { + return err + } + if err := headgate.SetCursor(ctx, cursor); err != nil { + return err + } + _, err = headgate.PersistOutput(ctx, 1, bytes) + return err +} + +func (c *workflowCursor) normalize() { + if c.Revision == 0 { + c.Revision = 1 + } + if c.Generation == 0 { + c.Generation = 1 + } +} + +func (c *workflowCursor) recordEvent(event, node string, atMs *int64) error { + sequence := uint64(1) + if len(c.Events) != 0 { + if c.Events[len(c.Events)-1].Sequence == math.MaxUint64 { + return errors.New("headgate workflow: event sequence overflow") + } + sequence = c.Events[len(c.Events)-1].Sequence + 1 + } + c.Events = append(c.Events, WorkflowEvent{ + Sequence: sequence, Event: event, Node: node, + Revision: c.Revision, Generation: c.Generation, AtMs: atMs, + }) + if len(c.Events) > maxWorkflowEvents { + c.Events = append([]WorkflowEvent(nil), c.Events[len(c.Events)-maxWorkflowEvents:]...) + } + return nil +} + +type tickResult uint8 + +const ( + tickWaiting tickResult = iota + tickSucceeded + tickFailed +) + +func tick(ctx context.Context, inspect headgate.InspectStore, workflow CoordinatorArgs) (tickResult, error) { + cursor := workflowCursor{Revision: 1} + return tickWithCursor(ctx, inspect, workflow, &cursor, nil) +} + +func effectiveWorkflow(base CoordinatorArgs, cursor workflowCursor) CoordinatorArgs { + nodes := make([]nodeSpec, 0, len(base.Nodes)+len(cursor.Grafts)) + nodes = append(nodes, base.Nodes...) + nodes = append(nodes, cursor.Grafts...) + return CoordinatorArgs{ + WorkflowID: base.WorkflowID, Nodes: nodes, + FailedSubgraphRetry: base.FailedSubgraphRetry, + RetryPolicy: base.RetryPolicy, + } +} + +func enqueueAutomaticRetry( + ctx context.Context, + inspect headgate.InspectStore, + base CoordinatorArgs, + cursor *workflowCursor, + queue string, +) error { + if !cursor.Failed { + return errors.New("headgate workflow: automatic retry is pending for a non-failed workflow") + } + if cursor.Revision == math.MaxUint64 { + return errors.New("headgate workflow: retry revision would overflow") + } + retry := RetryArgs{WorkflowID: base.WorkflowID, ExpectedRevision: cursor.Revision} + payload, err := json.Marshal(retry) + if err != nil { + return err + } + receipt := headgate.Envelope{ + ID: retryReceiptID(base.WorkflowID, cursor.Revision+1), Kind: RetryKind, + SchemaVersion: 1, Payload: payload, Queue: queue, Pending: true, + RetentionMs: defaultRetention, Fingerprint: headgate.Fingerprint(RetryKind, payload), + } + if err := inspect.Enqueue(ctx, []headgate.Envelope{receipt}); err != nil { + return err + } + cursor.AutomaticRetryPending = false + return headgate.SetCursor(ctx, *cursor) +} + +func rejectGraft(ctx context.Context, inspect headgate.InspectStore, receiptID string, nodes []nodeSpec) error { + jobIDs := make([]string, 0, len(nodes)+1) + for _, node := range nodes { + jobIDs = append(jobIDs, node.JobID) + } + jobIDs = append(jobIDs, receiptID) + for _, jobID := range jobIDs { + job, err := inspect.GetJob(ctx, jobID, false) + if err != nil { + return err + } + if job == nil { + continue + } + switch job.State { + case "pending", "scheduled", "available", "retryable": + if err := inspect.DeleteJob(ctx, jobID); err != nil { + return err + } + default: + return fmt.Errorf("headgate workflow: rejected graft job %q is already %q", jobID, job.State) + } + } + return nil +} + +func failedNodesToRetry(ctx context.Context, inspect headgate.InspectStore, workflow CoordinatorArgs) ([]string, error) { + retry := make([]string, 0) + for _, node := range workflow.Nodes { + job, err := inspect.GetJob(ctx, node.JobID, false) + if err != nil { + return nil, err + } + if job == nil { + return nil, fmt.Errorf("headgate workflow: retry-enabled node %q is missing", node.JobID) + } + switch job.State { + case "archived", "cancelled": + retry = append(retry, node.JobID) + case "pending", "scheduled", "retryable", "available", "running", "completed": + default: + return nil, fmt.Errorf("headgate workflow: node %q cannot be retried from %q", node.JobID, job.State) + } + } + return retry, nil +} + +func retryFailedChildren(ctx context.Context, inspect headgate.InspectStore, workflow CoordinatorArgs) error { + checkpointStore, ok := inspect.(headgate.CheckpointInspectStore) + if !ok { + return errors.New("headgate workflow: child retry propagation requires checkpoint inspection support") + } + for _, node := range workflow.Nodes { + if normalizedKind(node) != workflowChild { + continue + } + link, err := inspect.GetJob(ctx, node.JobID, false) + if err != nil { + return err + } + if link == nil || (link.State != "archived" && link.State != "cancelled") { + continue + } + childID := node.ChildWorkflowID + ":coordinator" + child, err := inspect.GetJob(ctx, childID, false) + if err != nil { + return err + } + if child == nil { + return fmt.Errorf("headgate workflow: child workflow %q is missing", node.ChildWorkflowID) + } + if child.State != "archived" { + continue + } + checkpoint, err := checkpointStore.GetJobCheckpoint(ctx, childID) + if err != nil { + return err + } + if checkpoint == nil || len(checkpoint.Cursor) == 0 { + return fmt.Errorf("headgate workflow: child workflow %q has no checkpoint", node.ChildWorkflowID) + } + var childCursor workflowCursor + if err := json.Unmarshal(checkpoint.Cursor, &childCursor); err != nil { + return err + } + childCursor.normalize() + if _, err := RequestFailedSubgraphRetry(ctx, inspect, node.ChildWorkflowID, childCursor.Revision); err != nil { + return err + } + } + return nil +} + +func reopenFailedNodes(ctx context.Context, inspect headgate.InspectStore, jobs []string) error { + for _, jobID := range jobs { + if err := inspect.OperatorRetry(ctx, jobID); err != nil { + return err + } + } + return nil +} + +func reconcileRetry( + ctx context.Context, + inspect headgate.InspectStore, + base CoordinatorArgs, + cursor *workflowCursor, + persist func(workflowCursor) error, +) (tickResult, bool, error) { + cursor.normalize() + if cursor.PendingRetryReceipt != "" { + receipt, err := inspect.GetJob(ctx, cursor.PendingRetryReceipt, false) + if err != nil { + return tickWaiting, true, err + } + if receipt == nil { + return tickWaiting, true, fmt.Errorf("headgate workflow: accepted retry receipt %q is missing", cursor.PendingRetryReceipt) + } + switch receipt.State { + case "pending": + jobs, err := failedNodesToRetry(ctx, inspect, effectiveWorkflow(base, *cursor)) + if err != nil { + return tickWaiting, true, err + } + if err := reopenFailedNodes(ctx, inspect, jobs); err != nil { + return tickWaiting, true, err + } + if err := inspect.PromoteJob(ctx, cursor.PendingRetryReceipt); err != nil { + return tickWaiting, true, err + } + return tickWaiting, true, nil + case "available", "running": + return tickWaiting, true, nil + case "completed": + cursor.PendingRetryReceipt = "" + if persist != nil { + if err := persist(*cursor); err != nil { + return tickWaiting, true, err + } + } + default: + return tickWaiting, true, fmt.Errorf("headgate workflow: accepted retry receipt entered %q", receipt.State) + } + } + if cursor.Revision == ^uint64(0) { + return tickWaiting, true, errors.New("headgate workflow: revision would overflow") + } + receiptID := retryReceiptID(base.WorkflowID, cursor.Revision+1) + receipt, err := inspect.GetJob(ctx, receiptID, true) + if err != nil { + return tickWaiting, true, err + } + if receipt == nil { + return tickWaiting, false, nil + } + if receipt.State != "pending" { + return tickWaiting, true, fmt.Errorf("headgate workflow: unaccepted retry receipt %q entered %q", receiptID, receipt.State) + } + var retry RetryArgs + if err := json.Unmarshal(receipt.Payload, &retry); err != nil || !base.FailedSubgraphRetry || !cursor.Failed || retry.WorkflowID != base.WorkflowID || retry.ExpectedRevision != cursor.Revision { + if rejectErr := rejectGraft(ctx, inspect, receiptID, nil); rejectErr != nil { + return tickWaiting, true, rejectErr + } + if cursor.Failed { + return tickFailed, true, nil + } + return tickWaiting, true, nil + } + competingGraftID := graftReceiptID(base.WorkflowID, cursor.Revision+1) + competing, err := inspect.GetJob(ctx, competingGraftID, true) + if err != nil { + return tickWaiting, true, err + } + if competing != nil { + if competing.State != "pending" { + return tickWaiting, true, fmt.Errorf("headgate workflow: competing graft receipt %q entered %q", competingGraftID, competing.State) + } + var graft GraftArgs + if err := json.Unmarshal(competing.Payload, &graft); err != nil { + graft.Nodes = nil + } + if err := rejectGraft(ctx, inspect, competingGraftID, graft.Nodes); err != nil { + return tickWaiting, true, err + } + } + workflow := effectiveWorkflow(base, *cursor) + if err := retryFailedChildren(ctx, inspect, workflow); err != nil { + return tickWaiting, true, err + } + jobs, err := failedNodesToRetry(ctx, inspect, workflow) + if err != nil { + if rejectErr := rejectGraft(ctx, inspect, receiptID, nil); rejectErr != nil { + return tickWaiting, true, rejectErr + } + return tickFailed, true, nil + } + if cursor.Generation == ^uint32(0) { + return tickWaiting, true, errors.New("headgate workflow: generation would overflow") + } + cursor.Revision++ + cursor.Generation++ + cursor.Failed = false + if err := cursor.recordEvent("workflow_retry_accepted", "", nil); err != nil { + return tickWaiting, true, err + } + cursor.PendingRetryReceipt = receiptID + if persist != nil { + if err := persist(*cursor); err != nil { + return tickWaiting, true, err + } } - return batch, nil + if err := reopenFailedNodes(ctx, inspect, jobs); err != nil { + return tickWaiting, true, err + } + if err := inspect.PromoteJob(ctx, receiptID); err != nil { + return tickWaiting, true, err + } + return tickWaiting, true, nil } -func validateGraph(nodes []draftNode) error { - if len(nodes) > maxWorkflowNodes { - return fmt.Errorf("headgate workflow: must contain at most %d tasks", maxWorkflowNodes) - } - names := make(map[string]struct{}, len(nodes)) - edges := 0 - for _, node := range nodes { - if node.name == "" { - return errors.New("headgate workflow: task names must not be empty") +func reconcileGraft( + ctx context.Context, + inspect headgate.InspectStore, + base CoordinatorArgs, + cursor *workflowCursor, + persist func(workflowCursor) error, +) (tickResult, bool, error) { + cursor.normalize() + if cursor.PendingGraftReceipt != "" { + receipt, err := inspect.GetJob(ctx, cursor.PendingGraftReceipt, false) + if err != nil { + return tickWaiting, true, err } - if _, exists := names[node.name]; exists { - return fmt.Errorf("headgate workflow: task name %q is repeated", node.name) + if receipt == nil { + return tickWaiting, true, fmt.Errorf("headgate workflow: accepted graft receipt %q is missing", cursor.PendingGraftReceipt) } - if len(node.name) > 128 { - return fmt.Errorf("headgate workflow: task name %q exceeds 128 bytes", node.name) + switch receipt.State { + case "pending": + if err := inspect.PromoteJob(ctx, cursor.PendingGraftReceipt); err != nil { + return tickWaiting, true, err + } + return tickWaiting, true, nil + case "available", "running": + return tickWaiting, true, nil + case "completed": + cursor.PendingGraftReceipt = "" + if persist != nil { + if err := persist(*cursor); err != nil { + return tickWaiting, true, err + } + } + default: + return tickWaiting, true, fmt.Errorf("headgate workflow: accepted graft receipt entered %q", receipt.State) } - edges += len(node.deps) - names[node.name] = struct{}{} } - if edges > maxWorkflowEdges { - return fmt.Errorf("headgate workflow: must contain at most %d dependency edges", maxWorkflowEdges) + if cursor.Revision == ^uint64(0) { + return tickWaiting, true, errors.New("headgate workflow: revision would overflow") } - degree := make(map[string]int, len(nodes)) - outgoing := make(map[string][]string) - for _, node := range nodes { - seen := map[string]struct{}{} - for _, dep := range node.deps { - if _, exists := names[dep]; !exists { - return fmt.Errorf("headgate workflow: task %q depends on missing task %q", node.name, dep) - } - if _, exists := seen[dep]; exists { - return fmt.Errorf("headgate workflow: task %q repeats dependency %q", node.name, dep) - } - seen[dep] = struct{}{} - degree[node.name]++ - outgoing[dep] = append(outgoing[dep], node.name) + nextRevision := cursor.Revision + 1 + receiptID := graftReceiptID(base.WorkflowID, nextRevision) + receipt, err := inspect.GetJob(ctx, receiptID, true) + if err != nil { + return tickWaiting, true, err + } + if receipt == nil { + return tickWaiting, false, nil + } + if receipt.State != "pending" { + return tickWaiting, true, fmt.Errorf("headgate workflow: unaccepted graft receipt %q entered %q", receiptID, receipt.State) + } + var graft GraftArgs + if err := json.Unmarshal(receipt.Payload, &graft); err != nil { + if rejectErr := rejectGraft(ctx, inspect, receiptID, nil); rejectErr != nil { + return tickWaiting, true, rejectErr } + return tickWaiting, true, nil } - ready := make([]string, 0, len(nodes)) - for name := range names { - if degree[name] == 0 { - ready = append(ready, name) + if graft.WorkflowID != base.WorkflowID || graft.ExpectedRevision != cursor.Revision || len(graft.Nodes) == 0 || cursor.Failed { + if err := rejectGraft(ctx, inspect, receiptID, graft.Nodes); err != nil { + return tickWaiting, true, err } + return tickWaiting, true, nil } - visited := 0 - for len(ready) > 0 { - name := ready[0] - ready = ready[1:] - visited++ - for _, child := range outgoing[name] { - degree[child]-- - if degree[child] == 0 { - ready = append(ready, child) - } + candidate := effectiveWorkflow(base, *cursor) + candidate.Nodes = append(candidate.Nodes, graft.Nodes...) + if err := validateCoordinator(candidate); err != nil { + if rejectErr := rejectGraft(ctx, inspect, receiptID, graft.Nodes); rejectErr != nil { + return tickWaiting, true, rejectErr } + return tickWaiting, true, nil } - if visited != len(nodes) { - return errors.New("headgate workflow: dependency graph contains a cycle") + cursor.Revision = nextRevision + cursor.Grafts = append(cursor.Grafts, graft.Nodes...) + if err := cursor.recordEvent("workflow_graft_accepted", "", nil); err != nil { + return tickWaiting, true, err } - return nil -} - -type nodeSpec struct { - Name string `json:"name"` - JobID string `json:"job_id"` - Deps []string `json:"deps"` -} - -type CoordinatorArgs struct { - WorkflowID string `json:"workflow_id"` - Nodes []nodeSpec `json:"nodes"` -} - -func (CoordinatorArgs) Kind() string { return CoordinatorKind } - -// RegisterCoordinator installs the durable dependency resolver. Each tick performs one -// bounded point read per node; it never scans queue depth. -func RegisterCoordinator(registry *headgate.Registry, inspect headgate.InspectStore, poll time.Duration) error { - if poll < time.Millisecond { - return errors.New("headgate workflow: poll interval must be at least 1ms") + cursor.PendingGraftReceipt = receiptID + if persist != nil { + if err := persist(*cursor); err != nil { + return tickWaiting, true, err + } } - return headgate.RegisterFunc[CoordinatorArgs](registry, func(ctx context.Context, job *headgate.Job[CoordinatorArgs]) error { - return headgate.StepCursor(ctx, "headgate:workflow-state", func(ctx context.Context, cursor workflowCursor) error { - completed := completedSet(job.Args, cursor.Completed) - result, err := tickWithEvidence(ctx, inspect, job.Args, completed, func(cursor workflowCursor) error { - return headgate.SetCursor(ctx, cursor) - }) - if err != nil { - return err - } - switch result { - case tickWaiting: - return headgate.Snooze(poll) - case tickFailed: - return headgate.ErrSkipJob - default: - return nil - } - }) - }) -} - -type workflowCursor struct { - Completed []string `json:"completed"` -} - -type tickResult uint8 - -const ( - tickWaiting tickResult = iota - tickSucceeded - tickFailed -) - -func tick(ctx context.Context, inspect headgate.InspectStore, workflow CoordinatorArgs) (tickResult, error) { - return tickWithEvidence(ctx, inspect, workflow, make(map[string]struct{}), nil) + if err := inspect.PromoteJob(ctx, receiptID); err != nil { + return tickWaiting, true, err + } + return tickWaiting, true, nil } +// tickWithEvidence remains a narrow test seam for the retained-completion behavior. func tickWithEvidence( ctx context.Context, inspect headgate.InspectStore, workflow CoordinatorArgs, completed map[string]struct{}, persist func(workflowCursor) error, +) (tickResult, error) { + cursor := workflowCursor{Revision: 1, Completed: completedNames(workflow, completed)} + result, err := tickWithCursor(ctx, inspect, workflow, &cursor, persist) + for _, name := range cursor.Completed { + completed[name] = struct{}{} + } + return result, err +} + +func tickWithCursor( + ctx context.Context, + inspect headgate.InspectStore, + workflow CoordinatorArgs, + cursor *workflowCursor, + persist func(workflowCursor) error, ) (tickResult, error) { if err := validateCoordinator(workflow); err != nil { return tickWaiting, err } + completed := completedSet(workflow, cursor.Completed) state := make(map[string]*headgate.JobSummary, len(workflow.Nodes)) type readResult struct { name string @@ -278,45 +2026,125 @@ func tickWithEvidence( } state[result.name] = result.job } + before := make(map[string]struct{}, len(completed)) + for name := range completed { + before[name] = struct{}{} + } changed := false for _, node := range workflow.Nodes { - if job := state[node.Name]; job != nil && job.State == "completed" { + if kind := normalizedKind(node); kind == workflowTask || kind == workflowChild { + if job := state[node.Name]; job != nil && job.State == "completed" { + if _, exists := completed[node.Name]; !exists { + completed[node.Name] = struct{}{} + changed = true + } + if job.FinalizedAtMs != nil { + if cursor.CompletedAtMs == nil { + cursor.CompletedAtMs = make(map[string]int64) + } + if prior, exists := cursor.CompletedAtMs[node.Name]; !exists || prior != *job.FinalizedAtMs { + cursor.CompletedAtMs[node.Name] = *job.FinalizedAtMs + changed = true + } + } + } + } + } + for { + added := false + for _, node := range workflow.Nodes { + if kind := normalizedKind(node); kind != workflowSignal && kind != workflowTimer && kind != workflowCondition { + continue + } if _, exists := completed[node.Name]; !exists { + job := state[node.Name] + if job == nil || job.State != "completed" || !dependenciesCompleted(node, completed) { + continue + } completed[node.Name] = struct{}{} + if job.FinalizedAtMs != nil { + if cursor.CompletedAtMs == nil { + cursor.CompletedAtMs = make(map[string]int64) + } + cursor.CompletedAtMs[node.Name] = *job.FinalizedAtMs + } changed = true + added = true } } + if !added { + break + } } - if changed && persist != nil { - if err := persist(workflowCursor{Completed: completedNames(workflow, completed)}); err != nil { - return tickWaiting, err + for _, node := range workflow.Nodes { + _, wasComplete := before[node.Name] + _, isComplete := completed[node.Name] + if !wasComplete && isComplete { + var atMs *int64 + if completedAt, ok := cursor.CompletedAtMs[node.Name]; ok { + value := completedAt + atMs = &value + } + if err := cursor.recordEvent("node_completed", node.Name, atMs); err != nil { + return tickWaiting, err + } + } + } + if changed { + cursor.Completed = completedNames(workflow, completed) + if persist != nil { + if err := persist(*cursor); err != nil { + return tickWaiting, err + } } } + failedNodes := workflowFailedSet(workflow, state, completed) type mutation struct { jobID string delete bool } mutations := make([]mutation, 0) for _, node := range workflow.Nodes { - job := effectiveJob(state[node.Name], node.Name, completed) - if job == nil || job.State != "pending" { + job := effectiveJob(state[node.Name], node, completed) + if _, failed := failedNodes[node.Name]; failed { + if !workflow.FailedSubgraphRetry && job != nil && deletableWorkflowState(job.State) { + mutations = append(mutations, mutation{jobID: node.JobID, delete: true}) + } continue } - depFailed := false - depsComplete := true - for _, dep := range node.Deps { - upstream := effectiveJob(state[dep], dep, completed) - if upstream == nil || isFailed(upstream.State) { - depFailed = true + if (normalizedKind(node) == workflowTask || normalizedKind(node) == workflowChild) && + job != nil && job.State == "pending" && + dependenciesComplete(workflow, node, state, completed) { + mutations = append(mutations, mutation{jobID: node.JobID}) + } + if normalizedKind(node) == workflowCondition && job != nil && job.State == "pending" && + dependenciesComplete(workflow, node, state, completed) { + ready, err := evaluateCondition(node, cursor, workflow, state, completed) + if err != nil { + return tickWaiting, err } - if upstream == nil || upstream.State != "completed" { - depsComplete = false + if ready { + mutations = append(mutations, mutation{jobID: node.JobID}) } } - if depFailed { - mutations = append(mutations, mutation{jobID: node.JobID, delete: true}) - } else if depsComplete { - mutations = append(mutations, mutation{jobID: node.JobID}) + if normalizedKind(node) == workflowTimer && node.DelayMs > 0 && + job != nil && job.State == "pending" && + dependenciesComplete(workflow, node, state, completed) { + scheduler, ok := inspect.(headgate.PendingScheduleStore) + if !ok { + return tickWaiting, errors.New("headgate workflow: backend cannot schedule pending timers") + } + anchor, err := dependencyCompletionAnchor(node, cursor.CompletedAtMs) + if err != nil { + return tickWaiting, err + } + if node.DelayMs > math.MaxInt64-anchor { + return tickWaiting, fmt.Errorf("headgate workflow: timer %q deadline overflow", node.Name) + } + if err := scheduler.SchedulePendingJob(ctx, node.JobID, anchor+node.DelayMs); err != nil { + return tickWaiting, err + } + return tickWaiting, nil } } if len(mutations) > 0 { @@ -365,7 +2193,11 @@ func tickWithEvidence( } failed := false for _, node := range workflow.Nodes { - job := effectiveJob(state[node.Name], node.Name, completed) + if _, nodeFailed := failedNodes[node.Name]; nodeFailed { + failed = true + continue + } + job := effectiveJob(state[node.Name], node, completed) if job == nil || isFailed(job.State) { failed = true continue @@ -404,16 +2236,159 @@ func completedNames(workflow CoordinatorArgs, completed map[string]struct{}) []s return names } -func effectiveJob(job *headgate.JobSummary, name string, completed map[string]struct{}) *headgate.JobSummary { +func effectiveJob(job *headgate.JobSummary, node nodeSpec, completed map[string]struct{}) *headgate.JobSummary { + if _, ok := completed[node.Name]; ok { + return &headgate.JobSummary{State: "completed"} + } + if kind := normalizedKind(node); (kind == workflowSignal || kind == workflowTimer || kind == workflowCondition) && job != nil && job.State == "completed" { + return &headgate.JobSummary{State: "pending"} + } if job != nil { return job } - if _, ok := completed[name]; ok { - return &headgate.JobSummary{State: "completed"} - } return nil } +func dependenciesCompleted(node nodeSpec, completed map[string]struct{}) bool { + for _, dep := range node.Deps { + if _, ok := completed[dep]; !ok { + return false + } + } + return true +} + +func dependencyCompletionAnchor(node nodeSpec, completedAtMs map[string]int64) (int64, error) { + if len(node.Deps) == 0 { + return 0, fmt.Errorf("headgate workflow: relative timer %q requires at least one dependency", node.Name) + } + var anchor int64 + for _, dependency := range node.Deps { + completedAt, ok := completedAtMs[dependency] + if !ok { + return 0, fmt.Errorf( + "headgate workflow: timer %q has no durable completion timestamp for %q", + node.Name, dependency, + ) + } + if completedAt > anchor { + anchor = completedAt + } + } + return anchor, nil +} + +func evaluateCondition( + node nodeSpec, + cursor *workflowCursor, + workflow CoordinatorArgs, + state map[string]*headgate.JobSummary, + completed map[string]struct{}, +) (bool, error) { + env, err := conditionEnv() + if err != nil { + return false, err + } + ast, issues := env.Compile(node.Condition) + if issues != nil && issues.Err() != nil { + return false, fmt.Errorf("headgate workflow: condition %q: %w", node.Name, issues.Err()) + } + program, err := env.Program(ast) + if err != nil { + return false, err + } + states := make(map[string]string, len(workflow.Nodes)) + completion := make(map[string]bool, len(workflow.Nodes)) + for _, candidate := range workflow.Nodes { + job := effectiveJob(state[candidate.Name], candidate, completed) + states[candidate.Name] = "missing" + if job != nil { + states[candidate.Name] = job.State + } + _, completion[candidate.Name] = completed[candidate.Name] + } + out, _, err := program.Eval(map[string]any{ + "revision": cursor.Revision, "generation": uint64(cursor.Generation), + "states": states, "completed": completion, + }) + if err != nil { + return false, fmt.Errorf("headgate workflow: condition %q failed: %w", node.Name, err) + } + value, ok := out.Value().(bool) + if !ok { + return false, fmt.Errorf("headgate workflow: condition %q must return bool", node.Name) + } + return value, nil +} + +func dependenciesComplete(workflow CoordinatorArgs, node nodeSpec, state map[string]*headgate.JobSummary, completed map[string]struct{}) bool { + for _, dep := range node.Deps { + upstream := effectiveJob(state[dep], findNode(workflow, dep), completed) + if upstream == nil || upstream.State != "completed" { + return false + } + } + return true +} + +func dependencyFailed(workflow CoordinatorArgs, node nodeSpec, state map[string]*headgate.JobSummary, completed map[string]struct{}) bool { + for _, dep := range node.Deps { + if _, failed := workflowFailedSet(workflow, state, completed)[dep]; failed { + return true + } + } + return false +} + +func workflowFailedSet(workflow CoordinatorArgs, state map[string]*headgate.JobSummary, completed map[string]struct{}) map[string]struct{} { + failed := make(map[string]struct{}) + for _, node := range workflow.Nodes { + job := effectiveJob(state[node.Name], node, completed) + if job == nil || isFailed(job.State) { + failed[node.Name] = struct{}{} + } + } + for { + before := len(failed) + for _, node := range workflow.Nodes { + for _, dependency := range node.Deps { + if _, upstreamFailed := failed[dependency]; upstreamFailed { + failed[node.Name] = struct{}{} + break + } + } + } + if len(failed) == before { + return failed + } + } +} + +func deletableWorkflowState(state string) bool { + switch state { + case "pending", "scheduled", "available", "retryable": + return true + default: + return false + } +} + +func normalizedKind(node nodeSpec) workflowNodeKind { + if node.Kind == "" { + return workflowTask + } + return node.Kind +} + +func findNode(workflow CoordinatorArgs, name string) nodeSpec { + for _, node := range workflow.Nodes { + if node.Name == name { + return node + } + } + return nodeSpec{Name: name} +} + func validateCoordinator(workflow CoordinatorArgs) error { if workflow.WorkflowID == "" { return errors.New("headgate workflow: coordinator workflow id must not be empty") @@ -421,10 +2396,25 @@ func validateCoordinator(workflow CoordinatorArgs) error { if len(workflow.Nodes) == 0 || len(workflow.Nodes) > maxWorkflowNodes { return fmt.Errorf("headgate workflow: coordinator must contain 1-%d tasks", maxWorkflowNodes) } + if workflow.RetryPolicy != nil && (workflow.RetryPolicy.MaxGenerations < 2 || + workflow.RetryPolicy.BackoffMs <= 0 || !workflow.FailedSubgraphRetry) { + return errors.New("headgate workflow: coordinator contains an invalid retry policy") + } names := make(map[string]struct{}, len(workflow.Nodes)) edges := 0 for _, node := range workflow.Nodes { - if node.Name == "" || node.JobID == "" || len(node.Name) > 128 || len(node.JobID) > headgate.MaxJobIdentifierLen { + node.Kind = normalizedKind(node) + if node.Name == "" || node.JobID == "" || len(node.Name) > 128 || len(node.JobID) > headgate.MaxJobIdentifierLen || + (node.Kind == workflowSignal && node.Signal == "") || + (node.Kind == workflowTimer && (!validTimerSchedule(node.WakeAtMs, node.DelayMs) || + (node.DelayMs > 0 && len(node.Deps) == 0))) || + (node.Kind != workflowSignal && node.Signal != "") || (node.Kind != workflowTimer && node.WakeAtMs != 0) || + (node.Kind != workflowTimer && node.DelayMs != 0) || + (node.Kind == workflowChild && node.ChildWorkflowID == "") || + (node.Kind != workflowChild && node.ChildWorkflowID != "") || + (node.Kind == workflowCondition && validateCondition(node.Condition) != nil) || + (node.Kind != workflowCondition && node.Condition != "") || + (node.Kind != workflowTask && node.Kind != workflowSignal && node.Kind != workflowTimer && node.Kind != workflowChild && node.Kind != workflowCondition) { return errors.New("headgate workflow: coordinator contains an invalid task") } if _, exists := names[node.Name]; exists { @@ -436,13 +2426,71 @@ func validateCoordinator(workflow CoordinatorArgs) error { if edges > maxWorkflowEdges { return fmt.Errorf("headgate workflow: coordinator must contain at most %d dependency edges", maxWorkflowEdges) } + degree := make(map[string]int, len(workflow.Nodes)) + outgoing := make(map[string][]string) for _, node := range workflow.Nodes { + seen := make(map[string]struct{}, len(node.Deps)) for _, dep := range node.Deps { if _, exists := names[dep]; !exists { return errors.New("headgate workflow: coordinator contains a missing dependency") } + if _, exists := seen[dep]; exists { + return errors.New("headgate workflow: coordinator repeats a dependency") + } + seen[dep] = struct{}{} + degree[node.Name]++ + outgoing[dep] = append(outgoing[dep], node.Name) + } + } + ready := make([]string, 0, len(workflow.Nodes)) + for name := range names { + if degree[name] == 0 { + ready = append(ready, name) + } + } + visited := 0 + for len(ready) > 0 { + name := ready[0] + ready = ready[1:] + visited++ + for _, child := range outgoing[name] { + degree[child]-- + if degree[child] == 0 { + ready = append(ready, child) + } } } + if visited != len(workflow.Nodes) { + return errors.New("headgate workflow: coordinator dependency graph contains a cycle") + } + return nil +} + +func validTimerSchedule(wakeAtMs, delayMs int64) bool { + return (wakeAtMs > 0 && delayMs == 0) || (wakeAtMs == 0 && delayMs > 0) +} + +func conditionEnv() (*cel.Env, error) { + return cel.NewEnv( + cel.Variable("revision", cel.UintType), + cel.Variable("generation", cel.UintType), + cel.Variable("states", cel.MapType(cel.StringType, cel.StringType)), + cel.Variable("completed", cel.MapType(cel.StringType, cel.BoolType)), + ) +} + +func validateCondition(expression string) error { + if len(expression) == 0 || len(expression) > 1_024 { + return errors.New("CEL condition must contain 1-1024 bytes") + } + env, err := conditionEnv() + if err != nil { + return err + } + _, issues := env.Compile(expression) + if issues != nil && issues.Err() != nil { + return fmt.Errorf("invalid CEL condition: %w", issues.Err()) + } return nil } diff --git a/go/headgateworkflow/workflow_test.go b/go/headgateworkflow/workflow_test.go index 5490263..6dcc71e 100644 --- a/go/headgateworkflow/workflow_test.go +++ b/go/headgateworkflow/workflow_test.go @@ -5,19 +5,62 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "strings" + "sync" "testing" "time" headgate "github.com/mujhtech/headgate/go" + "github.com/mujhtech/headgate/go/headgatetest" ) type workflowInspect struct { headgate.InspectStore - jobs map[string]*headgate.JobSummary + mu sync.RWMutex + jobs map[string]*headgate.JobSummary + checkpoints map[string]*headgate.Checkpoint + events map[string][]headgate.DurableEvent +} + +func (s *workflowInspect) AppendDurableEvent(_ context.Context, event headgate.DurableEvent) (headgate.DurableEvent, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.events == nil { + s.events = make(map[string][]headgate.DurableEvent) + } + for _, existing := range s.events[event.Scope] { + if existing.IdempotencyKey == event.IdempotencyKey { + if existing.Topic != event.Topic || string(existing.Payload) != string(event.Payload) || string(existing.Source) != string(event.Source) { + return headgate.DurableEvent{}, false, &headgate.InvalidError{Msg: "durable event idempotency key was reused with different content"} + } + return existing, false, nil + } + } + event.EventID = uint64(len(s.events[event.Scope]) + 1) + event.RecordedAtMs = int64(event.EventID) + s.events[event.Scope] = append([]headgate.DurableEvent{event}, s.events[event.Scope]...) + return event, true, nil +} + +func (s *workflowInspect) ListDurableEvents(_ context.Context, scope string, before uint64, limit uint32) ([]headgate.DurableEvent, error) { + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]headgate.DurableEvent, 0, limit) + for _, event := range s.events[scope] { + if before == 0 || event.EventID < before { + out = append(out, event) + if len(out) == int(limit) { + break + } + } + } + return out, nil } func (s *workflowInspect) GetJob(_ context.Context, id string, _ bool) (*headgate.JobSummary, error) { + s.mu.RLock() + defer s.mu.RUnlock() if job := s.jobs[id]; job != nil { copy := *job return ©, nil @@ -26,6 +69,8 @@ func (s *workflowInspect) GetJob(_ context.Context, id string, _ bool) (*headgat } func (s *workflowInspect) PromoteJob(_ context.Context, id string) error { + s.mu.Lock() + defer s.mu.Unlock() job := s.jobs[id] if job == nil || job.State != "pending" { return errors.New("invalid promotion") @@ -34,11 +79,112 @@ func (s *workflowInspect) PromoteJob(_ context.Context, id string) error { return nil } +func (s *workflowInspect) SchedulePendingJob(_ context.Context, id string, atMs int64) error { + s.mu.Lock() + defer s.mu.Unlock() + job := s.jobs[id] + if job == nil || job.State != "pending" || atMs <= 0 { + return errors.New("invalid pending schedule") + } + job.State = "scheduled" + job.ScheduledAtMs = atMs + return nil +} + func (s *workflowInspect) DeleteJob(_ context.Context, id string) error { + s.mu.Lock() + defer s.mu.Unlock() delete(s.jobs, id) return nil } +func (s *workflowInspect) OperatorRetry(_ context.Context, id string) error { + s.mu.Lock() + defer s.mu.Unlock() + job := s.jobs[id] + if job == nil || (job.State != "archived" && job.State != "cancelled" && job.State != "undecodable") { + return errors.New("invalid retry") + } + job.State = "available" + return nil +} + +func (s *workflowInspect) OperatorCancel(_ context.Context, id string) error { + s.mu.Lock() + defer s.mu.Unlock() + job := s.jobs[id] + if job == nil || !cancellableWorkflowState(job.State) { + return errors.New("invalid cancel") + } + job.State = "cancelled" + return nil +} + +func (s *workflowInspect) QuarantineRelease(_ context.Context, fingerprint string) (uint64, error) { + s.mu.Lock() + defer s.mu.Unlock() + var released uint64 + for _, job := range s.jobs { + if job.Fingerprint == fingerprint && job.State == "quarantined" { + job.State = "available" + released++ + } + } + return released, nil +} + +func (s *workflowInspect) EditPayload( + _ context.Context, + id string, + payload []byte, + schemaVersion uint32, + fingerprint string, +) error { + s.mu.Lock() + defer s.mu.Unlock() + job := s.jobs[id] + if job == nil || job.State != "undecodable" || schemaVersion == 0 { + return errors.New("invalid payload edit") + } + job.Payload = append([]byte(nil), payload...) + job.SchemaVersion = schemaVersion + job.Fingerprint = fingerprint + return nil +} + +func (s *workflowInspect) Enqueue(_ context.Context, batch []headgate.Envelope) error { + s.mu.Lock() + defer s.mu.Unlock() + for _, env := range batch { + if existing := s.jobs[env.ID]; existing != nil { + if existing.Kind == env.Kind && string(existing.Payload) == string(env.Payload) && existing.Queue == env.Queue { + continue + } + return errors.New("id conflict") + } + state := "available" + if env.Pending { + state = "pending" + } + s.jobs[env.ID] = &headgate.JobSummary{ + ID: env.ID, Kind: env.Kind, Queue: env.Queue, State: state, Payload: append([]byte(nil), env.Payload...), + } + } + return nil +} + +func (s *workflowInspect) GetJobCheckpoint(_ context.Context, id string) (*headgate.Checkpoint, error) { + s.mu.RLock() + defer s.mu.RUnlock() + checkpoint := s.checkpoints[id] + if checkpoint == nil { + return nil, nil + } + copy := *checkpoint + copy.Cursor = append([]byte(nil), checkpoint.Cursor...) + return ©, nil +} + func task(kind string) headgate.Envelope { return headgate.Envelope{Kind: kind, Payload: []byte("{}"), Queue: "default"} } @@ -73,6 +219,528 @@ func TestPrepareBuildsCoordinatorAndPendingFanOutFanIn(t *testing.T) { } } +func TestInspectWorkflowReturnsTopologyAndExecutionState(t *testing.T) { + w := New("wf-inspect") + w.EnableFailedSubgraphRetry() + w.Add("extract", task("task:extract")) + w.Add("transform", task("task:transform"), "extract") + w.Add("publish", task("task:publish"), "transform") + batch, err := w.Prepare() + if err != nil { + t.Fatal(err) + } + store := &workflowInspect{ + jobs: make(map[string]*headgate.JobSummary), + checkpoints: make(map[string]*headgate.Checkpoint), + } + for _, env := range batch { + state := "pending" + if env.ID == "wf-inspect:coordinator" { + state = "running" + } + store.jobs[env.ID] = &headgate.JobSummary{ + ID: env.ID, Kind: env.Kind, State: state, Payload: env.Payload, + } + } + store.jobs["wf-inspect:extract"].State = "completed" + cursorBytes, err := json.Marshal(workflowCursor{ + Revision: 2, Generation: 3, Completed: []string{"extract"}, + CompletedAtMs: map[string]int64{"extract": 42}, + }) + if err != nil { + t.Fatal(err) + } + store.checkpoints["wf-inspect:coordinator"] = &headgate.Checkpoint{ + CursorStep: "headgate:workflow-state", Cursor: cursorBytes, + } + + snapshot, err := InspectWorkflow(context.Background(), store, "wf-inspect") + if err != nil { + t.Fatal(err) + } + if snapshot.Revision != 2 || snapshot.Generation != 3 || !snapshot.FailedSubgraphRetry { + t.Fatalf("unexpected snapshot metadata: %+v", snapshot) + } + extract := snapshot.Node("extract") + if extract == nil || extract.State != "completed" || extract.CompletedAtMs == nil || *extract.CompletedAtMs != 42 { + t.Fatalf("unexpected extract node: %+v", extract) + } + if got, ok := snapshot.Dependents("extract"); !ok || len(got) != 1 || got[0].Name != "transform" { + t.Fatalf("extract dependents = %+v, %v", got, ok) + } + if got, ok := snapshot.Dependencies("publish"); !ok || len(got) != 1 || got[0].Name != "transform" { + t.Fatalf("publish dependencies = %+v, %v", got, ok) + } + if snapshot.Node("extract").Dependencies == nil { + t.Fatal("root dependencies must be an empty array, not null") + } + if snapshot.Node("publish").Dependents == nil { + t.Fatal("terminal dependents must be an empty array, not null") + } + if _, err := WorkflowDependencies(context.Background(), store, "wf-inspect", "absent"); err == nil { + t.Fatal("missing node dependency lookup succeeded") + } +} + +func TestRevisionedGraftPersistsGraphBeforePromotingReceipt(t *testing.T) { + graft := NewGraft("wf-graft", 1) + graft.Add("after", task("task:after"), "root") + batch, err := graft.Prepare() + if err != nil { + t.Fatal(err) + } + if len(batch) != 2 || batch[0].ID != "wf-graft:graft:2" || batch[1].ID != "wf-graft:g2:after" { + t.Fatalf("unexpected graft batch: %+v", batch) + } + if !batch[0].Pending || !batch[1].Pending { + t.Fatal("graft receipt and task must both start pending") + } + + base := CoordinatorArgs{WorkflowID: "wf-graft", Nodes: []nodeSpec{{ + Name: "root", JobID: "wf-graft:root", Kind: workflowTask, Deps: []string{}, + }}} + store := &workflowInspect{jobs: map[string]*headgate.JobSummary{ + batch[0].ID: {ID: batch[0].ID, Kind: batch[0].Kind, State: "pending", Payload: batch[0].Payload}, + batch[1].ID: {ID: batch[1].ID, Kind: batch[1].Kind, State: "pending"}, + "wf-graft:root": {ID: "wf-graft:root", Kind: "task:root", State: "pending"}, + }} + cursor := workflowCursor{} + persisted := workflowCursor{} + result, handled, err := reconcileGraft(context.Background(), store, base, &cursor, func(next workflowCursor) error { + persisted = next + return nil + }) + if err != nil || !handled || result != tickWaiting { + t.Fatalf("reconcileGraft() = %v, %v, %v", result, handled, err) + } + if persisted.Revision != 2 || len(persisted.Grafts) != 1 || persisted.PendingGraftReceipt != batch[0].ID { + t.Fatalf("graft was not fenced into cursor before promotion: %+v", persisted) + } + if got := store.jobs[batch[0].ID].State; got != "available" { + t.Fatalf("receipt state = %q, want available", got) + } + + store.jobs[batch[0].ID].State = "completed" + _, handled, err = reconcileGraft(context.Background(), store, base, &cursor, func(next workflowCursor) error { + persisted = next + return nil + }) + if err != nil || handled || cursor.PendingGraftReceipt != "" { + t.Fatalf("completed receipt replay = handled %v, cursor %+v, err %v", handled, cursor, err) + } + effective := effectiveWorkflow(base, cursor) + if len(effective.Nodes) != 2 || effective.Nodes[1].Name != "after" { + t.Fatalf("effective graph = %+v", effective.Nodes) + } + if persisted.PendingGraftReceipt != "" { + t.Fatalf("completed receipt was not cleared: %+v", persisted) + } + + cycle := NewGraft("wf-graft-cycle", 1) + cycle.Add("a", task("task:a"), "b") + cycle.Add("b", task("task:b"), "a") + if _, err := cycle.Prepare(); err == nil || !strings.Contains(err.Error(), "cycle") { + t.Fatalf("cyclic graft error = %v", err) + } +} + +func TestInvalidCombinedGraftIsRemovedWithoutAdvancingRevision(t *testing.T) { + base := CoordinatorArgs{WorkflowID: "wf-reject", Nodes: []nodeSpec{{ + Name: "root", JobID: "wf-reject:root", Kind: workflowTask, Deps: []string{}, + }}} + graft := NewGraft("wf-reject", 1) + graft.Add("root", task("task:duplicate")) + batch, err := graft.Prepare() + if err != nil { + t.Fatal(err) + } + store := &workflowInspect{jobs: map[string]*headgate.JobSummary{ + batch[0].ID: {ID: batch[0].ID, State: "pending", Payload: batch[0].Payload}, + batch[1].ID: {ID: batch[1].ID, State: "pending"}, + }} + cursor := workflowCursor{} + result, handled, err := reconcileGraft(context.Background(), store, base, &cursor, nil) + if err != nil || !handled || result != tickWaiting { + t.Fatalf("rejected reconcile = %v, %v, %v", result, handled, err) + } + if cursor.Revision != 1 || len(cursor.Grafts) != 0 { + t.Fatalf("rejected graft advanced cursor: %+v", cursor) + } + if len(store.jobs) != 0 { + t.Fatalf("rejected graft left jobs behind: %+v", store.jobs) + } +} + +func TestFailedSubgraphRetryPreservesSuccessAndReopensOnlyFailure(t *testing.T) { + w := New("wf-retry").EnableFailedSubgraphRetry() + w.Add("prepare", task("task:prepare")) + failed := task("task:unstable") + failed.MaxAttempts = 1 + w.Add("unstable", failed, "prepare") + w.Add("finish", task("task:finish"), "unstable") + batch, err := w.Prepare() + if err != nil { + t.Fatal(err) + } + var base CoordinatorArgs + if err := json.Unmarshal(batch[0].Payload, &base); err != nil { + t.Fatal(err) + } + if !base.FailedSubgraphRetry { + t.Fatal("retry-enabled workflow did not encode its policy") + } + cursor := workflowCursor{Revision: 1, Generation: 1, Completed: []string{"prepare"}, Failed: true} + cursorBytes, err := json.Marshal(cursor) + if err != nil { + t.Fatal(err) + } + store := &workflowInspect{ + jobs: map[string]*headgate.JobSummary{ + "wf-retry:coordinator": {ID: "wf-retry:coordinator", Kind: CoordinatorKind, Queue: "headgate-workflow", State: "archived", Payload: batch[0].Payload}, + "wf-retry:prepare": {ID: "wf-retry:prepare", State: "completed"}, + "wf-retry:unstable": {ID: "wf-retry:unstable", State: "archived"}, + "wf-retry:finish": {ID: "wf-retry:finish", State: "pending"}, + }, + checkpoints: map[string]*headgate.Checkpoint{ + "wf-retry:coordinator": {CursorStep: "headgate:workflow-state", Cursor: cursorBytes}, + }, + } + receipt, err := RequestFailedSubgraphRetry(context.Background(), store, "wf-retry", 1) + if err != nil || receipt.Revision != 2 || receipt.Generation != 2 { + t.Fatalf("RequestFailedSubgraphRetry() = %+v, %v", receipt, err) + } + if store.jobs["wf-retry:coordinator"].State != "available" || store.jobs["wf-retry:retry:2"].State != "pending" { + t.Fatalf("request ordering = coordinator %q, receipt %q", store.jobs["wf-retry:coordinator"].State, store.jobs["wf-retry:retry:2"].State) + } + competing := NewGraft("wf-retry", 1) + competing.Add("late", task("task:late"), "prepare") + competingBatch, err := competing.Prepare() + if err != nil { + t.Fatal(err) + } + if err := store.Enqueue(context.Background(), competingBatch); err != nil { + t.Fatal(err) + } + + persisted := workflowCursor{} + result, handled, err := reconcileRetry(context.Background(), store, base, &cursor, func(next workflowCursor) error { + persisted = next + return nil + }) + if err != nil || !handled || result != tickWaiting { + t.Fatalf("reconcileRetry() = %v, %v, %v", result, handled, err) + } + if persisted.Revision != 2 || persisted.Generation != 2 || persisted.Failed || persisted.PendingRetryReceipt != "wf-retry:retry:2" { + t.Fatalf("retry was not fenced before release: %+v", persisted) + } + if store.jobs["wf-retry:prepare"].State != "completed" || store.jobs["wf-retry:unstable"].State != "available" || store.jobs["wf-retry:finish"].State != "pending" { + t.Fatalf("failed-subgraph states = prepare %q, unstable %q, finish %q", store.jobs["wf-retry:prepare"].State, store.jobs["wf-retry:unstable"].State, store.jobs["wf-retry:finish"].State) + } + if store.jobs["wf-retry:retry:2"].State != "available" { + t.Fatal("retry receipt was not released after cursor persistence") + } + if store.jobs["wf-retry:graft:2"] != nil || store.jobs["wf-retry:g2:late"] != nil { + t.Fatal("same-revision graft survived retry arbitration") + } + + store.jobs["wf-retry:retry:2"].State = "completed" + _, handled, err = reconcileRetry(context.Background(), store, base, &cursor, func(next workflowCursor) error { + persisted = next + return nil + }) + if err != nil || handled || cursor.PendingRetryReceipt != "" { + t.Fatalf("retry receipt completion = handled %v, cursor %+v, err %v", handled, cursor, err) + } + store.jobs["wf-retry:unstable"].State = "completed" + if result, err := tickWithCursor(context.Background(), store, base, &cursor, nil); err != nil || result != tickWaiting { + t.Fatalf("post-retry tick = %v, %v", result, err) + } + if store.jobs["wf-retry:finish"].State != "available" { + t.Fatalf("blocked descendant did not reopen: %q", store.jobs["wf-retry:finish"].State) + } +} + +func TestFailedSubgraphRetryRequiresExplicitTerminalRecovery(t *testing.T) { + w := New("wf-recover").EnableFailedSubgraphRetry() + w.Add("quarantined", task("task:poison")) + w.Add("undecodable", task("task:evolved"), "quarantined") + batch, err := w.Prepare() + if err != nil { + t.Fatal(err) + } + cursorBytes, err := json.Marshal(workflowCursor{Revision: 1, Generation: 1, Failed: true}) + if err != nil { + t.Fatal(err) + } + store := &workflowInspect{ + jobs: map[string]*headgate.JobSummary{ + "wf-recover:coordinator": { + ID: "wf-recover:coordinator", Kind: CoordinatorKind, Queue: "headgate-workflow", + State: "archived", Payload: batch[0].Payload, + }, + "wf-recover:quarantined": { + ID: "wf-recover:quarantined", Kind: "task:poison", State: "quarantined", + Fingerprint: "poison-fingerprint", + }, + "wf-recover:undecodable": { + ID: "wf-recover:undecodable", Kind: "task:evolved", State: "undecodable", + }, + }, + checkpoints: map[string]*headgate.Checkpoint{ + "wf-recover:coordinator": {CursorStep: "headgate:workflow-state", Cursor: cursorBytes}, + }, + } + + if _, err := RequestFailedSubgraphRetryWithRecovery( + context.Background(), store, "wf-recover", 1, nil, + ); err == nil || !strings.Contains(err.Error(), "requires recovery") { + t.Fatalf("retry without recovery error = %v", err) + } + + payload := []byte(`{"email":"new@example.com"}`) + receipt, err := RequestFailedSubgraphRetryWithRecovery( + context.Background(), store, "wf-recover", 1, + []WorkflowRecovery{ + {Node: "quarantined", ReleaseQuarantine: true}, + {Node: "undecodable", Payload: payload, SchemaVersion: 2}, + }, + ) + if err != nil { + t.Fatal(err) + } + if receipt.Revision != 2 || receipt.Generation != 2 { + t.Fatalf("receipt = %#v", receipt) + } + if got := store.jobs["wf-recover:quarantined"].State; got != "available" { + t.Fatalf("quarantined state = %q", got) + } + evolved := store.jobs["wf-recover:undecodable"] + if evolved.State != "available" || evolved.SchemaVersion != 2 || string(evolved.Payload) != string(payload) { + t.Fatalf("undecodable recovery = %#v", evolved) + } + if store.jobs["wf-recover:coordinator"].State != "available" || store.jobs["wf-recover:retry:2"].State != "pending" { + t.Fatal("recovery must finish before reopening the coordinator") + } +} + +func TestSignalEmissionIsDurableBufferedAndIdempotent(t *testing.T) { + w := New("wf-signals") + w.Add("prepare", task("task:prepare")) + w.AddSignal("approval", "approved", "prepare") + w.Add("publish", task("task:publish"), "approval") + batch, err := w.Prepare() + if err != nil { + t.Fatal(err) + } + var args CoordinatorArgs + if err := json.Unmarshal(batch[0].Payload, &args); err != nil { + t.Fatal(err) + } + store := &workflowInspect{jobs: map[string]*headgate.JobSummary{ + batch[0].ID: {ID: batch[0].ID, Kind: batch[0].Kind, State: "available", Payload: batch[0].Payload}, + }} + for _, env := range batch[1:] { + store.jobs[env.ID] = &headgate.JobSummary{ID: env.ID, Kind: env.Kind, State: "pending"} + } + + if _, err := tick(context.Background(), store, args); err != nil { + t.Fatal(err) + } + if got := store.jobs["wf-signals:approval"].State; got != "pending" { + t.Fatalf("coordinator self-promoted signal to %q", got) + } + if got := store.jobs["wf-signals:prepare"].State; got != "available" { + t.Fatalf("prepare state = %q, want available", got) + } + + receipt, err := EmitSignal(context.Background(), store, "wf-signals", "approved") + if err != nil || receipt.Matched != 1 || receipt.Promoted != 1 || !receipt.Inserted { + t.Fatalf("EmitSignal() = %#v, %v", receipt, err) + } + store.jobs["wf-signals:approval"].State = "completed" + if _, err := tick(context.Background(), store, args); err != nil { + t.Fatal(err) + } + if got := store.jobs["wf-signals:publish"].State; got != "pending" { + t.Fatalf("early signal bypassed dependency; publish state = %q", got) + } + + store.jobs["wf-signals:prepare"].State = "completed" + if _, err := tick(context.Background(), store, args); err != nil { + t.Fatal(err) + } + if got := store.jobs["wf-signals:publish"].State; got != "available" { + t.Fatalf("buffered signal was not consumed; publish state = %q", got) + } + receipt, err = EmitSignal(context.Background(), store, "wf-signals", "approved") + if err != nil || receipt.Matched != 1 || receipt.Promoted != 0 || receipt.Inserted { + t.Fatalf("repeated EmitSignal() = %#v, %v", receipt, err) + } + rich, err := EmitSignalWith(context.Background(), store, "wf-signals", SignalEmission{ + Signal: "approved", IdempotencyKey: "review-42", + Payload: json.RawMessage(`{"approved":true,"reviewer":"Ada"}`), + Source: json.RawMessage(`{"emitter":"admin-console"}`), + }) + if err != nil || !rich.Inserted || rich.Emission.Signal != "approved" || rich.Emission.RecordedAtMs == 0 { + t.Fatalf("rich signal = %#v, %v", rich, err) + } + replay, err := EmitSignalWith(context.Background(), store, "wf-signals", SignalEmission{ + Signal: "approved", IdempotencyKey: "review-42", + Payload: json.RawMessage(`{ "reviewer": "Ada", "approved": true }`), + Source: json.RawMessage(`{"emitter":"admin-console"}`), + }) + if err != nil || replay.Inserted || !reflect.DeepEqual(replay.Emission, rich.Emission) { + t.Fatalf("semantic signal replay = %#v, %v", replay, err) + } + history, err := ListSignals(context.Background(), store, "wf-signals", 0, 100) + if err != nil || len(history) != 2 || history[0].IdempotencyKey != "review-42" { + t.Fatalf("signal history = %#v, %v", history, err) + } + if _, err := EmitSignalWith(context.Background(), store, "wf-signals", SignalEmission{ + Signal: "approved", IdempotencyKey: "review-42", Payload: json.RawMessage(`false`), Source: json.RawMessage(`{"emitter":"admin-console"}`), + }); err == nil { + t.Fatal("idempotency key accepted different signal content") + } +} + +func TestTimerUsesStoreScheduleAndBuffersUntilDependenciesComplete(t *testing.T) { + w := New("wf-timer") + w.Add("prepare", task("task:prepare")) + w.AddTimerAt("release", 1_500, "prepare") + w.Add("publish", task("task:publish"), "release") + batch, err := w.Prepare() + if err != nil { + t.Fatal(err) + } + if batch[2].Kind != TimerKind || batch[2].Pending || batch[2].ScheduledAtMs != 1_500 { + t.Fatalf("timer envelope = %+v", batch[2]) + } + var args CoordinatorArgs + if err := json.Unmarshal(batch[0].Payload, &args); err != nil { + t.Fatal(err) + } + if args.Nodes[1].Kind != workflowTimer || args.Nodes[1].WakeAtMs != 1_500 { + t.Fatalf("timer node = %+v", args.Nodes[1]) + } + store := &workflowInspect{jobs: map[string]*headgate.JobSummary{ + "wf-timer:prepare": {ID: "wf-timer:prepare", State: "available"}, + "wf-timer:release": {ID: "wf-timer:release", State: "completed"}, + "wf-timer:publish": {ID: "wf-timer:publish", State: "pending"}, + }} + if _, err := tick(context.Background(), store, args); err != nil { + t.Fatal(err) + } + if got := store.jobs["wf-timer:publish"].State; got != "pending" { + t.Fatalf("early timer bypassed dependency; publish state = %q", got) + } + store.jobs["wf-timer:prepare"].State = "completed" + if _, err := tick(context.Background(), store, args); err != nil { + t.Fatal(err) + } + if got := store.jobs["wf-timer:publish"].State; got != "available" { + t.Fatalf("buffered timer was not consumed; publish state = %q", got) + } + + store.jobs["wf-timer:prepare"].State = "archived" + store.jobs["wf-timer:publish"].State = "pending" + got := tickWaiting + for range 4 { + got, err = tick(context.Background(), store, args) + if err != nil || got == tickFailed { + break + } + } + if err != nil || got != tickFailed { + t.Fatalf("failed timer dependency = %v, %v", got, err) + } +} + +func TestRelativeTimerCheckpointsBeforeStoreTimeSnooze(t *testing.T) { + w := New("wf-relative") + w.Add("prepare", task("task:prepare")) + if err := w.AddTimerAfter("wait", 250*time.Millisecond, "prepare"); err != nil { + t.Fatal(err) + } + batch, err := w.Prepare() + if err != nil { + t.Fatal(err) + } + var args CoordinatorArgs + if err := json.Unmarshal(batch[0].Payload, &args); err != nil { + t.Fatal(err) + } + completedAt := int64(1_000) + store := &workflowInspect{jobs: map[string]*headgate.JobSummary{ + "wf-relative:prepare": {ID: "wf-relative:prepare", State: "completed", FinalizedAtMs: &completedAt}, + "wf-relative:wait": {ID: "wf-relative:wait", State: "pending"}, + }} + cursor := workflowCursor{Revision: 1, Generation: 1} + if got, err := tickWithCursor(context.Background(), store, args, &cursor, nil); err != nil || got != tickWaiting { + t.Fatalf("timer scheduling = %v, %v", got, err) + } + if timer := store.jobs["wf-relative:wait"]; timer.State != "scheduled" || timer.ScheduledAtMs != 1_250 { + t.Fatalf("dependency-anchored timer = %+v", timer) + } +} + +func TestChildWorkflowNodeMirrorsCoordinatorTerminalState(t *testing.T) { + w := New("parent") + w.AddChild("billing", "billing-child") + w.Add("finish", task("task:finish"), "billing") + batch, err := w.Prepare() + if err != nil { + t.Fatal(err) + } + if batch[1].Kind != ChildWorkflowKind || !batch[1].Pending { + t.Fatalf("child envelope = %+v", batch[1]) + } + var child ChildWorkflowArgs + if err := json.Unmarshal(batch[1].Payload, &child); err != nil { + t.Fatal(err) + } + if child.ParentWorkflowID != "parent" || child.ChildWorkflowID != "billing-child" { + t.Fatalf("child args = %+v", child) + } + + inspect := &workflowInspect{jobs: map[string]*headgate.JobSummary{ + "billing-child:coordinator": {ID: "billing-child:coordinator", State: "completed"}, + "failed-child:coordinator": {ID: "failed-child:coordinator", State: "archived"}, + }} + store := headgatetest.New() + registry := headgate.NewRegistry() + if err := RegisterCoordinator(registry, inspect, time.Millisecond); err != nil { + t.Fatal(err) + } + envelope := func(id, childID string) headgate.Envelope { + payload, marshalErr := json.Marshal(ChildWorkflowArgs{ParentWorkflowID: "parent", ChildWorkflowID: childID}) + if marshalErr != nil { + t.Fatal(marshalErr) + } + return headgate.Envelope{ + ID: id, Kind: ChildWorkflowKind, Payload: payload, Queue: "headgate-workflow", + Fingerprint: headgate.Fingerprint(ChildWorkflowKind, payload), RetentionMs: 60_000, + } + } + if err := store.Enqueue(context.Background(), []headgate.Envelope{ + envelope("parent:billing", "billing-child"), + envelope("parent:failed", "failed-child"), + }); err != nil { + t.Fatal(err) + } + runner := headgate.NewRunner(store, registry, headgate.Config{ + Queues: map[string]headgate.QueueConfig{"headgate-workflow": {MaxWorkers: 2}}, + }) + if done, err := runner.Drain(context.Background(), 2); err != nil || len(done) != 2 { + t.Fatalf("child drain = %v, %v", done, err) + } + if _, state, _ := store.JobState("parent:billing"); state != "completed" { + t.Fatalf("successful child node state = %q", state) + } + if _, state, _ := store.JobState("parent:failed"); state != "archived" { + t.Fatalf("failed child node state = %q", state) + } +} + func TestPrepareRaisesShortChildRetentionToWorkflowRetention(t *testing.T) { const retention = 3 * time.Hour w := New("wf-retention") @@ -119,6 +787,93 @@ func TestWorkflowAndCoordinatorResourceBounds(t *testing.T) { } } +func TestAutomaticRetryPolicyAndCELConditionAreValidated(t *testing.T) { + w := New("wf-auto") + if err := w.AutomaticRetry(3, 25*time.Millisecond); err != nil { + t.Fatal(err) + } + w.Add("prepare", task("task:prepare")) + w.AddCondition("ready", `completed.prepare && states.prepare == "completed" && generation == 1u`, "prepare") + batch, err := w.Prepare() + if err != nil { + t.Fatal(err) + } + var coordinator CoordinatorArgs + if err := json.Unmarshal(batch[0].Payload, &coordinator); err != nil { + t.Fatal(err) + } + if coordinator.RetryPolicy == nil || coordinator.RetryPolicy.MaxGenerations != 3 || coordinator.RetryPolicy.BackoffMs != 25 { + t.Fatalf("retry policy = %#v", coordinator.RetryPolicy) + } + cursor := workflowCursor{Revision: 1, Generation: 1, Completed: []string{"prepare"}} + completed := completedSet(coordinator, cursor.Completed) + states := map[string]*headgate.JobSummary{"prepare": {ID: "wf-auto:prepare", State: "completed"}} + matched, err := evaluateCondition(coordinator.Nodes[1], &cursor, coordinator, states, completed) + if err != nil || !matched { + t.Fatalf("condition = %v, %v", matched, err) + } + bad := New("bad-cel").AddCondition("ready", "completed[") + if _, err := bad.Prepare(); err == nil { + t.Fatal("malformed CEL expression passed validation") + } +} + +func TestAtomicBundleRejectsCrossWorkflowCycles(t *testing.T) { + parent := New("parent").AddChild("child", "child") + child := New("child").Add("work", task("task:child")) + batch, err := PrepareBundle(parent, child) + if err != nil || len(batch) != 4 { + t.Fatalf("valid bundle = %d jobs, %v", len(batch), err) + } + left := New("left").AddChild("right", "right") + right := New("right").AddChild("left", "left") + if _, err := PrepareBundle(left, right); err == nil || !strings.Contains(err.Error(), "cycle") { + t.Fatalf("cycle error = %v", err) + } +} + +func TestWorkflowHistoryIsBoundedAndMonotonic(t *testing.T) { + cursor := workflowCursor{Revision: 1, Generation: 1} + for i := 0; i < maxWorkflowEvents+7; i++ { + if err := cursor.recordEvent("tick", fmt.Sprint(i), nil); err != nil { + t.Fatal(err) + } + } + if len(cursor.Events) != maxWorkflowEvents || cursor.Events[0].Sequence != 8 || cursor.Events[len(cursor.Events)-1].Sequence != 263 { + t.Fatalf("bounded events = %#v .. %#v", cursor.Events[0], cursor.Events[len(cursor.Events)-1]) + } +} + +func TestCancelWorkflowPropagatesToChildrenAndAllLiveBranches(t *testing.T) { + child := New("child").Add("child-work", task("task:child")) + parent := New("parent").Add("left", task("task:left")).Add("right", task("task:right")).AddChild("child", "child") + bundle, err := PrepareBundle(parent, child) + if err != nil { + t.Fatal(err) + } + store := &workflowInspect{jobs: make(map[string]*headgate.JobSummary), checkpoints: make(map[string]*headgate.Checkpoint)} + if err := store.Enqueue(context.Background(), bundle); err != nil { + t.Fatal(err) + } + for _, job := range store.jobs { + if job.State == "pending" { + job.State = "available" + } + } + receipt, err := CancelWorkflow(context.Background(), store, "parent", true) + if err != nil { + t.Fatal(err) + } + if receipt.Workflows != 2 || receipt.Jobs != 6 { + t.Fatalf("cancel receipt = %#v", receipt) + } + for id, job := range store.jobs { + if job.State != "cancelled" { + t.Fatalf("job %s state = %s", id, job.State) + } + } +} + func TestCoordinatorPromotesFanOutThenFanInAndPropagatesFailure(t *testing.T) { args := CoordinatorArgs{WorkflowID: "wf", Nodes: []nodeSpec{ {Name: "root", JobID: "root"}, diff --git a/scripts/check-migrations.py b/scripts/check-migrations.py index ee6ce26..0979f8b 100644 --- a/scripts/check-migrations.py +++ b/scripts/check-migrations.py @@ -374,6 +374,36 @@ "crates/headgate-migrate/migrations/mysql/0012_worker_control_state.down.sql", "go/headgatemigrate/migrations/mysql/0012_worker_control_state.down.sql", ), + ( + "Postgres driver ↔ Rust migrator up v13", + "crates/headgate-postgres/migrations/0013_durable_events.sql", + "crates/headgate-migrate/migrations/postgres/0013_durable_events.up.sql", + ), + ( + "MySQL driver ↔ Rust migrator up v13", + "crates/headgate-mysql/migrations/0013_durable_events.sql", + "crates/headgate-migrate/migrations/mysql/0013_durable_events.up.sql", + ), + ( + "Postgres Rust ↔ Go up v13", + "crates/headgate-migrate/migrations/postgres/0013_durable_events.up.sql", + "go/headgatemigrate/migrations/postgres/0013_durable_events.up.sql", + ), + ( + "Postgres Rust ↔ Go down v13", + "crates/headgate-migrate/migrations/postgres/0013_durable_events.down.sql", + "go/headgatemigrate/migrations/postgres/0013_durable_events.down.sql", + ), + ( + "MySQL Rust ↔ Go up v13", + "crates/headgate-migrate/migrations/mysql/0013_durable_events.up.sql", + "go/headgatemigrate/migrations/mysql/0013_durable_events.up.sql", + ), + ( + "MySQL Rust ↔ Go down v13", + "crates/headgate-migrate/migrations/mysql/0013_durable_events.down.sql", + "go/headgatemigrate/migrations/mysql/0013_durable_events.down.sql", + ), ] @@ -421,11 +451,11 @@ def main() -> int: failed = True cargo = (ROOT / "Cargo.toml").read_text() - gowork = (ROOT / "go/go.work").read_text() + gowork = (ROOT / "go.work").read_text() if '"crates/headgate-migrate"' not in cargo: print("FAIL: headgate-migrate is not a Cargo workspace member") failed = True - if "./headgatemigrate" not in gowork: + if "./go/headgatemigrate" not in gowork: print("FAIL: headgatemigrate is not a Go workspace module") failed = True diff --git a/scripts/run-scenarios.py b/scripts/run-scenarios.py index aedb386..2147d6f 100755 --- a/scripts/run-scenarios.py +++ b/scripts/run-scenarios.py @@ -69,8 +69,11 @@ PGHOST = os.environ.get("PGHOST", "/tmp") PGPORT = os.environ.get("PGPORT", "5433") PGDATABASE = os.environ.get("PGDATABASE", "hg") +PGPASSWORD = os.environ.get("PGPASSWORD", "") REDIS_PORT = os.environ.get("REDIS_PORT", "6380") PGCONN = f"host={PGHOST} port={PGPORT} user=postgres dbname={PGDATABASE}" +if PGPASSWORD: + PGCONN += f" password={PGPASSWORD}" REDIS_URL = f"redis://127.0.0.1:{REDIS_PORT}" PASSED = 0 diff --git a/scripts/test-admission.sh b/scripts/test-admission.sh index 6127dd1..8a01088 100755 --- a/scripts/test-admission.sh +++ b/scripts/test-admission.sh @@ -16,6 +16,9 @@ PGH=${PGHOST:-/tmp}; PGP=${PGPORT:-5433}; PGD=${PGDATABASE:-hg}; RP=${REDIS_PORT PSQL="psql -h $PGH -p $PGP -U postgres -d $PGD -qtA" RED="redis-cli -p $RP" export HG_PG="host=$PGH port=$PGP user=postgres dbname=$PGD" +if [ -n "${PGPASSWORD:-}" ]; then + export HG_PG="$HG_PG password=$PGPASSWORD" +fi H=target/debug/hg-pg-harness pass=0; fail=0; skip=0; guarded=0 From 1af10d0b93e06b96fd98243863b366b009e1690d Mon Sep 17 00:00:00 2001 From: Muhideen Mujeeb Adeoye Date: Sat, 5 Sep 2026 13:50:36 +0100 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=93=9A=20docs(workflow):=20document?= =?UTF-8?q?=20orchestration=20guarantees?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/guides/workflows.mdx | 393 ++++++++++++++++++++++++++++++++++- docs/workflow-experiments.md | 326 +++++++++++++++++++++++++---- docs/workflows.md | 116 +++++------ 3 files changed, 725 insertions(+), 110 deletions(-) diff --git a/docs/guides/workflows.mdx b/docs/guides/workflows.mdx index f3a296b..ffec3a1 100644 --- a/docs/guides/workflows.mdx +++ b/docs/guides/workflows.mdx @@ -131,9 +131,398 @@ Cursor persistence is fence-verified. If the lease has already been reclaimed, t fails and the stale attempt must stop; the new attempt resumes from the last cursor that the store accepted. +## Inspect graph state + +Graph reads are available directly from both workflow libraries; they do not depend on +the console. A snapshot includes the accepted base graph plus additive grafts, current +node and coordinator states, revision, retry generation, dependency edges, reverse +dependent edges, and virtual-node configuration. Application payloads are excluded. + + +```rust Rust +let graph = headgate_workflow::inspect_workflow(store.as_ref(), "daily-import").await?; +let index = graph.node("index").ok_or("index node missing")?; +let prerequisites = graph.dependencies("index").unwrap(); + +let downstream = headgate_workflow::workflow_dependents( + store.as_ref(), + "daily-import", + "extract", +).await?; +let page = headgate_workflow::list_workflows(store.as_ref(), None, 50).await?; +``` + +```go Go +graph, err := headgateworkflow.InspectWorkflow(ctx, store, "daily-import") +if err != nil { return err } +index := graph.Node("index") +prerequisites, ok := graph.Dependencies("index") + +downstream, err := headgateworkflow.WorkflowDependents(ctx, store, "daily-import", "extract") +page, err := headgateworkflow.ListWorkflows(ctx, store, "", 50) +``` + + +Use `workflow_node` / `GetWorkflowNode`, `workflow_dependencies` / +`WorkflowDependencies`, and `workflow_dependents` / `WorkflowDependents` for convenient +single-relation reads. When several questions are needed, fetch one snapshot and use its +in-memory methods so the graph is internally consistent and the store is read only once. + +The same reads are available over HTTP. `GET /api/v1/workflows` returns a bounded, +cursor-paginated coordinator list; `GET /api/v1/workflows/{id}` and the +`/nodes/{node}`, `/dependencies`, and `/dependents` subresources. A retained completion +remains `completed` after its job row expires; an unrecorded absent row is explicitly +reported as `missing`. + +## Durable signals, payloads, and source history + +A signal is declared as a workflow node and stored as a retained pending internal job, so +delivery survives worker restarts on PostgreSQL, Redis, and MySQL. Emitting before +`prepare` completes is safe: Headgate records the emission first and releases the signal +job second. The coordinator consumes the completed signal only after its dependencies are +complete. + +Use the rich emission API when the event carries business data. `payload` is the fact the +workflow is waiting for; `source` describes who or what emitted it. Both are arbitrary JSON +values. The `Idempotency-Key` (or SDK `idempotency_key`) identifies one emission: replaying +the same key and content returns the original record, while reusing the key with different +content is rejected. + + +```rust Rust +use headgate_workflow::{Workflow, SignalEmission, emit_signal_with, list_signals}; + +let batch = Workflow::new("onboarding:42") + .add("prepare", prepare_job, Vec::::new()) + .add_signal("approval", "approved", ["prepare"]) + .add("welcome", welcome_job, ["approval"]) + .prepare()?; +store.enqueue(&batch).await?; + +// Safe before or after `prepare` completes. +let receipt = emit_signal_with( + store.as_ref(), + "onboarding:42", + SignalEmission { + signal: "approved".into(), + idempotency_key: "approval:ticket-1842".into(), + payload: serde_json::json!({"approved": true, "reviewer": "Ada"}), + source: serde_json::json!({"emitter": "admin-api", "actor": "operator-42"}), + }, +).await?; +assert_eq!(receipt.matched, 1); +assert!(receipt.inserted); + +let history = list_signals(store.as_ref(), "onboarding:42", None, 100).await?; +assert_eq!(history[0].payload["reviewer"], "Ada"); +``` + +```go Go +workflow := headgateworkflow.New("onboarding:42") +workflow.Add("prepare", prepareJob) +workflow.AddSignal("approval", "approved", "prepare") +workflow.Add("welcome", welcomeJob, "approval") + +batch, err := workflow.Prepare() +if err != nil { + return err +} +if err := store.Enqueue(ctx, batch); err != nil { + return err +} + +// Safe before or after prepare completes. +receipt, err := headgateworkflow.EmitSignalWith(ctx, store, "onboarding:42", headgateworkflow.SignalEmission{ + Signal: "approved", + IdempotencyKey: "approval:ticket-1842", + Payload: json.RawMessage(`{"approved":true,"reviewer":"Ada"}`), + Source: json.RawMessage(`{"emitter":"admin-api","actor":"operator-42"}`), +}) +if err != nil { + return err +} +history, err := headgateworkflow.ListSignals(ctx, store, "onboarding:42", 0, 100) +``` + + +The control API exposes the same contract: + +```bash +curl -X POST "$HEADGATE_URL/api/v1/workflows/onboarding%3A42/signals" \ + -H 'Content-Type: application/json' \ + -H 'Idempotency-Key: approval:ticket-1842' \ + -d '{ + "signal": "approved", + "payload": {"approved": true, "reviewer": "Ada"}, + "source": {"emitter": "admin-api", "actor": "operator-42"} + }' + +curl "$HEADGATE_URL/api/v1/workflows/onboarding%3A42/signals?limit=100" +``` + +Signal history is newest-first and retains the latest 100 emissions per workflow. That is +also the idempotency horizon: once an old record is trimmed, its key may be accepted again. +Payload is limited to 64 KiB and source metadata to 16 KiB. The store assigns the timestamp; +worker clocks are not trusted. `source` is caller-supplied metadata, not proof of identity— +authenticate the control API upstream and populate it from that trusted identity rather +than accepting arbitrary end-user values. The console shows matching signal history when +you inspect a signal node or a task directly waiting on one. + +The short `emit_signal` / `EmitSignal` helpers remain for name-only events. They store a +`null` payload and empty source object, and use one legacy key per signal name; use the rich +API whenever multiple emissions or audit context matter. + +Absolute timers use the backend's store clock and scheduled-job promoter: + + +```rust Rust +let batch = Workflow::new("release:42") + .add("prepare", prepare_job, Vec::::new()) + .add_timer_at("release-window", release_at_ms, ["prepare"]) + .add("publish", publish_job, ["release-window"]) + .prepare()?; + +let delayed = Workflow::new("follow-up:42") + .add("prepare", prepare_job, Vec::::new()) + .add_timer_after("cooldown", Duration::from_secs(30 * 60), ["prepare"])? + .add("notify", notify_job, ["cooldown"]) + .prepare()?; +``` + +```go Go +workflow := headgateworkflow.New("release:42") +workflow.Add("prepare", prepareJob) +workflow.AddTimerAt("release-window", releaseAtMs, "prepare") +workflow.Add("publish", publishJob, "release-window") + +delayed := headgateworkflow.New("follow-up:42") +delayed.Add("prepare", prepareJob) +if err := delayed.AddTimerAfter("cooldown", 30*time.Minute, "prepare"); err != nil { + return err +} +delayed.Add("notify", notifyJob, "cooldown") +``` + + +Absolute deadlines are Unix milliseconds. A relative timer is anchored to the latest +dependency's store-stamped `finalized_at_ms`. The coordinator durably records that evidence +before atomically changing the timer from `pending` to `scheduled` at `anchor + delay`, so +worker clock skew and coordinator polling latency do not move the deadline. + +### CEL waits + +Use a condition node for a bounded boolean decision over workflow state. Available +variables are `revision` and `generation` (unsigned integers), `states` (node name to +state), and `completed` (node name to boolean). + + +```rust Rust +let workflow = Workflow::new("approval:42") + .add("prepare", prepare_job, Vec::::new()) + .add_condition( + "eligible", + "completed.prepare && states.prepare == 'completed'", + ["prepare"], + ); +``` + +```go Go +workflow := headgateworkflow.New("approval:42") +workflow.Add("prepare", prepareJob) +workflow.AddCondition("eligible", `completed.prepare && states.prepare == "completed"`, "prepare") +``` + + +Expressions are limited to 1,024 bytes, compile during `prepare`, cannot perform I/O, and +must return a boolean. + +### Experimental graph grafts + +Add ordinary tasks to a workflow that is still running by preparing a batch against the +current graph revision. Enqueue the returned receipt and tasks together in one atomic +store call. + + +```rust Rust +let graft = WorkflowGraft::new("onboarding:42", 1) + .queue("workflows") + .add("send-survey", survey_job, ["send-welcome"]) + .prepare()?; + +store.enqueue(&graft).await?; +``` + +```go Go +graft := headgateworkflow.NewGraft("onboarding:42", 1).Queue("workflows") +graft.Add("send-survey", surveyJob, "send-welcome") +batch, err := graft.Prepare() +if err != nil { return err } + +if err := store.Enqueue(ctx, batch); err != nil { return err } +``` + + +Revision `1` is the initial graph, so this batch requests revision `2`. The receipt id is +deterministic (`onboarding:42:graft:2`): concurrent writers using the same expected +revision collide at atomic enqueue instead of both changing the graph. The coordinator +validates the combined DAG, checkpoints the new revision and nodes, and only then releases +the receipt handler. A crash in between replays the same accepted receipt. + +Grafts are additive and currently accept ordinary tasks only. They cannot replace or +delete an existing node or revive a completed coordinator. Accepted grafts enter the +bounded workflow history. `POST /api/v1/workflows/{id}/grafts` provides the authenticated, +idempotency-key-protected control route. The console remains read-only while its mutation +controls are reviewed. + +#### Non-task nodes cannot be grafted + +Signals, absolute or relative timers, CEL conditions, and child-workflow links must be +declared when the initial workflow is prepared. Neither `WorkflowGraft::add` / +`WorkflowGraft.Add` nor `POST /api/v1/workflows/{id}/grafts` accepts those node types. +Attempting to represent one as an ordinary task does not give it signal, timer, +condition, or child-workflow semantics. + +This boundary is deliberate. An ordinary task graft only adds a pending job and dependency +edges. Each non-task node needs additional durable rules: + +- a signal needs a stable buffered-delivery identity and idempotent emission lookup; +- a relative timer needs a store-stamped dependency-completion anchor; +- a condition must be compiled, bounded, and evaluated against a defined workflow state; +- a child link changes the cross-workflow graph and therefore needs cycle detection, + creation atomicity, cancellation propagation, and retry propagation. + +Adding any of these after execution has started therefore requires more than accepting a +different payload in the graft receipt. It needs a versioned mutation contract, combined +graph validation, deterministic replay, history records, and matching behavior across all +stores and both language implementations. Headgate does not claim that contract yet. + +Plan control-flow nodes up front, then graft ordinary work whose dependencies can point to +those existing nodes. If an unforeseen signal, timer, condition, or child workflow changes +the orchestration itself, create a new workflow ID with the new graph. A separately +enqueued workflow remains a separate execution; it is not retroactively part of the +original graph. + + +An accepted node and its dependency edges are immutable, including while the workflow is +active. A graft can append new ordinary tasks that depend on existing nodes, but cannot +replace, rename, remove, or rewire accepted work. Use a new workflow ID when a change +requires a different interpretation of the existing graph. + + + +Once the coordinator records a terminal workflow outcome, that execution's graph and +history are immutable. Signals and grafts cannot revive it, and operators cannot append, +replace, remove, rename, or rewire its nodes. Start different work with a new workflow ID. +Failed-subgraph retry is the narrow exception: when enabled before enqueue, it advances to +a new generation of the same unchanged graph and preserves completed ancestors. + + +### Experimental failed-subgraph retry + +Enable retry before the workflow is enqueued. This retains dependency-blocked pending +jobs when a node exhausts rather than deleting them. + + +```rust Rust +let batch = Workflow::new("onboarding:42") + .failed_subgraph_retry() + .add("create-account", account_job, Vec::::new()) + .add("send-welcome", email_job, ["create-account"]) + .prepare()?; +store.enqueue(&batch).await?; + +// The failed coordinator checkpoint currently says revision 1. +let retry = request_failed_subgraph_retry(&store, "onboarding:42", 1).await?; +assert_eq!(retry.generation, 2); +``` + +```go Go +workflow := headgateworkflow.New("onboarding:42").EnableFailedSubgraphRetry() +workflow.Add("create-account", accountJob) +workflow.Add("send-welcome", emailJob, "create-account") +batch, err := workflow.Prepare() +if err != nil { return err } +if err := store.Enqueue(ctx, batch); err != nil { return err } + +retry, err := headgateworkflow.RequestFailedSubgraphRetry(ctx, store, "onboarding:42", 1) +if err != nil { return err } +``` + + +The request requires an archived coordinator, checkpoint inspection, and the exact graph +revision. It enqueues the deterministic retry receipt before reopening the coordinator. +The coordinator checkpoints generation 2 and revision 2 before applying operator retry +to failed archived/cancelled nodes. Completed ancestors remain completed, while their +pending descendants resume only after the retried node succeeds. + +For automatic retry, use `automatic_retry(max_generations, backoff)` in Rust or +`AutomaticRetry(maxGenerations, backoff)` in Go. The generation limit includes the first +run and the backoff uses the ordinary store-timed snooze path. + +`POST /api/v1/workflows/{id}/retry` can also repair states that are unsafe to reopen +blindly. A quarantined node requires `release_quarantine: true` (the release applies to its +fingerprint). An undecodable node requires a replacement base64 payload and positive +`schema_version`. Completed ancestors remain untouched. A failed child link requests the +child coordinator's failed-subgraph retry before the parent link is reopened. + +```bash +curl -X POST http://127.0.0.1:8080/api/v1/workflows/onboarding:42/retry \ + -H 'Content-Type: application/json' \ + -H 'Idempotency-Key: onboarding-42-retry-1' \ + -d '{ + "expected_revision": 1, + "recoveries": [ + {"node": "send-welcome", "payload": "eyJlbWFpbCI6Im5ld0BleGFtcGxlLmNvbSJ9", "schema_version": 2} + ] + }' +``` + +The server checks every workflow node after applying the requested repairs. It will not +reopen the coordinator while any node remains quarantined or undecodable. + +### Experimental child workflows + +Create the child and parent builders, link the child coordinator as a parent node, and use +one atomic bundle: + + +```rust Rust +let child = Workflow::new("billing:42") + .add("charge", charge_job, Vec::::new()); + +let parent = Workflow::new("checkout:42") + .add_child("billing", "billing:42", Vec::::new()) + .add("receipt", receipt_job, ["billing"]); +store.enqueue(&prepare_bundle(vec![parent, child])?).await?; +``` + +```go Go +child := headgateworkflow.New("billing:42") +child.Add("charge", chargeJob) +parent := headgateworkflow.New("checkout:42") +parent.AddChild("billing", "billing:42") +parent.Add("receipt", receiptJob, "billing") +bundle, err := headgateworkflow.PrepareBundle(parent, child) +if err != nil { return err } +if err := store.Enqueue(ctx, bundle); err != nil { return err } +``` + + +The bundle verifies that every child is present, rejects cross-workflow cycles, and makes +parent/child creation atomic. A completed child releases downstream parent work; a failed +child archives the link and invokes ordinary parent failure propagation. Workflow cancel +propagates to linked children by default and visits all active parallel branches; callers +may explicitly disable child propagation. + +`register_coordinator` / `RegisterCoordinator` installs the coordinator and all internal +workflow handlers. The mutation API requires `Idempotency-Key` and uses the same upstream +authentication boundary as the rest of the control API. + -This first workflow slice is immutable. Signals, timers, graph mutation, nested workflows, -and workflow-level retry are not claimed yet. +The experiment now has durable signals, dependency-anchored timers, CEL waits, additive +grafts, atomic child bundles, retry/repair, cancellation propagation, authenticated API +routes, and a bounded 256-event workflow history. The console remains intentionally +read-only while its mutation controls are reviewed. diff --git a/docs/workflow-experiments.md b/docs/workflow-experiments.md index d16e8ec..13a3f3a 100644 --- a/docs/workflow-experiments.md +++ b/docs/workflow-experiments.md @@ -1,46 +1,284 @@ # Dynamic workflow experiments -This branch explores dynamic workflow behavior without changing the shipped static -coordinator or claiming durable support. Rust exposes the reducer under -`headgate_workflow::experimental`; Go mirrors it in `headgateworkflow/experimental`. - -The reducer exists to settle semantics before a migration, store port, HTTP API, or UI -makes an accidental contract permanent. - -## Current decisions - -| Capability | Experimental behavior | -| --- | --- | -| Signals | Signals are named, idempotent, and buffered. A signal received before its dependencies complete is retained and consumed when the wait node becomes eligible. | -| Timers | Timer deadlines are absolute milliseconds advanced by store time. Moving time backwards is rejected; worker clocks are not accepted as durable workflow time. | -| Graph mutation | Grafts are additive and require the caller's expected graph revision. Existing nodes cannot be rewritten, and the combined graph must still have unique names, valid dependencies, and no cycle. | -| Nested workflows | A child workflow is an explicit node. The parent dispatches it only after its dependencies succeed and settles it through the same success/failure boundary as a task. | -| Workflow retry | Retry increments the workflow generation, resets failed and dependency-blocked nodes, and preserves successful ancestors. It does not silently rerun already successful effects. | - -The reducer emits actions instead of performing I/O: dispatch a task, wait for a signal, -arm a timer, start a child workflow, or record terminal workflow state. Rust and Go tests -drive the same signal → timer chain, revision-conflicted graft, nested failure, and -failed-subgraph retry. - -## Durability boundary still to build - -A production implementation must commit the state transition and its emitted actions in -one store transaction or script. Otherwise a coordinator can persist `active` and crash -before dispatching the action, or dispatch twice after a crash. The eventual action -identity should include workflow ID, graph revision, generation, node name, and action -kind so replay is deterministic. - -The current experiment deliberately has no: - -- PostgreSQL, MySQL, or Redis persistence; -- signal, graft, retry, or child-workflow control API; -- authorization and `Idempotency-Key` contract for those mutations; -- migration from the v1 immutable coordinator payload; -- dynamic workflow UI controls; or -- conformance claim. - -Before promoting the reducer, the design still needs decisions for cancellation of active -parallel branches, propagation of parent cancellation into children, relative timers that -start after a dependency completes, event-history retention, and bounded graph/event -limits. The existing immutable coordinator remains the compatibility baseline throughout -the experiment. +This branch extends the immutable workflow layer without moving orchestration into +`headgate-core` or changing admission. Rust exposes the features from +`headgate-workflow`; Go exposes the same contract from `headgateworkflow`. + +## What is implemented + +- durable, buffered, idempotent signals; +- absolute timers and relative timers anchored to the latest dependency's store-stamped + `finalized_at_ms`; +- additive revision-checked graph grafts; +- CEL boolean waits over `revision`, `generation`, `states`, and `completed`; +- child workflows, atomic parent/child bundles, and cross-workflow cycle rejection; +- manual and automatic failed-subgraph retry while preserving successful ancestors; +- explicit repair of quarantined and undecodable nodes; +- bounded parent-to-child cancellation and failed-child retry propagation; +- a bounded durable event history; and +- package and HTTP graph inspection for nodes, dependencies, dependents, revision, + generation, and execution state; and +- HTTP routes for graph inspection, history, signal, graft, retry, and cancellation. Every mutation uses + the API's existing upstream-authentication boundary and requires `Idempotency-Key`. + +The console remains read-only while its mutation controls are reviewed. + +## Signals and conditions + +A signal declaration creates a retained pending internal job. The rich emission API first +appends a store-timestamped record containing the signal name, idempotency key, JSON +payload, and caller-supplied JSON source, then promotes that job. Delivery before its +dependencies finish is buffered, and replay with the same key and content returns the +original emission. Reusing a key with different content is rejected. A condition is also a +pending internal job, but the coordinator only promotes it when its CEL expression +evaluates to `true`. + +```rust +let batch = Workflow::new("approval:42") + .add("prepare", prepare_job, Vec::::new()) + .add_condition( + "eligible", + "completed.prepare && states.prepare == 'completed'", + ["prepare"], + ) + .add_signal("approval", "approved", ["eligible"]) + .add("publish", publish_job, ["approval"]) + .prepare()?; +store.enqueue(&batch).await?; +emit_signal_with(store.as_ref(), "approval:42", SignalEmission { + signal: "approved".into(), + idempotency_key: "approval:ticket-1842".into(), + payload: serde_json::json!({"approved": true}), + source: serde_json::json!({"emitter": "admin-api", "actor": "operator-42"}), +}).await?; +``` + +```go +workflow := headgateworkflow.New("approval:42") +workflow.Add("prepare", prepareJob) +workflow.AddCondition("eligible", `completed.prepare && states.prepare == "completed"`, "prepare") +workflow.AddSignal("approval", "approved", "eligible") +workflow.Add("publish", publishJob, "approval") +batch, err := workflow.Prepare() +if err != nil { return err } +if err := store.Enqueue(ctx, batch); err != nil { return err } +_, err = headgateworkflow.EmitSignalWith(ctx, store, "approval:42", headgateworkflow.SignalEmission{ + Signal: "approved", IdempotencyKey: "approval:ticket-1842", + Payload: json.RawMessage(`{"approved":true}`), + Source: json.RawMessage(`{"emitter":"admin-api","actor":"operator-42"}`), +}) +``` + +`list_signals` / `ListSignals` and `GET /api/v1/workflows/{id}/signals` return the newest +100 emissions. Payload is capped at 64 KiB and source at 16 KiB. Trimming also ends the +idempotency guarantee for the removed key. `source` is descriptive, not independently +authenticated; deployments should derive it at their trusted API boundary. + +Expressions are limited to 1,024 bytes and must compile before enqueue. They cannot make +network or store calls. Evaluation is bounded by the workflow's node/edge limits and the +CEL implementation; the expression result must be boolean. + +## Timers + +Absolute timers are ordinary scheduled jobs. Relative timers remain pending until all +dependencies complete. The coordinator records their store-stamped finalization times, +uses the latest timestamp as the anchor, and atomically changes the timer from `pending` +to `scheduled` at `anchor + delay`. A worker clock is never used. + +```rust +let workflow = Workflow::new("follow-up:42") + .add("prepare", prepare_job, Vec::::new()) + .add_timer_after("cooldown", Duration::from_secs(1800), ["prepare"])? + .add("notify", notify_job, ["cooldown"]); +``` + +```go +workflow := headgateworkflow.New("follow-up:42") +workflow.Add("prepare", prepareJob) +if err := workflow.AddTimerAfter("cooldown", 30*time.Minute, "prepare"); err != nil { return err } +workflow.Add("notify", notifyJob, "cooldown") +``` + +## Graph grafts + +`WorkflowGraft::new(id, expected_revision)` and `NewGraft(id, expectedRevision)` return +one atomic batch: `{workflow}:graft:{next_revision}` plus the new pending jobs. The +coordinator validates the combined graph, fences it into its checkpoint, and only then +releases the receipt. Competing writers for the same revision collide on the deterministic +receipt ID. Grafts are additive, accept ordinary tasks only, and cannot revive a terminal +coordinator. + +### Why grafts cannot add control-flow nodes + +Signals, timers, CEL conditions, and child-workflow links can be part of the initial graph, +but cannot currently be appended through a graft. Rust exposes only +`WorkflowGraft::add`; Go exposes only `WorkflowGraft.Add`; and the HTTP graft schema accepts +ordinary task envelopes. Encoding an internal node kind manually is invalid and is not a +supported escape hatch. + +The distinction is behavioral rather than cosmetic. A task graft adds a pending job and +dependency edges. A signal additionally needs durable buffered-delivery identity; a +relative timer needs a store-clock completion anchor; a condition needs bounded CEL +compilation and evaluation; and a child link changes the cross-workflow graph, including +cycle detection, atomic creation, cancellation, and retry propagation. Those rules must be +validated against the combined graph, replay safely after interruption, enter workflow +history, and behave identically in Rust and Go across PostgreSQL, Redis, and MySQL. + +Until that versioned mutation contract exists, declare every control-flow node in the +initial workflow and use grafts only for ordinary tasks that depend on existing nodes. If +execution discovers that it needs a new signal, timer, condition, or child workflow link, +start the revised graph under a new workflow ID. Enqueuing another workflow separately does +not mutate or attach it to the original execution. + +## Accepted graph immutability + +Once a workflow revision is accepted, its existing nodes and dependency edges are +immutable—even while the coordinator is active. Operators cannot replace a node's job, +rename or remove a node, add or remove one of its dependencies, or insert a task by +rewiring an existing edge. A node may already be running or completed, so rewriting it +would make the durable history disagree with the execution that actually occurred. + +The supported extension is an additive, revision-checked graft. A graft may append new +ordinary task nodes and connect them to existing nodes, but it does not alter the existing +subgraph. If the desired change requires replacing or rewiring accepted work, create a new +workflow ID with the new graph. Headgate does not plan to infer downstream invalidation, +rollback completed side effects, or silently reinterpret an in-flight execution. + +## Terminal workflow immutability + +A workflow's accepted graph becomes permanently immutable when its coordinator reaches a +terminal outcome. Operators cannot append, replace, remove, rename, or rewire nodes on that +workflow, and a signal or graft cannot turn the terminal execution back into a live one. +This preserves the meaning of its terminal result and keeps its event history auditable. + +Failed-subgraph retry is a deliberately separate operation, not a graph mutation. It is +available only when retry was enabled before enqueue, requires the failed revision, advances +the workflow generation, and reuses the same graph while preserving completed ancestors. +Starting different work from a terminal workflow requires a new workflow ID; a future fork +operation, if added, would likewise create a distinct execution rather than rewrite history. + +## Parent and child workflows + +Use `prepare_bundle` / `PrepareBundle` when creating related workflows. It requires every +child link to resolve inside the bundle, rejects cycles across workflow boundaries, and +returns one batch for one atomic `enqueue` call. Separately enqueued children remain +supported, but only a complete bundle can prove global acyclicity and atomic creation. + +Cancellation visits live jobs in every active parallel branch. Child propagation defaults +to `true` in the HTTP API and can be explicitly disabled. Traversal is iterative and +bounded; a retry of a failed child-link also requests the child's failed-subgraph retry +before reopening the parent link. + +## Retry and repair + +Manual retry must be enabled with `failed_subgraph_retry`. Automatic retry also enables +that behavior and declares a generation limit plus store-timed backoff: + +```rust +let workflow = Workflow::new("import:42") + .automatic_retry(3, Duration::from_secs(30))? + .add("download", download_job, Vec::::new()) + .add("index", index_job, ["download"]); +``` + +```go +workflow := headgateworkflow.New("import:42") +if err := workflow.AutomaticRetry(3, 30*time.Second); err != nil { return err } +workflow.Add("download", downloadJob) +workflow.Add("index", indexJob, "download") +``` + +The generation limit includes the first run. Retry increments graph revision and +generation, reopens only the failed subgraph, and never reruns completed ancestors. +Quarantined nodes require `release_quarantine: true`; release is fingerprint-wide because +that is the underlying quarantine contract. Undecodable nodes require replacement payload +bytes and a positive schema version before operator retry. Recovery is replay-safe if a +request stops between repair and coordinator reopening. + +## Control API and history + +The Rust crate and Go package expose graph inspection independently of the console. One +snapshot reads the accepted base graph plus grafts and joins each node to its current job +state without returning application payloads. Query the snapshot repeatedly when several +topology questions are needed; the convenience functions perform a fresh snapshot read. + +```rust +let graph = headgate_workflow::inspect_workflow(store.as_ref(), "import:42").await?; +let publish = graph.node("publish").ok_or("publish node missing")?; +let prerequisites = graph.dependencies("publish").unwrap(); + +// Convenience point reads are useful when only one relation is needed. +let downstream = headgate_workflow::workflow_dependents( + store.as_ref(), + "import:42", + "download", +).await?; + +let page = headgate_workflow::list_workflows(store.as_ref(), None, 50).await?; +``` + +```go +graph, err := headgateworkflow.InspectWorkflow(ctx, store, "import:42") +if err != nil { return err } +publish := graph.Node("publish") +prerequisites, ok := graph.Dependencies("publish") + +// Convenience point reads are useful when only one relation is needed. +downstream, err := headgateworkflow.WorkflowDependents(ctx, store, "import:42", "download") + +page, err := headgateworkflow.ListWorkflows(ctx, store, "", 50) +``` + +Each node reports its workflow-local name, underlying job ID and kind, durable role, +current state, immediate dependencies and dependents, virtual-node configuration, and +recorded completion time. The snapshot also reports coordinator state, graph revision, +retry generation, failure status, and configured retry policy. A retained completion is +reported as completed even if retention has removed its job row; an unrecorded missing row +is reported as `missing`. + +The following routes share the normal API authorization boundary. Mutations require a +non-empty `Idempotency-Key`: + +```text +GET /api/v1/workflows +GET /api/v1/workflows/{id} +GET /api/v1/workflows/{id}/events +GET /api/v1/workflows/{id}/signals +GET /api/v1/workflows/{id}/nodes/{node} +GET /api/v1/workflows/{id}/nodes/{node}/dependencies +GET /api/v1/workflows/{id}/nodes/{node}/dependents +POST /api/v1/workflows/{id}/signals +POST /api/v1/workflows/{id}/grafts +POST /api/v1/workflows/{id}/retry +POST /api/v1/workflows/{id}/cancel +``` + +While the coordinator is active, history lives in its fence-verified checkpoint and is +mirrored through the same fence-gated output write. Terminal completion clears the active +cursor, so history reads fall back to that durable output copy. It records starts, node +completions, graft/retry decisions, automatic retry scheduling, and terminal outcome. Only +the newest 256 events are retained; sequence numbers remain monotonic after trimming. + +## Resource and behavior limits + +- 999 nodes and 10,000 dependency edges per workflow; +- 1,000 jobs per atomic workflow bundle; +- 1,024 bytes per CEL expression; +- newest 256 workflow events; +- newest 100 signal emissions per workflow, including 64 KiB payload and 16 KiB source; +- cancellation/child traversal is bounded by the workflow node limit; +- active cancellation targets all live branches, not only the coordinator; +- grafts do not replace/delete nodes and currently contain ordinary tasks only; +- accepted nodes and dependency edges are immutable across revisions; and +- terminal workflow graphs and their recorded outcomes are immutable. + +The six-cell integration scenario is defined for PostgreSQL, Redis, and MySQL in both +Rust and Go. It exercises an early signal, CEL wait, dependency-anchored relative timer, +automatic failed-subgraph retry, preserved execution order, and durable history. Cells +run when their `HG_TEST_PG`, `HG_TEST_REDIS`, or `HG_TEST_MYSQL` environment is present. + +## Still intentionally excluded + +The dynamic backend is implemented, but UI mutation controls are deliberately deferred +for review. Grafting signals, timers, conditions, or child workflows; in-place mutation of +accepted nodes; and unbounded/full-lifetime event history are not supported. diff --git a/docs/workflows.md b/docs/workflows.md index 7e32e97..48852c6 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -1,66 +1,54 @@ # Workflows and DAG dependencies -Workflows are an opt-in layer, not part of `headgate-core`. Rust uses the -`headgate-workflow` crate; Go uses `github.com/mujhtech/headgate/go/headgateworkflow`. - -A workflow builder validates unique task names, missing dependencies, repeated edges, -and cycles before anything is enqueued. `prepare` returns one batch containing: - -- one ordinary `headgate:workflow` coordinator job; and -- every application job in the durable `pending` state. - -Enqueue that batch through the normal client/store path. Workers serving the coordinator -queue must install `register_coordinator` / `RegisterCoordinator` and also register the -application task handlers. - -Runnable fan-out/fan-in construction examples are available for -[Rust](../examples/rust/src/bin/workflow.rs) and [Go](../examples/go/workflow/main.go). -They validate the graph and print the atomic coordinator-plus-children batch without -requiring a database. The live coordinator execution proof is -[`crates/headgate-workflow/tests/live.rs`](../crates/headgate-workflow/tests/live.rs). - -The coordinator performs bounded point reads—one per graph node. Roots are promoted -first. A node is promoted only after every dependency is `completed`; fan-out and fan-in -therefore use the same mechanism. While work is active the coordinator snoozes without -consuming an attempt. When all nodes complete it completes normally. If an upstream job -archives, is cancelled, quarantined, becomes undecodable, is revoked, or disappears, any -still-pending descendants are deleted before they can run and the coordinator archives. - -Every child's retention is raised to at least the workflow retention (seven days by -default). The coordinator also records observed completions in its own fenced checkpoint -before promoting descendants. That completion evidence survives an early child's retention -expiry, so a long retry cannot turn work that already succeeded into a missing dependency. -An unrecorded missing child still fails the workflow because the coordinator has no durable -proof that it completed. - -Retention is measured from each child's own finalization time, not from workflow completion. -To keep every child detail visible after a long workflow finishes, configure at least the -expected maximum workflow runtime plus the desired post-completion inspection window. The -checkpoint evidence protects dependency correctness; it is not a replacement for the -expired child's payload, logs, result, or attempt history. - -The runner renews the lease of a long-running child, but it cannot renew while its host is -suspended or disconnected. Reclaiming that expired lease and incrementing the crash count -is expected. Long stages should use `JobCtx::step_cursor` / `headgate.StepCursor` and save -their cursor after each safely repeatable unit. Cursor writes are fence-verified, so the -expired holder stops and the replacement attempt resumes from the last accepted cursor -instead of restarting the full stage. - -## Deliberate boundaries - -The topology is immutable after the atomic enqueue. This first slice supports durable -DAG dependencies, fan-out/fan-in, failure propagation, and read-only graph inspection. -It does not claim River Pro's signals, timers, CEL wait expressions, dynamic -grafting/appending, nested workflows, or workflow retry. Those can layer on later -without moving policy evaluation into workers or changing the admission gate. - -The embedded operations console now provides read-only graph inspection at `/workflows`. -It decodes the immutable coordinator payload and reads each child through the ordinary -bounded job-detail API. This is an operator view, not a new workflow control surface: -signals, timers, graph mutation, workflow-level retry, and workflow-level cancellation -remain outside this slice. - -Pending jobs cannot be operator-cancelled by the current core transition table. Failed -dependency propagation therefore deletes descendants that have never run rather than -inventing a new transition. The archived coordinator remains the durable workflow-level -failure record. +Workflows are an opt-in orchestration layer: Rust uses `headgate-workflow`, and Go uses +`github.com/mujhtech/headgate/go/headgateworkflow`. Core admission remains responsible only +for deciding which ordinary jobs may run. + +`prepare` validates names, dependencies, edges, and cycles, then returns one atomic batch +containing a coordinator and pending application jobs. Install `register_coordinator` / +`RegisterCoordinator` on workers that serve the coordinator queue. + +The coordinator performs bounded point reads. Roots are promoted first; fan-out and +fan-in follow only after durable completion evidence exists. It copies each child's +store-stamped `finalized_at_ms` into its fenced cursor before promoting descendants, so +retention of an old child row cannot erase dependency correctness. + +Every child is retained for at least the workflow retention (seven days by default). +Retention still begins at that child's finalization. If operators need payload, logs, and +attempt history after a long workflow completes, configure expected maximum workflow +duration plus the desired inspection window. + +Long-running jobs need normal lease renewal. A sleeping laptop or disconnected host cannot +renew; reclaim then cancels the stale handler and increments its crash count. Use resumable +steps and persist a cursor after each safely repeatable unit. Fence verification prevents +an expired attempt from advancing the cursor or acknowledging success. + +The dynamic feature contract—signals, timers, CEL waits, grafts, nested workflow bundles, +retry/repair, cancellation, API routes, event history, limits, and examples—is documented +in [Dynamic workflow experiments](workflow-experiments.md). The embedded console currently +inspects the merged graph, revision, generation, and task details but intentionally exposes +no workflow mutation controls while that interaction is reviewed. + +Inspection is also a first-class library and HTTP capability, not a console-only feature. +`list_workflows` / `ListWorkflows` pages coordinators, while `inspect_workflow` / +`InspectWorkflow` returns the accepted graph and current execution state; node, +dependency, and dependent helpers answer topology questions without parsing coordinator +payloads. HTTP clients can use `GET /api/v1/workflows`, `GET /api/v1/workflows/{id}`, and its +`nodes/{node}` relationship subresources. These reads never expose application payloads. + +Terminal workflow executions are immutable. Their accepted graph and recorded outcome +cannot be rewritten or extended; new work requires a new workflow ID. An explicitly +preconfigured failed-subgraph retry advances the generation of the unchanged graph and is +not treated as graph mutation. + +Existing nodes and edges are also immutable while a workflow is active. Dynamic extension +means appending ordinary tasks through a revision-checked graft, not replacing, renaming, +removing, or rewiring work already accepted by the store. Changes that require a different +existing graph belong to a new workflow ID. + +Grafts cannot currently add signals, timers, CEL conditions, or child-workflow links. +Those nodes carry durable orchestration rules beyond an ordinary pending job—buffered +delivery, store-clock anchoring, expression evaluation, or cross-workflow cycle and +propagation behavior—and must be present in the initial graph. Plan control-flow nodes up +front and graft only ordinary tasks that depend on them. If execution needs an unforeseen +control-flow node, start the revised graph with a new workflow ID. From 36d0abf08f47b6c031333034b44ed42419f88f0a Mon Sep 17 00:00:00 2001 From: Muhideen Mujeeb Adeoye Date: Sat, 5 Sep 2026 13:50:47 +0100 Subject: [PATCH 4/9] =?UTF-8?q?=E2=9C=A8=20feat(ui):=20enrich=20workflow?= =?UTF-8?q?=20inspection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ui/package.json | 1 + ui/pnpm-lock.yaml | 15 + ui/public/favicon.svg | 17 + ui/src/components/app-sidebar.tsx | 10 +- ui/src/components/workflow-graph.test.ts | 24 ++ ui/src/components/workflow-graph.tsx | 283 +++++++++------- ui/src/components/workflow-node-detail.tsx | 319 ++++++++++++++++++ ui/src/lib/workflow.test.ts | 149 ++++++++ ui/src/lib/workflow.ts | 213 +++++++++++- ui/src/routes/__root.tsx | 5 +- .../_console.workflows_.$workflowId.tsx | 1 + ui/src/views/jobs.tsx | 7 + ui/src/views/workflows.tsx | 67 ++-- 13 files changed, 945 insertions(+), 166 deletions(-) create mode 100644 ui/public/favicon.svg create mode 100644 ui/src/components/workflow-node-detail.tsx diff --git a/ui/package.json b/ui/package.json index 306e9d6..947b5ad 100644 --- a/ui/package.json +++ b/ui/package.json @@ -13,6 +13,7 @@ }, "dependencies": { "@base-ui/react": "^1.7.0", + "@dagrejs/dagre": "^3.1.1", "@fontsource-variable/geist": "^5.3.0", "@tailwindcss/vite": "^4.3.3", "@tanstack/react-query": "^5.102.8", diff --git a/ui/pnpm-lock.yaml b/ui/pnpm-lock.yaml index 0548055..0ba0c26 100644 --- a/ui/pnpm-lock.yaml +++ b/ui/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@base-ui/react': specifier: ^1.7.0 version: 1.7.0(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8) + '@dagrejs/dagre': + specifier: ^3.1.1 + version: 3.1.1 '@fontsource-variable/geist': specifier: ^5.3.0 version: 5.3.0 @@ -316,6 +319,12 @@ packages: resolution: {integrity: sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==} engines: {node: '>=20.19.0'} + '@dagrejs/dagre@3.1.1': + resolution: {integrity: sha512-zroZB1dFOFiGgv4Xcrn1DckB1o4aOikPqD2NDQPV0WM//CXGcS6xiD0rNkqHmw6FEg4tabt4nxPLwgCWT+Vb2A==} + + '@dagrejs/graphlib@4.0.5': + resolution: {integrity: sha512-7xrBTqIts3o+PMUZX97wSc+7TUbW+/rULzGNCTP6yooNVDXbzw4Wutg/H/xOutTB/c/k0YqOAavgPh4/Zk9PFA==} + '@exodus/bytes@1.15.1': resolution: {integrity: sha512-S6mL0yNB/Abt9Ei4tq8gDhcczc4S3+vQ4ra7vxnAf+YHC02srtqxKKZghx2Dq6p0e66THKwR6r8N6P95wEty7Q==} engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0} @@ -2256,6 +2265,12 @@ snapshots: '@csstools/css-tokenizer@4.0.0': {} + '@dagrejs/dagre@3.1.1': + dependencies: + '@dagrejs/graphlib': 4.0.5 + + '@dagrejs/graphlib@4.0.5': {} + '@exodus/bytes@1.15.1': {} '@floating-ui/core@1.8.0': diff --git a/ui/public/favicon.svg b/ui/public/favicon.svg new file mode 100644 index 0000000..93a8896 --- /dev/null +++ b/ui/public/favicon.svg @@ -0,0 +1,17 @@ + + Headgate + + + diff --git a/ui/src/components/app-sidebar.tsx b/ui/src/components/app-sidebar.tsx index ce7022a..b0e0d98 100644 --- a/ui/src/components/app-sidebar.tsx +++ b/ui/src/components/app-sidebar.tsx @@ -67,9 +67,13 @@ export function AppSidebar(props: React.ComponentProps) { size="lg" tooltip="headgate" > -
- h -
+
headgate operations console diff --git a/ui/src/components/workflow-graph.test.ts b/ui/src/components/workflow-graph.test.ts index a76200f..0088d39 100644 --- a/ui/src/components/workflow-graph.test.ts +++ b/ui/src/components/workflow-graph.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { buildWorkflowGraph, type WorkflowGraphItem, + workflowGraphInitialView, workflowGraphLayers, } from "@/components/workflow-graph"; @@ -104,4 +105,27 @@ describe("workflow graph", () => { }); expect(graph.edges[0]?.style).toMatchObject({ stroke: "var(--success)" }); }); + + it("opens every graph as a topology overview", () => { + const smallGraph = buildWorkflowGraph(items, "wf"); + expect(workflowGraphInitialView(smallGraph, items.length).fitView).toBe( + true + ); + + const largeItems = Array.from({ length: 14 }, (_, index) => ({ + deps: index === 0 ? [] : [`task-${index - 1}`], + job: { + id: `wf:task-${index}`, + kind: "demo:step", + state: index === 8 ? "running" : index < 8 ? "completed" : "pending", + }, + job_id: `wf:task-${index}`, + name: `task-${index}`, + })); + const largeGraph = buildWorkflowGraph(largeItems, "wf"); + const initialView = workflowGraphInitialView(largeGraph, largeItems.length); + + expect(initialView.fitView).toBe(true); + expect(initialView.viewport).toEqual({ x: 24, y: 24, zoom: 1 }); + }); }); diff --git a/ui/src/components/workflow-graph.tsx b/ui/src/components/workflow-graph.tsx index c451721..63e5586 100644 --- a/ui/src/components/workflow-graph.tsx +++ b/ui/src/components/workflow-graph.tsx @@ -1,3 +1,4 @@ +import dagre from "@dagrejs/dagre"; import { useNavigate } from "@tanstack/react-router"; import { Background, @@ -5,7 +6,6 @@ import { Controls, type Edge, Handle, - MarkerType, MiniMap, type Node, type NodeMouseHandler, @@ -14,11 +14,16 @@ import { Position, ReactFlow, } from "@xyflow/react"; -import { CheckCircle2Icon, CircleDashedIcon } from "lucide-react"; +import { + CheckCircle2Icon, + CircleDashedIcon, + Clock3Icon, + GitBranchIcon, + RadioIcon, + WorkflowIcon, +} from "lucide-react"; import { memo, useCallback, useMemo } from "react"; -import { Badge } from "@/components/ui/badge"; - interface WorkflowGraphJob { id: string; kind: string; @@ -29,6 +34,7 @@ export interface WorkflowGraphItem { deps: string[]; job: WorkflowGraphJob | null; job_id: string; + kind?: "task" | "signal" | "timer" | "child_workflow" | "condition"; name: string; recordedCompletion?: boolean; } @@ -37,26 +43,21 @@ interface TaskNodeData extends Record { dependencyText: string; inspectable: boolean; jobId: string; - kind: string; + jobKind: string; name: string; + nodeKind: "task" | "signal" | "timer" | "child_workflow" | "condition"; selected: boolean; state: string; workflowId: string; } -interface StageNodeData extends Record { - label: string; -} - type TaskNode = Node; -type StageNode = Node; -type WorkflowNode = TaskNode | StageNode; +type WorkflowNode = TaskNode; -const nodeWidth = 260; -const nodeHeight = 124; -const stageGap = 104; -const nodeGap = 28; -const taskTop = 44; +const nodeWidth = 176; +const nodeHeight = 52; +const rankGap = 34; +const nodeGap = 18; const failedStates = new Set([ "archived", @@ -153,72 +154,73 @@ export function buildWorkflowGraph( workflowId: string, selectedJobId?: string ) { - const layers = workflowGraphLayers(items); - const largestStage = Math.max( - 1, - ...layers.map(([, stageItems]) => stageItems.length) - ); - const graphHeight = - largestStage * nodeHeight + Math.max(0, largestStage - 1) * nodeGap; + const layout = new dagre.graphlib.Graph().setDefaultEdgeLabel(() => ({})); + layout.setGraph({ + align: "UL", + marginx: 24, + marginy: 24, + nodesep: nodeGap, + rankdir: "LR", + ranker: "network-simplex", + ranksep: rankGap, + }); + for (const item of items) { + layout.setNode(item.name, { height: nodeHeight, width: nodeWidth }); + } + const itemNames = new Set(items.map((item) => item.name)); + for (const item of items) { + for (const dependency of item.deps) { + if (itemNames.has(dependency)) { + layout.setEdge(dependency, item.name); + } + } + } + dagre.layout(layout); + const positioned = new Map(); const nodes: WorkflowNode[] = []; - - layers.forEach(([level, stageItems], stageIndex) => { - const x = stageIndex * (nodeWidth + stageGap); - const stageHeight = - stageItems.length * nodeHeight + - Math.max(0, stageItems.length - 1) * nodeGap; - const startY = taskTop + (graphHeight - stageHeight) / 2; - - nodes.push({ - data: { label: `Stage ${level + 1}` }, + for (const item of items) { + const point = layout.node(item.name); + const state = + item.job?.state ?? (item.recordedCompletion ? "completed" : "missing"); + const blockedBy = item.deps.filter((dependency) => { + const candidate = items.find( + (candidateItem) => candidateItem.name === dependency + ); + return ( + candidate?.job?.state !== "completed" && !candidate?.recordedCompletion + ); + }); + const dependencyText = item.deps.length + ? blockedBy.length + ? `Waiting for ${blockedBy.join(", ")}` + : `${item.deps.length} ${item.deps.length === 1 ? "dependency" : "dependencies"} satisfied` + : "Root task"; + const node: TaskNode = { + data: { + dependencyText, + inspectable: item.job !== null, + jobId: item.job_id, + jobKind: item.job?.kind ?? item.job_id, + name: item.name, + nodeKind: item.kind ?? "task", + selected: selectedJobId === item.job_id, + state, + workflowId, + }, draggable: false, focusable: false, - id: `stage:${level}`, - position: { x, y: 0 }, + id: item.name, + position: { + x: point.x - nodeWidth / 2, + y: point.y - nodeHeight / 2, + }, selectable: false, - type: "stage", - }); - - stageItems.forEach((item, itemIndex) => { - const state = - item.job?.state ?? (item.recordedCompletion ? "completed" : "missing"); - const blockedBy = item.deps.filter((dependency) => { - const candidate = items.find( - (candidateItem) => candidateItem.name === dependency - ); - return ( - candidate?.job?.state !== "completed" && - !candidate?.recordedCompletion - ); - }); - const dependencyText = item.deps.length - ? blockedBy.length - ? `Waiting for ${blockedBy.join(", ")}` - : `${item.deps.length} ${item.deps.length === 1 ? "dependency" : "dependencies"} satisfied` - : "Root task"; - const node: TaskNode = { - data: { - dependencyText, - inspectable: item.job !== null, - jobId: item.job_id, - kind: item.job?.kind ?? item.job_id, - name: item.name, - selected: selectedJobId === item.job_id, - state, - workflowId, - }, - draggable: false, - focusable: false, - id: item.name, - position: { x, y: startY + itemIndex * (nodeHeight + nodeGap) }, - selectable: false, - type: "task", - }; - positioned.set(item.name, node); - nodes.push(node); - }); - }); + type: "task", + }; + positioned.set(item.name, node); + nodes.push(node); + } const edges: Edge[] = items.flatMap((target) => target.deps.flatMap((dependency) => { @@ -234,22 +236,16 @@ export function buildWorkflowGraph( animated: running, focusable: false, id: `${dependency}:${target.name}`, - markerEnd: { - color, - height: 16, - type: MarkerType.ArrowClosed, - width: 16, - }, selectable: false, source: dependency, style: { - opacity: satisfied ? 0.78 : 0.55, + opacity: satisfied ? 0.7 : 0.42, stroke: color, - strokeDasharray: satisfied ? undefined : "6 5", - strokeWidth: 2, + strokeDasharray: satisfied ? undefined : "4 4", + strokeWidth: 1.25, }, target: target.name, - type: "bezier", + type: "smoothstep", }, ]; }) @@ -258,51 +254,80 @@ export function buildWorkflowGraph( return { edges, nodes }; } +export function workflowGraphInitialView( + graph: ReturnType, + taskCount: number +) { + return { + fitView: taskCount > 0 && graph.nodes.length > 0, + viewport: { x: 24, y: 24, zoom: 1 }, + }; +} + +function NodeKindIcon({ kind }: { kind: TaskNodeData["nodeKind"] }) { + const className = "size-3 shrink-0"; + if (kind === "signal") { + return