diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47dff97..aac4214 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,3 +23,15 @@ jobs: - run: agent-ultra doctor - run: agent-ultra demo - run: agent-ultra classify "rm -rf build" + + leak-gate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install -e . + # Scan the whole tree plus the head commit message. No whitelist: + # a hit fails the build; the fix is to change the offending text. + - run: agent-ultra-leakgate . --message "$(git log -1 --pretty=%B)" diff --git a/CHANGELOG.md b/CHANGELOG.md index 58ddd9d..1b6e941 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,55 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ### Added +- **ultracode** (`agent_ultra.ultracode`) — deterministic multi-agent workflow + engine. Workflows are plain Python modules (`META` + `async def run(wf)`) that + fan work across bounded agents via `wf.agent` (one model call, optional + JSON-Schema validation with feedback retries), `wf.parallel` (barrier) and + `wf.pipeline` (no barrier), under hard `wf.budget` call/token ceilings, with + broker-gated `wf.run_check`. Every run writes a typed event stream + (`events.jsonl`), a replayable journal, per-agent artifacts, and a checksummed + receipt (`verify_receipt`). **Resume** replays completed calls from the + journal and spends budget only on new ones (0 calls when unchanged). A + terminal-safe status card renders from the journal (plain ASCII unless stdout + is a proven interactive UTF terminal; never crashes the run). CLI: + `agent-ultra ultracode run|list|status|resume`, with `--mock` for a fully + offline, keyless route. Bundled examples: `smoke` and `review` + (finders -> skeptic votes -> synthesis). `doctor` and `demo` now cover it; see + [docs/ultracode.md](docs/ultracode.md). 14 offline tests. +- **Receipts bus** (`agent_ultra.receipts_bus`). A unified, authenticated + receipt index over SQLite (WAL, default auto-checkpoint). Each receipt carries + two hashes over its canonical body: `receipt_sha256` (integrity) and a keyed + `receipt_hmac` (authenticity) — kept **separate**, so a hand-authored envelope + with a valid sha256 but no HMAC is `authentic=False` and rejected by + enforce-mode consumers. `kind`/`actor`/`verdict` are closed enums validated at + write time. A failed read raises `BusUnavailable` (never an empty list). + `resolve` + `binds` implement the claim-binding rule (cited / verify-command + with scope+freshness / task_ref; command-, manual-, repair-, route_health-kind + never complete; a verifier candidate binds only on `claim_sha256` match; weak + evidence yields a distinct `ledger-weak` clause). `append_audit` / + `verify_audit` maintain a hash-chained `gate_audit`. New CLI: + `agent-ultra receipts list|show|verify|why|attest` (`attest` is + interactive-only and writes `kind=manual`, which never satisfies completion). +- **Verifier** (`agent_ultra.verifier`). Refute-first claim verification with an + injected check-runner and judge (no network, no host commands by default). + Engine re-check wins when a verify command is declared (exit 0 confirms, + non-zero refutes); otherwise a refute-first `judge` runs, defaulting to + refuted on insufficient evidence; with neither channel, refuted. Emits a + `kind=verifier` receipt stamped with `claim_sha256`; a refuted verdict maps to + a non-acceptable receipt verdict so it can never satisfy a gate. `Verifier` + wraps it with a configurable escalation budget (per-session/per-window caps + + dedup on identical `claim_sha256`). +- **Leak gate** (`agent_ultra.leakgate`, `agent-ultra-leakgate`). Scans the + whole tree (and a commit message / PR body) against a base64 denylist decoded + at runtime, so the denylist file never contains plaintext. Case-insensitive + except a whole-word case-sensitive collision token; path separators + normalised; no file whitelist. Wired into CI as a dedicated job. + +### Changed +- Renamed the optional memory adapter to `agent_ultra.adapters.external_memory` + (`ExternalMemoryHooks`) and made the worker-layer leakage test consume the + shared base64 denylist, so no host-specific identifier remains in the tree. + - **Structural PANEL enforcement** (`agent_ultra.panel_receipt`). A phase labelled PANEL is no longer proof — only a valid panel execution receipt with real executed lenses satisfies it. The loop writes @@ -95,7 +144,7 @@ First public release. Portable, stdlib-only core with thin optional adapters. - **Memory hooks** (`agent_ultra.memory`): five generic events, all no-ops by default; no memory system required. - **Adapters** (all optional, none imported by the core): generic CLI, - LiteLLM, Docker sandbox, Mneme-style memory, Hermes-style, Ktisis-style. + LiteLLM, Docker sandbox, external-memory, Hermes-style, Ktisis-style. - **Install layer**: `agent-ultra doctor` / `init` / `demo`, one-command installers for PowerShell / bash / CMD, `config.example.yaml`, `.env.example`, `INSTALL.md` with an AI-agent handoff prompt, and a troubleshooting guide. diff --git a/README.md b/README.md index 0a5b2ca..34c088d 100644 --- a/README.md +++ b/README.md @@ -106,12 +106,47 @@ agent-ultra doctor --live # probe the endpoint agent-ultra panel "Is this change production-safe?" --evidence-dir ./src ``` +## ultracode — deterministic multi-agent workflows + +Where the panel debates one question, **ultracode** runs a *script* that fans +work across many bounded agents and proves what happened. A workflow is a plain +Python module — `META` + `async def run(wf)` — using `wf.agent` (one model +call, optionally schema-validated), `wf.parallel` (barrier), `wf.pipeline` (no +barrier), `wf.budget` (hard call/token ceilings), and `wf.run_check` +(broker-gated host commands). Every run writes a replayable **journal** and a +checksummed **receipt**; a terminal status card renders from the journal, so a +model's own text can never fake progress. + +```console +$ agent-ultra ultracode run smoke --mock # offline, no API key +ULTRACODE +Run: 20260707T201444-smoke-18eb5f +Name: smoke +Phase: Chain +Agents: 4/4 complete +Failed: 0 +Blocked: 0 +Progress: [####] +Status: COMPLETE +Result: {"pings": ["pong-one", "pong-two"], "chained": [...], "calls_spent": 4} +Resume: agent-ultra ultracode resume 20260707T201444-smoke-18eb5f + +$ agent-ultra ultracode resume 20260707T201444-smoke-18eb5f --mock +Status: COMPLETE +Result: {..., "calls_spent": 0} # every call replayed from the journal +``` + +Commands: `ultracode run ` · `list` · `status` · `resume `. +Bundled workflows: **smoke** (fan-out + pipeline) and **review** (finders → +skeptic votes → synthesis). Full guide: [docs/ultracode.md](docs/ultracode.md). + ## What's in the box | module | what it does | |--------|--------------| | **panel** | Adversarial review: parallel critic *lenses* → steelman → judge cross-exam → synthesis. Verdicts: `real_now` / `real_later` / `theoretical` / `wrong`. Panel agents are roles, not models — one healthy route runs a whole panel. | | **ultra_loop** | build → test → panel → classify → fix → re-test → re-panel → **ship gate**. Red tests stop before the panel; low-context panels are refused; only proof-gated work ships. | +| **ultracode** | Deterministic multi-agent workflows: `META` + `async run(wf)` scripts fan work across bounded agents (`parallel`/`pipeline`) under hard budgets, with a resumable journal, a checksummed receipt, and a terminal-safe status card. Fan-out → journal → resume → receipt → status. | | **broker** | Every model-authored command classified SAFE / ELEVATED / DANGEROUS, ledgered, and executed only if its tier allows. DANGEROUS with no approval path is **denied by default**. | | **proof** | "Done" requires recorded evidence. Accepted findings become gates; `assert_shippable()` raises on unsupported completion claims. | | **evidence** | Bounded source gathering with secret redaction; low-context detection. | @@ -119,8 +154,8 @@ agent-ultra panel "Is this change production-safe?" --evidence-dir ./src | **artifacts** | Uniform JSON + Markdown run records and JSONL ledgers for every run. | | **memory** | Five generic write-back hooks (`on_panel_decision`, `on_finding_accepted`, `on_command_run`, `on_task_complete`, `on_lesson_learned`). No memory system required. | -Adapters (all optional): generic CLI, LiteLLM, Docker sandbox, memory -(Mneme-style), Hermes-style and Ktisis-style runtimes. +Adapters (all optional): generic CLI, LiteLLM, Docker sandbox, external +memory, Hermes-style and Ktisis-style runtimes. ### Workers: router (default) vs Deep Agents (optional) @@ -205,7 +240,8 @@ Full guide: [docs/adapter-guide.md](docs/adapter-guide.md). ## Docs [architecture](docs/architecture.md) · [panel](docs/panel.md) · -[ULTRA loop](docs/ultra-loop.md) · [command broker](docs/command-broker.md) · +[ULTRA loop](docs/ultra-loop.md) · [ultracode](docs/ultracode.md) · +[command broker](docs/command-broker.md) · [proof gates](docs/proof-gates.md) · [adapter guide](docs/adapter-guide.md) · [security](docs/security.md) · [troubleshooting](docs/troubleshooting.md) · [releasing](docs/releasing.md) · [INSTALL](INSTALL.md) · diff --git a/docs/adapter-guide.md b/docs/adapter-guide.md index ffcf233..b84f4ae 100644 --- a/docs/adapter-guide.md +++ b/docs/adapter-guide.md @@ -65,7 +65,7 @@ For the ULTRA loop, provide `builder(workspace, task) -> str` and | generic CLI | `adapters.cli` | run panel/ultra/classify from the shell | | LiteLLM | `adapters.litellm_routes` | build a pool over a LiteLLM proxy | | Docker sandbox | `adapters.docker_sandbox` | `sandbox_argv` for the broker | -| memory (Mneme-style) | `adapters.mneme_memory` | wire hooks to a memory system | +| external memory | `adapters.external_memory` | wire hooks to a memory system | | Hermes | `adapters.hermes` | env-driven pool + broker for Hermes-style agents | | Ktisis | `adapters.ktisis` | env-driven ULTRA loop for Ktisis-style build agents | diff --git a/docs/architecture.md b/docs/architecture.md index c9dbca0..aa26456 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,7 +21,7 @@ core imports an adapter; adapters may make runtime-specific assumptions. cli ── generic shell entrypoint (zero third-party deps) litellm_routes ── build a pool over a LiteLLM proxy docker_sandbox ── run non-approved commands in a container - mneme_memory ── wire hooks to a memory system + external_memory ── wire hooks to a memory system hermes / ktisis ── env-driven wiring for those runtimes ``` diff --git a/docs/receipts-bus.md b/docs/receipts-bus.md new file mode 100644 index 0000000..fd67fc6 --- /dev/null +++ b/docs/receipts-bus.md @@ -0,0 +1,85 @@ +# Receipts bus + +A unified, **authenticated** receipt index. Every unit of agent work can leave +a signed receipt — a record of what was done, what it claimed, and what +evidence backs the claim. The bus stores receipts, verifies them, and resolves +a claim to the receipts that actually *bind* to it. + +```python +from agent_ultra.receipts_bus import ReceiptsBus, Candidate + +bus = ReceiptsBus(db_path="receipts.db", key_path="install.key") + +env = bus.append(kind="panel", actor="engine_a", verdict="shipped", + session_id="s1", workspace="/repo") + +bus.verify(env["receipt_id"]) +# -> {ok, authentic, integrity, refs_ok, receipt} +``` + +## Authenticity is separate from integrity + +Two hashes cover the canonical body (the envelope minus its two hash fields, +`json.dumps(sort_keys=True)`): + +- **`receipt_sha256`** — SHA-256. Integrity only: proves the body was not + altered. +- **`receipt_hmac`** — HMAC-SHA256 keyed by a per-install key read from + `key_path`. This is the authenticity control. + +A hand-authored envelope with a correct `receipt_sha256` but no/invalid HMAC is +`authentic=False`, and any enforce-mode consumer must reject it (`ok=False`). + +> **Key custody.** The host is responsible for fencing `key_path` from +> untrusted writers. Honest residual: a same-user process that can *read* the +> key can still forge an authentic receipt — closing that needs an +> out-of-process signer, which is out of scope for v1. + +## A failed read is never empty + +The store is SQLite in WAL mode with the **default** auto-checkpoint left in +place. A read that fails after a bounded retry raises `BusUnavailable` — it +never degrades to `[]`, because a caller cannot tell "no receipts" from "the +store was unreachable" and would wrongly conclude a claim is unsupported. + +## Binding rule + +`resolve(claim_ctx)` returns origin-annotated `Candidate`s scoped by +session / workspace / window (optionally federating external sources via +injected readers). `binds(claim_ctx, candidate) -> (bool, clause)` decides +which candidates back a **completion** claim: + +- an explicitly **cited** receipt id, or +- a canonical **verify-command** match with scope coverage **and** freshness, or +- a **task_ref** match. + +Never binds: a `command`-kind candidate to a completion claim alone; a +`verifier`-kind candidate unless its `claim_sha256` equals the claim's; +`manual` / `repair` / `route_health` to completion; anything with a +non-acceptable verdict; anything unauthenticated. Weak-but-present evidence +(uncovered / stale) returns the distinct `ledger-weak` clause — not a pass. + +## Gate audit (hash chain) + +```python +bus.append_audit("ship", "enforce", "allow", "engine_a", "tests green") +bus.verify_audit("ship") # -> {ok, first_broken} +bus.get_audit_event(event_id) # explain one row +``` + +Each row carries `prev_sha256` + `row_sha256`; `verify_audit` recomputes the +chain and reports the first broken link. + +## CLI + +``` +agent-ultra receipts list [--session S] [--workspace W] +agent-ultra receipts show RECEIPT_ID +agent-ultra receipts verify RECEIPT_ID +agent-ultra receipts why [--claim-sha ...] [--verify-command ...] [--cited ...] +agent-ultra receipts attest # interactive only; writes kind=manual +``` + +`attest` is the only creation path outside engine code. It refuses +non-interactive use and writes a `kind=manual` receipt — and manual receipts +never satisfy a completion gate. diff --git a/docs/ultracode.md b/docs/ultracode.md new file mode 100644 index 0000000..1fa3c2f --- /dev/null +++ b/docs/ultracode.md @@ -0,0 +1,195 @@ +# ultracode — deterministic multi-agent workflows + +`ultracode` runs an operator-authored Python **workflow script** that fans work +across many bounded agents, journals every step, and proves what happened with +a checksummed receipt. It's the fan-out sibling of the panel (which debates one +question) and the ULTRA loop (which runs one task's build/test lifecycle). + +The pattern it proves, end to end: + +> **multi-agent fan-out → journal → resume → receipt → status** + +It is stdlib at the core. The model endpoint is any OpenAI-compatible URL, and +the offline **mock route needs no API key**, so everything below runs on a +fresh machine with zero setup. + +--- + +## Install (one command) + +```bash +pip install git+https://github.com/trollbot2012/agent-ultra-kit.git +``` + +Then prove it works — no key required: + +```bash +agent-ultra doctor # environment + all subsystems, incl. ultracode +agent-ultra demo # offline loop; must end with "DEMO PASSED" +``` + +`doctor` includes an `ultracode:` line proving fan-out → journal → resume +(cached) → valid receipt. `demo` runs the smoke workflow and resumes it. + +--- + +## First run (offline, then real) + +```bash +# offline, deterministic — works anywhere, no key +agent-ultra ultracode run smoke --mock + +# see what's available +agent-ultra ultracode list --mock + +# resume the run you just did — every call replays from the journal (0 calls) +agent-ultra ultracode resume --mock +``` + +Point it at a real model — any OpenAI-compatible endpoint (OpenAI, LiteLLM, +vLLM, Ollama, LM Studio). Routes and the key **env var name** come from +`agent-ultra.yaml` + environment; no secrets are ever written to config: + +```bash +agent-ultra init # writes agent-ultra.yaml + .env.example +export OPENAI_API_KEY=sk-... # or whatever env var your config names +agent-ultra ultracode run review --args '{"target": "./src"}' +``` + +`review` is the flagship example: parallel finders (one per lens) hunt problems +in `target`, findings are deduped, each faces independent skeptic votes, and +only findings a majority fail to refute survive to a synthesis. + +--- + +## Writing a workflow + +A workflow is a Python module with a `META` dict and an `async def run(wf)`: + +```python +META = {"name": "smoke", "description": "...", "phases": ["Ping", "Chain"]} + +async def run(wf): + wf.phase("Ping") + a, b = await wf.parallel([ + lambda: wf.agent("Reply with exactly: pong-one", label="ping-1"), + lambda: wf.agent("Reply with exactly: pong-two", label="ping-2"), + ]) + + wf.phase("Chain") + out = await wf.pipeline( + ["alpha", "bravo"], + lambda word: wf.agent( + f"Return JSON {{'upper': '{word}' uppercased}}.", + schema={"type": "object", + "properties": {"upper": {"type": "string"}}, + "required": ["upper"]}, + label=f"upper:{word}"), + lambda parsed, word: {"word": word, "upper": (parsed or {}).get("upper")}, + ) + return {"pings": [a, b], "chained": out} +``` + +Drop it in `~/.agent-ultra/ultracode/scripts/` (override the home with +`AGENT_ULTRA_HOME`) and run it by name, or pass any `.py` path directly. + +### The `wf` surface + +| call | what it does | +|------|--------------| +| `await wf.agent(prompt, *, schema=None, route="", label="", evidence=None, max_tokens=None)` | One bounded agent = one model call. With `schema` (a JSON-Schema subset) the reply is validated and re-asked on mismatch; returns the parsed object, or the raw text without `schema`, or `None` on failure. | +| `await wf.parallel([thunk, ...])` | Run zero-arg async thunks concurrently and **wait for all** (barrier). A thunk that raises resolves to `None`. | +| `await wf.pipeline(items, stage1, stage2, ...)` | Run each item through all stages independently — **no barrier** between stages. Stages take `(prev[, item[, index]])`, sync or async. A stage that raises drops that item to `None`. | +| `wf.budget` | Hard ceilings: `wf.budget.calls`, `.remaining_calls()`. Exceeding `--max-calls`/`--token-budget` ends the run as **HOLD** with everything journaled. | +| `wf.run_check(command, reason="")` | Execute a host command through the risk-tiered broker in CRITIC mode — only SAFE (pure-read) commands auto-run; everything else is classified, ledgered, and returned as `requires_approval`. | +| `wf.phase(title, kind="")` | Mark a phase (drives the status card). `kind` of `verification`/`synthesis`/`fan_in` tags it explicitly; otherwise the title is matched heuristically. | +| `wf.evidence(paths, ...)` | Bounded, read-only file excerpts to ground a prompt. | +| `wf.log(msg)` | Append a progress line (also written to the journal). | +| `wf.args` | Whatever you passed via `--args` (JSON or bare text). | + +--- + +## Journal, resume, and the receipt (the source of truth) + +Every run writes to `~/.agent-ultra/ultracode/runs//`: + +- **`events.jsonl`** — typed runtime events (`agent_planned/deployed/completed/ + failed/blocked`, `phase_started`, terminal `ultracode_*`). The status card + replays *this* — a workflow's own text output can never move it. +- **`journal.jsonl`** — one line per agent call; the replay cache for resume. +- **`agents/aNNN.json`** — the full record (prompt hash, output, route, timing) + each event points at. +- **`receipt.json`** — a checksummed manifest: model-call count and per-agent + prompt/output hashes. `verify_receipt()` recomputes the checksum, so a + hand-edited receipt fails validation. +- **`report.json`** — the complete run report. + +**Resume** replays completed calls from the journal and spends budget only on +new or changed ones: + +```bash +agent-ultra ultracode resume # infers the workflow from the journal +``` + +A resumed run reports `0` model calls when nothing changed — proof the cache, +not the model, produced the result. + +--- + +## Status card (terminal-safe) + +`agent-ultra ultracode status` prints a card per in-progress run, rebuilt from +`events.jsonl`. It's **plain ASCII by default** and only uses richer glyphs +when stdout is provably an interactive UTF-capable terminal — redirected +output, `TERM=dumb`, CI, `NO_COLOR`, or `AGENT_ULTRA_PLAIN=1` all force plain, +and any rendering error degrades rather than crashing the run. + +``` +ULTRACODE +Run: 20260707T201444-review-9a1c55 +Phase: Verify +Agents: 6/12 complete +Failed: 0 +Blocked: 0 +Progress: [######......] +Status: RUNNING +``` + +`#` = completed, `.` = deployed/running, `!` = failed or blocked (in rich mode: +`●`, `○`, `⚠`). + +--- + +## Using it from Python + +```python +from agent_ultra.ultracode import UltracodeEngine, demo_pool, verify_receipt + +eng = UltracodeEngine(demo_pool()) # offline, keyless +report = eng.run_script(eng.resolve("smoke")) +print(report.final_state) # COMPLETE + +# resume replays with zero model calls +resumed = eng.run_script(eng.resolve("smoke"), resume_run_id=report.run_id) +assert resumed.model_calls == 0 +``` + +For a real endpoint, build a `RoutePool` over `OpenAIChatClient` (see +`agent_ultra.routes`) and pass it to `UltracodeEngine(pool)`. + +--- + +## Troubleshooting + +| symptom | fix | +|---------|-----| +| `no workflow named 'X'` | `agent-ultra ultracode list` to see valid names; or pass a `.py` path. | +| `run` hangs or errors on a real endpoint | Try `--mock` first to isolate the engine from the network, then `agent-ultra doctor --live` to probe your endpoint. | +| every agent fails: `all routes failed` | Your model routes are unreachable. Check `base_url`/routes in `agent-ultra.yaml` and that the key env var is set. `--mock` needs neither. | +| garbled dots/box characters | Force plain output: `AGENT_ULTRA_PLAIN=1` (or it's automatic when piped). | +| run ended as **HOLD** | It hit `--max-calls`/`--token-budget`. Raise the ceiling or narrow the workflow; partial results are journaled and resumable. | +| schema agent returns `None` | The model didn't produce valid JSON after retries — inspect `agents/aNNN.json` for its raw output; loosen the schema or the prompt. | +| where are my runs? | `~/.agent-ultra/ultracode/runs/` (or under `$AGENT_ULTRA_HOME`). `ultracode-log.jsonl` lists every run. | + +Everything above is exercised by `agent-ultra doctor` and the offline test +suite (`pytest -q`), so if those pass, the pattern works on your machine. diff --git a/docs/verifier.md b/docs/verifier.md new file mode 100644 index 0000000..76de8ee --- /dev/null +++ b/docs/verifier.md @@ -0,0 +1,51 @@ +# Verifier + +Refute-first verification. The verifier tries to **refute** a completion claim, +not to confirm it, and defaults to *refuted* whenever evidence is insufficient. +A claim only survives if something concrete backs it. + +```python +from agent_ultra.verifier import verify_claim + +# Engine re-check: a real command is the strongest evidence (no LLM). +verify_claim("tests pass", + declared_verify_cmd="pytest -q", + run_check=run_check) # exit 0 confirms, non-zero refutes + +# LLM refute: no declared command -> ask an injected judge with a +# refute-first prompt that defaults to refuted on insufficient evidence. +verify_claim("the fix works", judge=judge, refs=[...]) + +# Neither channel -> refuted (the honest default). +verify_claim("it works") +``` + +Both collaborators are **injected** — the kit runs no commands and makes no +network calls by default: + +- `run_check(cmd, cwd) -> {"exit_code": int, "ledger_ref": str}` +- `judge(prompt) -> {"refuted": bool, "confidence": float, "reason": str}` + +## Result and receipt + +`verify_claim` returns `{refuted, confidence, reason, checks_run, +claim_sha256[, receipt_id]}`. When `emit_receipt` is supplied it stamps a +`kind=verifier` receipt with the claim's `claim_sha256`, so the receipt can +only ever resolve the claim it actually checked. A **refuted** verdict maps to +a non-acceptable receipt verdict (`failed`), so it can never satisfy a gate. + +## Budget + +`Verifier.escalate` is the budgeted entry point. Identical claims (same +`claim_sha256`) are answered from cache and cost nothing; fresh claims are +charged against configurable per-session and per-window caps and raise +`BudgetExceeded` past a cap. + +```python +from agent_ultra.verifier import Verifier, EscalationBudget + +v = Verifier(run_check=run_check, judge=judge, + budget=EscalationBudget(per_session=20, per_window=5, + window_seconds=60)) +v.escalate("tests pass", session_id="s1", declared_verify_cmd="pytest -q") +``` diff --git a/pyproject.toml b/pyproject.toml index 89ea6e0..86f51fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ Homepage = "https://github.com/trollbot2012/agent-ultra-kit" [project.scripts] agent-ultra = "agent_ultra.adapters.cli:main" +agent-ultra-leakgate = "agent_ultra.leakgate:main" [tool.setuptools.packages.find] where = ["src"] diff --git a/src/agent_ultra/__init__.py b/src/agent_ultra/__init__.py index e5ef25f..afce16b 100644 --- a/src/agent_ultra/__init__.py +++ b/src/agent_ultra/__init__.py @@ -25,3 +25,17 @@ from .ultra_loop.loop import UltraLoop, UltraReport # noqa: F401 from .proof.gates import ProofGate, GateSet, ProofError # noqa: F401 from .memory.hooks import MemoryHooks, CompositeHooks, JsonlHooks # noqa: F401 +from .receipts_bus import ( # noqa: F401 + ReceiptsBus, + BusUnavailable, + Candidate, + build_envelope, + EnvelopeError, +) +from .verifier import ( # noqa: F401 + verify_claim, + Verifier, + VerifierResult, + EscalationBudget, + BudgetExceeded, +) diff --git a/src/agent_ultra/adapters/__init__.py b/src/agent_ultra/adapters/__init__.py index cbd8d2d..588c16d 100644 --- a/src/agent_ultra/adapters/__init__.py +++ b/src/agent_ultra/adapters/__init__.py @@ -5,5 +5,5 @@ dependencies. Import the one you need directly, e.g.: from agent_ultra.adapters.litellm_routes import litellm_pool - from agent_ultra.adapters.mneme_memory import MnemeHooks + from agent_ultra.adapters.external_memory import ExternalMemoryHooks """ diff --git a/src/agent_ultra/adapters/cli.py b/src/agent_ultra/adapters/cli.py index aef387b..e5c5060 100644 --- a/src/agent_ultra/adapters/cli.py +++ b/src/agent_ultra/adapters/cli.py @@ -169,6 +169,180 @@ def _panel_gate(args) -> int: return 0 if allowed else 1 +# -------------------------------------------------------------------------- +# ultracode — deterministic multi-agent workflow engine +# -------------------------------------------------------------------------- + +def _ultracode_engine(args): + from ..ultracode import UltracodeEngine, demo_pool + if getattr(args, "mock", False): + pool = demo_pool() + else: + pool = _build_pool(args) + routes = getattr(pool, "routes", None) or ["mock-a"] + try: + usage_source = pool.client_for(routes[0]) + except Exception: + usage_source = None + cfg = {} + if getattr(args, "max_calls", 0): + cfg["max_calls"] = args.max_calls + if getattr(args, "token_budget", 0): + cfg["token_budget"] = args.token_budget + return UltracodeEngine(pool, config=cfg, usage_source=usage_source) + + +def _ultracode_run(args) -> int: + from ..ultracode import WorkflowError, summarize_events, render_card, print_card + eng = _ultracode_engine(args) + wf_args = None + if getattr(args, "args", ""): + try: + wf_args = json.loads(args.args) + except ValueError: + wf_args = args.args # bare text is a valid wf.args + try: + path = eng.resolve(args.workflow) + rep = eng.run_script(path, args=wf_args, + resume_run_id=getattr(args, "resume", "") or "") + except WorkflowError as e: + print(f"ultracode error: {e}", file=sys.stderr) + return 2 + if getattr(args, "json", False): + from dataclasses import asdict + print(json.dumps(asdict(rep), indent=2, default=str, ensure_ascii=False)) + return 0 if rep.status == "completed" else 1 + print_card(summarize_events(rep.events_path)) + if rep.result is not None: + print("Result: " + json.dumps(rep.result, default=str, ensure_ascii=False)[:600]) + print("Report: " + rep.artifact_json) + print("Resume: agent-ultra ultracode resume " + rep.run_id) + return 0 if rep.status == "completed" else 1 + + +def _ultracode_list(args) -> int: + eng = _ultracode_engine(args) + listing = eng.list_scripts() + if listing["saved"]: + print(f"saved ({eng.scripts_dir()}):") + for s in listing["saved"]: + print(f" {s['name']:<20} {s['description']}") + else: + print(f"no saved workflows yet — drop .py files into {eng.scripts_dir()}") + print("bundled examples:") + for s in listing["examples"]: + print(f" {s['name']:<20} {s['description']}") + return 0 + + +def _ultracode_status(args) -> int: + from ..ultracode import summarize_events, render_card + eng = _ultracode_engine(args) + runs = eng.home / "runs" + files = sorted(runs.glob(".status-*.json"), + key=lambda p: p.stat().st_mtime, reverse=True) \ + if runs.exists() else [] + cards = [] + for f in files: + try: + st = json.loads(f.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + events = st.get("events_path") or str(runs / st.get("run_id", "?") / "events.jsonl") + summary = summarize_events(events) + if not summary.get("ok"): + summary = {"run_id": st.get("run_id"), "name": st.get("name"), + "phase": st.get("phase"), "counts": st.get("counts") or {}, + "dots": []} + cards.append(render_card(summary, rich=False)) + if not cards: + print("No ultracode run is currently in progress.") + return 0 + print("\n\n".join(cards)) + return 0 + + +def _ultracode_resume(args) -> int: + args.workflow = args.workflow_hint or _infer_workflow_for_run(args) + if not args.workflow: + print("could not infer which workflow produced run " + f"{args.run_id!r}; pass it: " + "agent-ultra ultracode resume ", file=sys.stderr) + return 2 + args.resume = args.run_id + return _ultracode_run(args) + + +def _infer_workflow_for_run(args) -> str: + """Read the run's journal run_start entry to recover the script path.""" + eng = _ultracode_engine(args) + journal = eng.home / "runs" / args.run_id / "journal.jsonl" + try: + for line in journal.read_text(encoding="utf-8").splitlines(): + entry = json.loads(line) + if entry.get("type") == "run_start" and entry.get("script"): + return entry["script"] + except (OSError, ValueError): + pass + return "" + + +def _receipts_bus(args): + from ..receipts_bus import ReceiptsBus + return ReceiptsBus(db_path=args.db or "", key_path=args.key or "") + + +def _receipts(args) -> int: + bus = _receipts_bus(args) + sub = args.receipts_cmd + if sub == "list": + rows = bus.list(session_id=args.session or "", workspace=args.workspace or "") + for r in rows: + print(f" {r['receipt_id'][:12]} {r['kind']:<12} {r['verdict']:<10} " + f"{r.get('ts', '')}") + print(f"\n{len(rows)} receipt(s)") + return 0 + if sub == "show": + r = bus.get(args.receipt_id) + if r is None: + print("not found", file=sys.stderr) + return 1 + print(json.dumps(r, indent=2, ensure_ascii=False)) + return 0 + if sub == "verify": + res = bus.verify(args.receipt_id) + print(f"ok={res['ok']} authentic={res['authentic']} " + f"integrity={res['integrity']} refs_ok={res['refs_ok']}") + return 0 if res["ok"] else 1 + if sub == "why": + ctx = {"session_id": args.session or "", "workspace": args.workspace or "", + "claim_sha256": args.claim_sha or "", + "verify_command": args.verify_command or ""} + if args.cited: + ctx["cited"] = args.cited + out = bus.why(ctx) + for c in out["candidates"]: + mark = "BINDS" if c["binds"] else " " + print(f" [{mark}] {c['receipt_id'][:12] if c['receipt_id'] else '?':<12} " + f"{c['kind']:<12} {c['clause']} ({c['origin']})") + print(f"\nclaim {'BINDS' if out['binds'] else 'is UNSUPPORTED'}") + return 0 if out["binds"] else 1 + if sub == "attest": + if not sys.stdin.isatty(): + print("attest is interactive-only; refusing in non-interactive mode", + file=sys.stderr) + return 2 + note = input("attestation note: ").strip() + env = bus.append(kind="manual", actor="operator", verdict="approved", + session_id=args.session or "", + workspace=args.workspace or "", + evidence=[{"type": "note", "value": note}]) + print(f"wrote manual receipt {env['receipt_id']} " + f"(manual receipts never satisfy a completion gate)") + return 0 + return 2 + + def main(argv=None) -> int: p = argparse.ArgumentParser(prog="agent-ultra", description="adversarial panel / ULTRA loop / broker") @@ -216,12 +390,79 @@ def main(argv=None) -> int: cp.add_argument("command") cp.set_defaults(func=_classify) + # -- ultracode: deterministic multi-agent workflows ------------------- + ucp = sub.add_parser("ultracode", + help="run deterministic multi-agent workflows " + "(fan-out -> journal -> resume -> receipt)") + ucsub = ucp.add_subparsers(dest="ultracode_cmd", required=True) + + def _uc_common(sp): + # accept --mock/--mode/--config after the subcommand too, so the + # friend-facing `agent-ultra ultracode run X --mock` form just works + # (argparse otherwise requires global flags before the subcommand). + sp.add_argument("--mock", action="store_true", + help="use the offline deterministic mock route (no key)") + sp.add_argument("--mode", default="single", choices=["single", "mixed"]) + sp.add_argument("--config", default="") + + ucr = ucsub.add_parser("run", help="run a workflow by name or path") + ucr.add_argument("workflow", help="saved name, bundled example, or a .py path") + ucr.add_argument("--args", default="", + help="JSON (or bare text) passed to the workflow as wf.args") + ucr.add_argument("--resume", default="", + help="run_id of a prior run to replay cached calls from") + ucr.add_argument("--max-calls", type=int, default=0, dest="max_calls") + ucr.add_argument("--token-budget", type=int, default=0, dest="token_budget") + ucr.add_argument("--json", action="store_true") + _uc_common(ucr) + ucr.set_defaults(func=_ultracode_run) + + ucl = ucsub.add_parser("list", help="list saved + bundled workflows") + _uc_common(ucl) + ucl.set_defaults(func=_ultracode_list) + + ucs = ucsub.add_parser("status", help="show in-progress runs (terminal-safe)") + _uc_common(ucs) + ucs.set_defaults(func=_ultracode_status) + + ucrs = ucsub.add_parser("resume", help="resume a prior run by run_id " + "(replays cached calls)") + ucrs.add_argument("run_id") + ucrs.add_argument("workflow_hint", nargs="?", default="", + help="the workflow name/path (inferred from the journal " + "if omitted)") + ucrs.add_argument("--args", default="") + ucrs.add_argument("--max-calls", type=int, default=0, dest="max_calls") + ucrs.add_argument("--token-budget", type=int, default=0, dest="token_budget") + ucrs.add_argument("--json", action="store_true") + _uc_common(ucrs) + ucrs.set_defaults(func=_ultracode_resume) + gp = sub.add_parser("panel-gate", help="block REPORT unless a valid panel execution " "receipt proves real panel-agent calls happened") gp.add_argument("run_dir", help="ULTRA run dir (has panel_execution_receipt.json)") gp.set_defaults(func=_panel_gate) + rp = sub.add_parser("receipts", + help="list/show/verify/why/attest authenticated receipts") + rp.add_argument("--db", default="", help="receipts store path (default in-memory)") + rp.add_argument("--key", default="", help="per-install HMAC key file path") + rp.add_argument("--session", default="") + rp.add_argument("--workspace", default="") + rsub = rp.add_subparsers(dest="receipts_cmd", required=True) + rsub.add_parser("list", help="list stored receipts") + rs_show = rsub.add_parser("show", help="print one receipt as JSON") + rs_show.add_argument("receipt_id") + rs_ver = rsub.add_parser("verify", help="check integrity + authenticity") + rs_ver.add_argument("receipt_id") + rs_why = rsub.add_parser("why", help="explain which receipts bind a claim") + rs_why.add_argument("--claim-sha", dest="claim_sha", default="") + rs_why.add_argument("--verify-command", dest="verify_command", default="") + rs_why.add_argument("--cited", default="") + rsub.add_parser("attest", help="write a manual receipt (interactive only)") + rp.set_defaults(func=_receipts) + from .doctor import run_doctor, run_init, run_demo dp = sub.add_parser("doctor", help="verify the install end to end") diff --git a/src/agent_ultra/adapters/doctor.py b/src/agent_ultra/adapters/doctor.py index 09e3383..145c0ec 100644 --- a/src/agent_ultra/adapters/doctor.py +++ b/src/agent_ultra/adapters/doctor.py @@ -119,6 +119,29 @@ def panel_demo(out_dir: Path): return ok, report +def ultracode_demo(out_dir: Path): + """Fan-out -> journal -> resume -> receipt, fully offline. Returns + (ok, info) where ok also proves the resume replayed with zero calls and + the receipt checksum validates.""" + from ..ultracode import UltracodeEngine, demo_pool, verify_receipt + import json as _json + home = out_dir / "ultracode-demo" + eng = UltracodeEngine(demo_pool(), home=home) + rep = eng.run_script(eng.resolve("smoke")) + receipt = _json.loads(Path(rep.receipt_path).read_text(encoding="utf-8")) + resumed = eng.run_script(eng.resolve("smoke"), resume_run_id=rep.run_id) + ok = (rep.final_state == "COMPLETE" + and rep.model_calls == 4 + and verify_receipt(receipt) + and resumed.model_calls == 0 + and resumed.cached_hits == 4) + return ok, {"final_state": rep.final_state, "calls": rep.model_calls, + "resume_calls": resumed.model_calls, + "resume_cached": resumed.cached_hits, + "receipt_valid": verify_receipt(receipt), + "run_dir": rep.artifact_dir} + + def broker_demo(out_dir: Path): from ..broker.broker import CommandBroker, TRUSTED_OWNER_TIERS ledger = out_dir / "broker-demo.jsonl" @@ -168,6 +191,68 @@ def fake_fixer(task, workspace): return ok, rep +def receipts_demo(out_dir: Path): + """Offline: sign a receipt, prove authenticity is separate from integrity, + reject a forgery, and verify a hash-chained gate audit.""" + from ..receipts_bus import ReceiptsBus + from ..receipts_bus.envelope import receipt_sha256 + key = out_dir / "install.key" + key.write_bytes(b"agent-ultra-doctor-install-key-0123456789") + bus = ReceiptsBus(db_path=out_dir / "receipts.db", key_path=key) + good = bus.append(kind="panel", actor="engine_a", verdict="shipped", + session_id="demo") + gv = bus.verify(good["receipt_id"]) + # a hand-authored envelope with a valid sha256 but no HMAC is a forgery + forged = dict(good) + forged["receipt_id"] = "forged-0001" + forged["receipt_hmac"] = "" + forged["receipt_sha256"] = receipt_sha256(forged) + bus.put(forged) + fv = bus.verify("forged-0001") + bus.append_audit("ship", "enforce", "allow", "engine_a", "demo evidence") + bus.append_audit("ship", "enforce", "block", "reviewer", "demo stale") + chain = bus.verify_audit("ship") + ok = (gv["ok"] and gv["authentic"] + and fv["integrity"] and not fv["authentic"] and not fv["ok"] + and chain["ok"]) + return ok, {"genuine_ok": gv["ok"], + "forgery_authentic": fv["authentic"], + "audit_chain_ok": chain["ok"]} + + +def verifier_demo(out_dir: Path): + """Offline: refute-first defaults, engine re-check via an injected runner, + and a REFUTED verdict producing a non-acceptable receipt.""" + from ..verifier import verify_claim + from ..receipts_bus import ReceiptsBus, Candidate + key = out_dir / "verifier.key" + key.write_bytes(b"agent-ultra-doctor-verifier-key-0123456789") + bus = ReceiptsBus(db_path=out_dir / "verifier.db", key_path=key) + + def emit(fields): + return bus.append(**fields)["receipt_id"] + + def run_check(cmd, cwd): + return {"exit_code": 0, "ledger_ref": "demo://1"} + + no_evidence = verify_claim("it works") # -> refuted (default) + confirmed = verify_claim("tests pass", declared_verify_cmd="pytest -q", + run_check=run_check, session_id="demo", + emit_receipt=emit) + refuted = verify_claim("bogus", session_id="demo", emit_receipt=emit) + refuted_receipt = bus.get(refuted["receipt_id"]) + # the refuted receipt must NOT satisfy a gate for its own claim + ok_bind, clause = bus.binds( + {"session_id": "demo", "claim_sha256": refuted["claim_sha256"]}, + Candidate(receipt=refuted_receipt)) + ok = (no_evidence["refuted"] and not confirmed["refuted"] + and refuted["refuted"] and refuted_receipt["verdict"] == "failed" + and not ok_bind) + return ok, {"no_evidence_refuted": no_evidence["refuted"], + "recheck_confirmed": not confirmed["refuted"], + "refuted_verdict": refuted_receipt["verdict"]} + + def run_demo(args) -> int: out = Path(tempfile.mkdtemp(prefix="agent-ultra-demo-")) print("agent-ultra demo (offline, deterministic mock route)\n") @@ -180,8 +265,20 @@ def run_demo(args) -> int: ok3, rep = ultra_demo(out) print(f"[{'ok' if ok3 else 'FAIL'}] ultra: tests green -> panel -> " f"{len(rep.fix_tasks)} fix task(s) -> fix loop -> proof saved") + ok4, rb = receipts_demo(out) + print(f"[{'ok' if ok4 else 'FAIL'}] receipts: genuine authentic, " + f"forgery rejected (authentic={rb['forgery_authentic']}), " + f"audit chain ok={rb['audit_chain_ok']}") + ok5, vb = verifier_demo(out) + print(f"[{'ok' if ok5 else 'FAIL'}] verifier: no-evidence refuted, " + f"re-check confirmed, refuted verdict={vb['refuted_verdict']} " + f"(cannot satisfy a gate)") + ok6, uc = ultracode_demo(out) + print(f"[{'ok' if ok6 else 'FAIL'}] ultracode: {uc['calls']} agents fan out " + f"-> {uc['final_state']} -> receipt valid={uc['receipt_valid']} " + f"-> resume replays {uc['resume_cached']} cached ({uc['resume_calls']} calls)") print(f"\nartifacts: {out}") - all_ok = ok1 and ok2 and ok3 + all_ok = ok1 and ok2 and ok3 and ok4 and ok5 and ok6 print("\nDEMO " + ("PASSED — the full loop works on this machine." if all_ok else "FAILED — run `agent-ultra doctor`.")) return 0 if all_ok else 1 @@ -275,6 +372,35 @@ def add(status, name, detail=""): except Exception as e: add(FAIL, "ULTRA demo", str(e)[:160]) + # 7a2. ultracode: multi-agent fan-out -> journal -> resume -> receipt + try: + ok, detail = ultracode_demo(scratch) + add(PASS if ok else FAIL, + "ultracode: fan-out -> journal -> resume (cached) -> valid receipt", + json.dumps(detail)[:140]) + except Exception as e: + add(FAIL, "ultracode", str(e)[:160]) + + # 7b. receipts bus: authenticity separate from integrity, forgery rejected, + # gate audit chain intact. + try: + ok, detail = receipts_demo(scratch) + add(PASS if ok else FAIL, + "receipts bus: HMAC authenticity, forgery rejected, audit chain", + json.dumps(detail)[:120]) + except Exception as e: + add(FAIL, "receipts bus", str(e)[:160]) + + # 7c. verifier: refute-first default, engine re-check, non-acceptable + # verdict on refutation. + try: + ok, detail = verifier_demo(scratch) + add(PASS if ok else FAIL, + "verifier: refute-first default, engine re-check, refuted!=gate", + json.dumps(detail)[:120]) + except Exception as e: + add(FAIL, "verifier", str(e)[:160]) + # 8. secret hygiene: redaction works, and nothing the demos wrote leaks try: from ..evidence.reader import redact_secrets, find_secrets diff --git a/src/agent_ultra/adapters/mneme_memory.py b/src/agent_ultra/adapters/external_memory.py similarity index 73% rename from src/agent_ultra/adapters/mneme_memory.py rename to src/agent_ultra/adapters/external_memory.py index 484c5b6..641bcf0 100644 --- a/src/agent_ultra/adapters/mneme_memory.py +++ b/src/agent_ultra/adapters/external_memory.py @@ -1,12 +1,13 @@ -"""Optional Mneme memory adapter (example of wiring the generic hooks to a +"""Optional external-memory adapter (example of wiring the generic hooks to a real memory system). -Mneme is one such system; this adapter shows the shape without importing it. -If a `mneme`-like client is available it is used; otherwise events fall back -to a local JSONL file, so importing this module never fails and never -requires the dependency. +This adapter shows the SHAPE of the integration without importing any specific +memory backend. If a note-writing client is supplied it is used; otherwise +events fall back to a local JSONL file, so importing this module never fails +and never requires the dependency. -Replace `_MnemeClientProtocol` calls with your memory system's real API. +Replace the ``client.write_note(...)`` calls with your memory system's real +API. """ from __future__ import annotations @@ -16,11 +17,11 @@ from ..memory.hooks import MemoryHooks, JsonlHooks -class MnemeHooks(MemoryHooks): +class ExternalMemoryHooks(MemoryHooks): """Write panel decisions and accepted findings to a memory client. - client: any object exposing `write_note(kind, title, body, tags)`. Pass - your Mneme (or other) client. If None, degrades to a JSONL fallback. + client: any object exposing ``write_note(kind, title, body, tags)``. Pass + your own memory client. If None, degrades to a JSONL fallback. """ def __init__(self, client=None, fallback_path: Path | str | None = None, diff --git a/src/agent_ultra/leakgate.py b/src/agent_ultra/leakgate.py new file mode 100644 index 0000000..9a8c5d4 --- /dev/null +++ b/src/agent_ultra/leakgate.py @@ -0,0 +1,175 @@ +"""Leak gate — scan the tracked tree (and commit messages / PR bodies) for +private strings before any release tag. + +The denylist lives in ``tests/private_strings.txt`` as base64, one token per +line. It is decoded at RUNTIME so the denylist file itself never contains the +plaintext tokens and therefore does not trip its own scan. There is no file +whitelist by design: a hit is a hit, and the fix is to change the offending +text, not to exempt a file. + +Matching rules (from the backport spec §0): + + * Case-insensitive for every token EXCEPT the one that collides with a + common English word — that token is matched case-sensitively and only as a + whole word, so ordinary prose using the lowercase word does not trip it. + * Path separators are normalised (``\\`` -> ``/``) in both the haystack and + the needles before matching, so a Windows-style path token matches its + POSIX-written form and vice versa. + +Usage: + from agent_ultra.leakgate import scan_tree, load_denylist + hits = scan_tree(repo_root) # [(path, line_no, token), ...] + +CLI: + agent-ultra-leakgate [REPO_ROOT] [--message "commit msg"] +""" + +from __future__ import annotations + +import base64 +import re +import sys +from pathlib import Path + +# Tokens matched whole-word and CASE-SENSITIVELY (they collide with common +# English words), identified by their decoded UPPER-CASE form. Stored +# base64-encoded so this source file holds no private plaintext and does not +# trip its own scan. Decode: base64 of the all-caps English-word collision. +_CASE_SENSITIVE_WHOLE_WORD_B64 = frozenset({"U09VTA=="}) +_CASE_SENSITIVE_WHOLE_WORD = frozenset( + base64.b64decode(b).decode("utf-8") for b in _CASE_SENSITIVE_WHOLE_WORD_B64) + +# Directories never worth scanning (build/VCS noise, not source). +_SKIP_DIRS = frozenset({ + ".git", "__pycache__", ".pytest_cache", ".mypy_cache", ".tox", ".venv", + "venv", "node_modules", "dist", "build", ".egg-info", ".idea", ".ruff_cache", +}) + +# Binary-ish extensions we do not read as text. +_SKIP_EXTS = frozenset({ + ".pyc", ".pyo", ".so", ".dll", ".dylib", ".png", ".jpg", ".jpeg", ".gif", + ".ico", ".pdf", ".zip", ".gz", ".whl", ".woff", ".woff2", ".ttf", +}) + +DENYLIST_RELPATH = "tests/private_strings.txt" + + +def _norm(text: str) -> str: + """Normalise path separators so \\ and / forms match interchangeably.""" + return text.replace("\\", "/") + + +def load_denylist(denylist_path: str | Path) -> list[str]: + """Decode the base64 denylist into plaintext tokens. Ignores comments and + blank lines. Raises if the file is missing (a silent empty denylist would + make the gate a no-op).""" + p = Path(denylist_path) + tokens: list[str] = [] + for raw in p.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + tokens.append(base64.b64decode(line).decode("utf-8")) + if not tokens: + raise ValueError(f"denylist {p} decoded to zero tokens") + return tokens + + +def _build_matchers(tokens: list[str]): + """Return (ci_needles, cs_word_patterns). + + ci_needles — normalised, lowercased plain substrings (case-insensitive). + cs_word_patterns — (token, compiled regex) for whole-word case-sensitive. + """ + ci: list[tuple[str, str]] = [] # (original_token, normalized_lower) + cs: list[tuple[str, re.Pattern]] = [] + for tok in tokens: + if tok in _CASE_SENSITIVE_WHOLE_WORD: + cs.append((tok, re.compile(rf"(? list[tuple[int, str]]: + """Scan a blob of text. Returns ``[(line_no, token), ...]`` (1-based).""" + ci, cs = _build_matchers(tokens) + hits: list[tuple[int, str]] = [] + for i, line in enumerate(text.splitlines(), start=1): + norm_lower = _norm(line).lower() + for original, needle in ci: + if needle and needle in norm_lower: + hits.append((i, original)) + for original, pat in cs: + if pat.search(line): # case-sensitive, whole-word, un-normalised + hits.append((i, original)) + return hits + + +def _iter_files(root: Path): + for p in sorted(root.rglob("*")): + if not p.is_file(): + continue + if any(part in _SKIP_DIRS for part in p.relative_to(root).parts): + continue + if p.suffix.lower() in _SKIP_EXTS: + continue + yield p + + +def scan_tree(root: str | Path, + denylist_path: str | Path | None = None) -> list[tuple[str, int, str]]: + """Scan every text file under ``root`` (minus VCS/build noise). Returns + ``[(relpath, line_no, token), ...]``. The denylist file is decoded and its + own lines are exempt from matching (they are base64, not plaintext).""" + root = Path(root) + denylist_path = Path(denylist_path or (root / DENYLIST_RELPATH)) + tokens = load_denylist(denylist_path) + denylist_resolved = denylist_path.resolve() + hits: list[tuple[str, int, str]] = [] + for p in _iter_files(root): + if p.resolve() == denylist_resolved: + continue # the base64 denylist is exempt (it holds no plaintext) + try: + text = p.read_text(encoding="utf-8", errors="strict") + except (OSError, UnicodeDecodeError): + continue # binary or unreadable: skip + for line_no, token in scan_text(text, tokens): + hits.append((str(p.relative_to(root)).replace("\\", "/"), + line_no, token)) + return hits + + +def scan_message(message: str, + denylist_path: str | Path) -> list[tuple[int, str]]: + """Scan a commit message / PR body string.""" + return scan_text(message, load_denylist(denylist_path)) + + +def main(argv=None) -> int: + import argparse + ap = argparse.ArgumentParser(prog="agent-ultra-leakgate", + description="scan the tree for private strings") + ap.add_argument("root", nargs="?", default=".") + ap.add_argument("--denylist", default="") + ap.add_argument("--message", default="", + help="also scan this commit message / PR body") + args = ap.parse_args(argv) + root = Path(args.root) + denylist = args.denylist or str(root / DENYLIST_RELPATH) + tree_hits = scan_tree(root, denylist) + msg_hits = scan_message(args.message, denylist) if args.message else [] + if not tree_hits and not msg_hits: + print("leak gate: PASS (no private strings found)") + return 0 + for path, line_no, token in tree_hits: + print(f"LEAK {path}:{line_no}: private token {token!r}") + for line_no, token in msg_hits: + print(f"LEAK :{line_no}: private token {token!r}") + print(f"\nleak gate: FAIL ({len(tree_hits) + len(msg_hits)} hit(s))") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/agent_ultra/memory/hooks.py b/src/agent_ultra/memory/hooks.py index 664a11a..545f037 100644 --- a/src/agent_ultra/memory/hooks.py +++ b/src/agent_ultra/memory/hooks.py @@ -2,8 +2,8 @@ Subclass MemoryHooks (all methods are no-ops) and pass an instance to the panel engine / ULTRA loop / broker wrapper of your choice. The kit never -imports a specific memory system; adapters (e.g. the optional Mneme adapter) -live in agent_ultra.adapters. +imports a specific memory system; adapters (e.g. the optional external-memory +adapter) live in agent_ultra.adapters. """ from __future__ import annotations diff --git a/src/agent_ultra/receipts_bus/__init__.py b/src/agent_ultra/receipts_bus/__init__.py new file mode 100644 index 0000000..771811e --- /dev/null +++ b/src/agent_ultra/receipts_bus/__init__.py @@ -0,0 +1,56 @@ +"""receipts_bus — a unified, authenticated receipt index. + +Every unit of agent work can leave a *receipt*: a signed record of what was +done, what it claimed, and what evidence backs the claim. The bus is the store +and the resolver: given a claim, it returns the receipts that could bind to it, +and it tells you — by an explicit rule — which ones actually do. + +Two properties are load-bearing: + + * **Authenticity is separate from integrity.** ``receipt_sha256`` proves the + body was not altered; ``receipt_hmac`` proves it was written by something + holding the per-install key. A hand-authored envelope with a correct + sha256 but no valid HMAC is ``authentic=False`` and any enforce-mode + consumer must reject it. + + * **A failed read never returns empty.** A busy or corrupt store raises + ``BusUnavailable`` rather than silently returning ``[]`` (which a caller + would misread as "no evidence exists"). + +Everything host-specific — the key path, the store path, TTLs, caps — is a +constructor parameter with a neutral default. Stdlib only. +""" + +from __future__ import annotations + +from .envelope import ( # noqa: F401 + KINDS, + ACTORS, + VERDICTS, + ACCEPTABLE_VERDICTS, + COMPLETION_KINDS, + canonical_body, + receipt_sha256, + build_envelope, + EnvelopeError, +) +from .bus import ( # noqa: F401 + ReceiptsBus, + BusUnavailable, + Candidate, +) + +__all__ = [ + "KINDS", + "ACTORS", + "VERDICTS", + "ACCEPTABLE_VERDICTS", + "COMPLETION_KINDS", + "canonical_body", + "receipt_sha256", + "build_envelope", + "EnvelopeError", + "ReceiptsBus", + "BusUnavailable", + "Candidate", +] diff --git a/src/agent_ultra/receipts_bus/bus.py b/src/agent_ultra/receipts_bus/bus.py new file mode 100644 index 0000000..a1b4361 --- /dev/null +++ b/src/agent_ultra/receipts_bus/bus.py @@ -0,0 +1,486 @@ +"""ReceiptsBus — the SQLite-backed store, resolver, and gate audit chain. + +Store +----- +SQLite in WAL mode with the DEFAULT auto-checkpoint left in place (we never +set ``wal_autocheckpoint=0``, which would grow the WAL without bound). Writes +take a ``busy_timeout``; reads get one bounded retry. A read that still fails +raises :class:`BusUnavailable` — it never degrades to an empty list, because a +caller cannot tell "no receipts" from "the store was unreachable" and would +wrongly conclude a claim is unsupported. + +Resolve + binding +----------------- +:meth:`resolve` returns candidate receipts scoped to the claim's session / +workspace / window, optionally federating other in-place sources through +injected reader callables (so the kit stays store-agnostic). :meth:`binds` +applies the binding rule that decides which candidates actually back a claim. + +Gate audit +---------- +:meth:`append_audit` writes a hash-chained row; :meth:`verify_audit` walks the +chain and reports the first broken link. +""" + +from __future__ import annotations + +import calendar +import hashlib +import json +import sqlite3 +import time +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable + +from .envelope import ( + build_envelope, + canonical_body, + receipt_sha256, + verify_hashes, + ACCEPTABLE_VERDICTS, + COMPLETION_KINDS, +) + + +class BusUnavailable(RuntimeError): + """The store could not be read. Distinct from 'no receipts found' — a + caller MUST NOT treat this as an empty result.""" + + +# A reader callable federates an external, in-place source into resolve(). +# It takes a claim context dict and returns a list of envelope dicts. +FederatedReader = Callable[[dict], list] + + +@dataclass +class Candidate: + """A receipt that could bind to a claim, annotated with its origin.""" + receipt: dict + origin: str = "bus" # "bus" or the name of a federated reader + + +@dataclass +class ReceiptsBus: + """Authenticated receipt index over a SQLite store. + + All host-specific values are parameters: + + db_path store location (default: in-memory shared cache). + key_path per-install HMAC key file. Read once at construction. + The HOST is responsible for fencing this path from + untrusted writers (see the honest residual below). + ttl_seconds receipts older than this are evicted on write. + per_session_cap keep at most this many receipts per session. + busy_timeout_ms SQLite busy timeout for writes. + read_retries bounded read retries before BusUnavailable. + freshness_seconds a verify-command candidate is only fresh within this. + + Honest residual: a same-user process that can READ ``key_path`` can forge + an authentic receipt. Closing that needs an out-of-process signer, which + is out of scope for v1. + """ + + db_path: str | Path = "" + key_path: str | Path = "" + ttl_seconds: int = 30 * 24 * 3600 + per_session_cap: int = 500 + busy_timeout_ms: int = 5000 + read_retries: int = 1 + freshness_seconds: int = 24 * 3600 + federated_readers: dict[str, FederatedReader] = field(default_factory=dict) + + _key: bytes = field(default=b"", init=False, repr=False) + + def __post_init__(self) -> None: + self._key = self._load_key() + self._init_db() + + # -- key custody ------------------------------------------------------- + def _load_key(self) -> bytes: + if not self.key_path: + # No key configured: the bus can still store/verify integrity, but + # nothing it writes is authentic. Enforce-mode consumers reject. + return b"" + p = Path(self.key_path) + try: + raw = p.read_bytes() + except OSError: + return b"" + return raw.strip() or b"" + + @property + def has_key(self) -> bool: + return bool(self._key) + + # -- store ------------------------------------------------------------- + def _connect(self) -> sqlite3.Connection: + target = str(self.db_path) if self.db_path else \ + "file:receipts_bus?mode=memory&cache=shared" + uri = bool(self.db_path) is False + conn = sqlite3.connect(target, uri=uri, timeout=self.busy_timeout_ms / 1000) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") # default autocheckpoint + conn.execute(f"PRAGMA busy_timeout={int(self.busy_timeout_ms)}") + return conn + + # keep one connection alive for the in-memory shared-cache case, so the + # schema/data survive between calls within a process. + _keepalive: sqlite3.Connection | None = field(default=None, init=False, + repr=False) + + def _init_db(self) -> None: + conn = self._connect() + if not self.db_path: + self._keepalive = conn + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS receipts ( + receipt_id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + run_id TEXT, + session_id TEXT, + workspace TEXT, + ts TEXT, + created_at REAL, + claim_sha256 TEXT, + verdict TEXT, + body TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS ix_receipts_session + ON receipts(session_id); + CREATE INDEX IF NOT EXISTS ix_receipts_claim + ON receipts(claim_sha256); + CREATE TABLE IF NOT EXISTS gate_audit ( + event_id TEXT PRIMARY KEY, + seq INTEGER, + gate TEXT NOT NULL, + mode TEXT, + decision TEXT, + source TEXT, + reason TEXT, + ts TEXT, + prev_sha256 TEXT, + row_sha256 TEXT + ); + """ + ) + conn.commit() + if self.db_path: + conn.close() + + def _read_conn(self) -> sqlite3.Connection: + """A read connection with a bounded retry; BusUnavailable on failure.""" + last: Exception | None = None + for attempt in range(self.read_retries + 1): + try: + if not self.db_path and self._keepalive is not None: + return self._keepalive + return self._connect() + except sqlite3.Error as e: # pragma: no cover - timing dependent + last = e + time.sleep(0.02 * (attempt + 1)) + raise BusUnavailable(f"receipts store unreadable: {last}") + + # -- write ------------------------------------------------------------- + def append( + self, *, kind: str, actor: str, verdict: str, **fields + ) -> dict: + """Build, sign (if a key is configured), and store a receipt. Returns + the stored envelope. Enum validation happens in ``build_envelope``.""" + env = build_envelope(kind=kind, actor=actor, verdict=verdict, + key=self._key or None, **fields) + self.put(env) + return env + + def put(self, env: dict) -> None: + """Store an already-built envelope (used by federation/import).""" + conn = self._connect() if self.db_path else self._keepalive + assert conn is not None + conn.execute( + "INSERT OR REPLACE INTO receipts (receipt_id, kind, run_id, " + "session_id, workspace, ts, created_at, claim_sha256, verdict, body)" + " VALUES (?,?,?,?,?,?,?,?,?,?)", + (env["receipt_id"], env["kind"], env.get("run_id", ""), + env.get("session_id", ""), env.get("workspace", ""), + env.get("ts", ""), time.time(), env.get("claim_sha256", ""), + env["verdict"], json.dumps(env, sort_keys=True)), + ) + conn.commit() + self._evict(conn, env.get("session_id", "")) + if self.db_path: + conn.close() + + def _evict(self, conn: sqlite3.Connection, session_id: str) -> None: + cutoff = time.time() - self.ttl_seconds + conn.execute("DELETE FROM receipts WHERE created_at < ?", (cutoff,)) + if session_id: + conn.execute( + "DELETE FROM receipts WHERE session_id = ? AND receipt_id NOT IN " + "(SELECT receipt_id FROM receipts WHERE session_id = ? " + " ORDER BY created_at DESC LIMIT ?)", + (session_id, session_id, self.per_session_cap), + ) + conn.commit() + + # -- read -------------------------------------------------------------- + def get(self, receipt_id: str) -> dict | None: + conn = self._read_conn() + try: + row = conn.execute( + "SELECT body FROM receipts WHERE receipt_id = ?", + (receipt_id,)).fetchone() + except sqlite3.Error as e: + raise BusUnavailable(f"receipts store unreadable: {e}") + finally: + if self.db_path: + conn.close() + return json.loads(row["body"]) if row else None + + def list(self, *, session_id: str = "", workspace: str = "", + kind: str = "", limit: int = 200) -> list[dict]: + """List stored receipts, newest first. Raises BusUnavailable on a + store error — never returns [] to hide an unreadable store.""" + clauses, params = [], [] + if session_id: + clauses.append("session_id = ?"); params.append(session_id) + if workspace: + clauses.append("workspace = ?"); params.append(workspace) + if kind: + clauses.append("kind = ?"); params.append(kind) + where = (" WHERE " + " AND ".join(clauses)) if clauses else "" + conn = self._read_conn() + try: + rows = conn.execute( + f"SELECT body FROM receipts{where} ORDER BY created_at DESC " + f"LIMIT ?", (*params, limit)).fetchall() + except sqlite3.Error as e: + raise BusUnavailable(f"receipts store unreadable: {e}") + finally: + if self.db_path: + conn.close() + return [json.loads(r["body"]) for r in rows] + + # -- verify ------------------------------------------------------------ + def verify(self, receipt_id: str) -> dict: + """Return ``{ok, authentic, integrity, refs_ok, receipt}``. + + ``authentic`` is separate from ``integrity`` (see envelope). ``ok`` is + the enforce-mode answer: integrity AND authentic AND refs resolve. + """ + receipt = self.get(receipt_id) + if receipt is None: + return {"ok": False, "authentic": False, "integrity": False, + "refs_ok": False, "receipt": None} + integrity, authentic = verify_hashes(receipt, self._key or None) + refs_ok = self._refs_ok(receipt) + return {"ok": bool(integrity and authentic and refs_ok), + "authentic": authentic, "integrity": integrity, + "refs_ok": refs_ok, "receipt": receipt} + + def _refs_ok(self, receipt: dict) -> bool: + """Every referenced artifact path that looks local must exist.""" + for p in receipt.get("artifact_paths", []): + try: + if p and not Path(p).exists(): + return False + except OSError: + return False + return True + + # -- resolve + binding ------------------------------------------------- + def resolve(self, claim_ctx: dict) -> list[Candidate]: + """Candidates scoped by session/workspace/window, plus any federated + readers. Origin-annotated. Raises BusUnavailable on a store error.""" + rows = self.list(session_id=claim_ctx.get("session_id", ""), + workspace=claim_ctx.get("workspace", ""), + limit=claim_ctx.get("limit", 200)) + out = [Candidate(receipt=r, origin="bus") for r in rows] + for name, reader in self.federated_readers.items(): + try: + for env in reader(claim_ctx) or []: + out.append(Candidate(receipt=env, origin=name)) + except Exception: + # A misbehaving federated reader must not sink the resolve; + # it simply contributes no candidates. + continue + return out + + def binds(self, claim_ctx: dict, candidate: Candidate) -> tuple[bool, str]: + """The binding rule. Returns ``(binds, clause)``. + + A candidate binds a COMPLETION claim only by: + * an explicitly-cited receipt id (``task_ref.cited`` / ``cited``), or + * a canonical verify-command match with scope coverage + freshness, or + * a ``task_ref`` match. + + Never binds: + * a command-kind candidate to a completion claim alone, + * a verifier-kind candidate unless its claim_sha256 == the claim's, + * manual / repair / route_health candidates to completion. + + Weak-but-present evidence (uncovered / stale / ad-hoc-equivalent) + returns the distinct ``"ledger-weak"`` clause, NOT a pass. + """ + r = candidate.receipt + kind = r.get("kind", "") + + # authenticity/integrity is a precondition to binding anything + integrity, authentic = verify_hashes(r, self._key or None) + if not (integrity and authentic): + return False, "unauthenticated" + + # a non-acceptable verdict can never satisfy a gate + if r.get("verdict") not in ACCEPTABLE_VERDICTS: + return False, "non-acceptable-verdict" + + # 1. explicit citation always wins (still must be authentic + acceptable) + cited = claim_ctx.get("cited") or claim_ctx.get("task_ref", {}).get("cited") + if cited and cited == r.get("receipt_id"): + return True, "cited" + + # kinds that never satisfy completion + if kind in ("command", "manual", "repair", "route_health"): + return False, "kind-cannot-complete" if kind != "command" \ + else "command-not-completion" + + # 2. verifier candidate: must match the claim hash exactly + if kind == "verifier": + if claim_ctx.get("claim_sha256") and \ + r.get("claim_sha256") == claim_ctx["claim_sha256"]: + return True, "verifier-claim-match" + return False, "verifier-claim-mismatch" + + # 3. canonical verify-command match with scope coverage + freshness + want_cmd = claim_ctx.get("verify_command", "") + if want_cmd and r.get("canonical_command") == want_cmd: + if not self._scope_covers(claim_ctx, r): + return False, "ledger-weak" # uncovered scope + if not self._fresh(r): + return False, "ledger-weak" # stale + return True, "verify-command" + + # 4. task_ref match (issue id / external ref) + want_ref = claim_ctx.get("task_ref", {}) + got_ref = r.get("task_ref", {}) + common = {k: want_ref[k] for k in want_ref + if k in got_ref and want_ref[k] and got_ref[k] == want_ref[k]} + if common: + if kind not in COMPLETION_KINDS: + return False, "kind-cannot-complete" + return True, "task-ref" + + # present but not a clean bind + if want_cmd or want_ref: + return False, "ledger-weak" + return False, "no-match" + + def _scope_covers(self, claim_ctx: dict, receipt: dict) -> bool: + cs, rs = claim_ctx.get("session_id", ""), receipt.get("session_id", "") + cw, rw = claim_ctx.get("workspace", ""), receipt.get("workspace", "") + if cs and rs and cs != rs: + return False + if cw and rw and cw != rw: + return False + return True + + def _fresh(self, receipt: dict) -> bool: + ts = receipt.get("ts", "") + if not ts: + return False + try: + t = time.strptime(ts, "%Y-%m-%dT%H:%M:%SZ") + age = time.time() - calendar.timegm(t) # ts is UTC (gmtime) + except (ValueError, OverflowError): + return False + return age <= self.freshness_seconds + + def why(self, claim_ctx: dict) -> dict: + """Explain resolution: every candidate with its binding clause.""" + results = [] + binding = False + for cand in self.resolve(claim_ctx): + ok, clause = self.binds(claim_ctx, cand) + binding = binding or ok + results.append({"receipt_id": cand.receipt.get("receipt_id"), + "kind": cand.receipt.get("kind"), + "origin": cand.origin, "binds": ok, + "clause": clause}) + return {"binds": binding, "candidates": results} + + # -- gate_audit (hash chain) ------------------------------------------ + def _last_audit(self, conn: sqlite3.Connection) -> sqlite3.Row | None: + return conn.execute( + "SELECT seq, row_sha256 FROM gate_audit ORDER BY seq DESC LIMIT 1" + ).fetchone() + + @staticmethod + def _audit_row_sha(seq, gate, mode, decision, source, reason, ts, + prev) -> str: + body = json.dumps({"seq": seq, "gate": gate, "mode": mode, + "decision": decision, "source": source, + "reason": reason, "ts": ts, "prev_sha256": prev}, + sort_keys=True) + return hashlib.sha256(body.encode("utf-8")).hexdigest() + + def append_audit(self, gate: str, mode: str, decision: str, + source: str, reason: str) -> str: + """Append a hash-chained audit row; returns the event_id.""" + conn = self._connect() if self.db_path else self._keepalive + assert conn is not None + last = self._last_audit(conn) + seq = (last["seq"] + 1) if last else 0 + prev = last["row_sha256"] if last else "" + ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + event_id = uuid.uuid4().hex + row_sha = self._audit_row_sha(seq, gate, mode, decision, source, + reason, ts, prev) + conn.execute( + "INSERT INTO gate_audit (event_id, seq, gate, mode, decision, " + "source, reason, ts, prev_sha256, row_sha256) " + "VALUES (?,?,?,?,?,?,?,?,?,?)", + (event_id, seq, gate, mode, decision, source, reason, ts, prev, + row_sha)) + conn.commit() + if self.db_path: + conn.close() + return event_id + + def verify_audit(self, gate: str = "") -> dict: + """Recompute the chain. Returns ``{ok, first_broken}`` (first_broken is + the event_id of the first row whose hash/link does not recompute).""" + conn = self._read_conn() + try: + where = " WHERE gate = ?" if gate else "" + params = (gate,) if gate else () + rows = conn.execute( + f"SELECT * FROM gate_audit{where} ORDER BY seq ASC", + params).fetchall() + except sqlite3.Error as e: + raise BusUnavailable(f"gate_audit unreadable: {e}") + finally: + if self.db_path: + conn.close() + prev = "" + for row in rows: + expect = self._audit_row_sha( + row["seq"], row["gate"], row["mode"], row["decision"], + row["source"], row["reason"], row["ts"], row["prev_sha256"]) + if row["prev_sha256"] != prev or row["row_sha256"] != expect: + return {"ok": False, "first_broken": row["event_id"]} + prev = row["row_sha256"] + return {"ok": True, "first_broken": None} + + def get_audit_event(self, event_id: str) -> dict | None: + conn = self._read_conn() + try: + row = conn.execute( + "SELECT * FROM gate_audit WHERE event_id = ?", + (event_id,)).fetchone() + except sqlite3.Error as e: + raise BusUnavailable(f"gate_audit unreadable: {e}") + finally: + if self.db_path: + conn.close() + return dict(row) if row else None diff --git a/src/agent_ultra/receipts_bus/envelope.py b/src/agent_ultra/receipts_bus/envelope.py new file mode 100644 index 0000000..407a2b6 --- /dev/null +++ b/src/agent_ultra/receipts_bus/envelope.py @@ -0,0 +1,171 @@ +"""Receipt envelope — shape, closed enums, and the two hash controls. + +A receipt is a plain dict (JSON-serialisable) so it can cross process and +language boundaries. This module owns: + + * the closed enums (``kind``, ``actor``, ``verdict``) validated at write time, + * the canonical body (the envelope minus its two hash fields), and + * the two hashes: ``receipt_sha256`` (integrity) and ``receipt_hmac`` + (authenticity, keyed). + +The enum members are GENERIC. A host maps its own engines/routes onto the +neutral actor names via config; it does not add members here. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import time +import uuid + +# ---- closed enums -------------------------------------------------------- +# Validated at write time. Generic members; hosts map their own concepts on. + +KINDS = frozenset({ + "panel", # an adversarial-panel run + "loop", # a build/test/fix loop iteration + "command", # a brokered command execution + "verification", # a check/gate result + "verifier", # a refute-first verifier verdict + "route_health", # a fleet/route probe result + "repair", # a repair/watchdog action + "manual", # an operator-attested receipt (never satisfies completion) +}) + +ACTORS = frozenset({ + "engine_a", # a build/execution engine + "engine_b", # a second, independent engine (backend diversity) + "engine_c", # a third engine + "reviewer", # a critic/review engine + "broker", # the command broker + "operator", # a human operator +}) + +VERDICTS = frozenset({ + "shipped", + "approved", + "completed", + "hold", + "conditional", + "blocked", + "failed", + "degraded", +}) + +# Verdicts a completion/enforce gate may accept. A refuted/blocked/failed +# receipt can never satisfy a gate — that is what makes refute-first safe. +ACCEPTABLE_VERDICTS = frozenset({"shipped", "approved", "completed"}) + +# Kinds that may, on their own, satisfy a completion claim. A command receipt +# proves a command ran, not that work is done; manual/repair/route_health are +# never completion evidence. +COMPLETION_KINDS = frozenset({"panel", "loop", "verification", "verifier"}) + + +class EnvelopeError(ValueError): + """A receipt violated the envelope contract (bad enum, missing field).""" + + +# fields excluded from the hashed body (they ARE the hashes) +_HASH_FIELDS = ("receipt_sha256", "receipt_hmac") + + +def _utc_now() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +def canonical_body(envelope: dict) -> str: + """The canonical JSON body that both hashes cover: the envelope minus its + two hash fields, ``json.dumps(sort_keys=True)``. Stable across dict + ordering so the same logical receipt always hashes the same.""" + body = {k: v for k, v in envelope.items() if k not in _HASH_FIELDS} + return json.dumps(body, sort_keys=True, ensure_ascii=False, + separators=(",", ":")) + + +def receipt_sha256(envelope: dict) -> str: + """Integrity hash of the canonical body.""" + return hashlib.sha256(canonical_body(envelope).encode("utf-8")).hexdigest() + + +def receipt_hmac(envelope: dict, key: bytes) -> str: + """Authenticity hash of the canonical body, keyed by the per-install key.""" + return hmac.new(key, canonical_body(envelope).encode("utf-8"), + hashlib.sha256).hexdigest() + + +def _validate(kind: str, actor: str, verdict: str) -> None: + if kind not in KINDS: + raise EnvelopeError(f"unknown kind {kind!r}; allowed: {sorted(KINDS)}") + if actor not in ACTORS: + raise EnvelopeError(f"unknown actor {actor!r}; allowed: {sorted(ACTORS)}") + if verdict not in VERDICTS: + raise EnvelopeError( + f"unknown verdict {verdict!r}; allowed: {sorted(VERDICTS)}") + + +def build_envelope( + *, + kind: str, + actor: str, + verdict: str, + run_id: str = "", + session_id: str = "", + workspace: str = "", + task_ref: dict | None = None, + canonical_command: str = "", + claim_sha256: str = "", + inputs_hash: str = "", + evidence: list | None = None, + artifact_paths: list | None = None, + key: bytes | None = None, + receipt_id: str = "", + ts: str = "", +) -> dict: + """Build a fully-formed, hashed (and, if ``key`` given, signed) envelope. + + ``kind``/``actor``/``verdict`` are validated against the closed enums. + ``task_ref`` is a small dict of optional claim keys (e.g. an issue id, a + cited receipt id). Passing ``key`` stamps the HMAC; omitting it leaves + ``receipt_hmac=""`` — such a receipt is integrity-checkable but NOT + authentic, and enforce-mode consumers must reject it. + """ + _validate(kind, actor, verdict) + env: dict = { + "receipt_id": receipt_id or uuid.uuid4().hex, + "kind": kind, + "run_id": run_id, + "session_id": session_id, + "ts": ts or _utc_now(), + "actor": actor, + "workspace": workspace, + "task_ref": dict(task_ref or {}), + "canonical_command": canonical_command, + "claim_sha256": claim_sha256, + "inputs_hash": inputs_hash, + "evidence": list(evidence or []), + "verdict": verdict, + "artifact_paths": list(artifact_paths or []), + } + env["receipt_sha256"] = receipt_sha256(env) + env["receipt_hmac"] = receipt_hmac(env, key) if key else "" + return env + + +def verify_hashes(envelope: dict, key: bytes | None) -> tuple[bool, bool]: + """Return ``(integrity, authentic)`` for an envelope. + + ``integrity`` — the stored ``receipt_sha256`` matches a recomputation. + ``authentic`` — a non-empty ``receipt_hmac`` matches an HMAC recomputed + with ``key`` (constant-time compare). No key or empty HMAC => not + authentic, regardless of integrity. + """ + integrity = hmac.compare_digest( + str(envelope.get("receipt_sha256", "")), receipt_sha256(envelope)) + stored_mac = str(envelope.get("receipt_hmac", "")) + if not key or not stored_mac: + return integrity, False + authentic = hmac.compare_digest(stored_mac, receipt_hmac(envelope, key)) + return integrity, authentic diff --git a/src/agent_ultra/ultracode/__init__.py b/src/agent_ultra/ultracode/__init__.py new file mode 100644 index 0000000..30e5860 --- /dev/null +++ b/src/agent_ultra/ultracode/__init__.py @@ -0,0 +1,38 @@ +"""Ultracode — deterministic multi-agent workflow engine for agent-ultra-kit. + +Public surface: + + from agent_ultra.ultracode import UltracodeEngine, demo_pool + eng = UltracodeEngine(demo_pool()) # offline, keyless + report = eng.run_script(eng.resolve("smoke")) + print(report.final_state) # COMPLETE + +Workflow scripts are Python modules with a ``META`` dict and an +``async def run(wf)`` coroutine. See engine.py for the ``wf`` surface +(agent / parallel / pipeline / budget / run_check / phase / log / evidence) +and the bundled examples/ for runnable workflows. +""" + +from .engine import ( + UltracodeEngine, + Workflow, + WorkflowError, + WorkflowReport, + BudgetExceeded, + summarize_events, + render_card, + print_card, + supports_rich, + build_receipt, + verify_receipt, + ultracode_home, + DEFAULTS, +) +from .mock import demo_pool, MockUltracodeClient + +__all__ = [ + "UltracodeEngine", "Workflow", "WorkflowError", "WorkflowReport", + "BudgetExceeded", "summarize_events", "render_card", "print_card", + "supports_rich", "build_receipt", "verify_receipt", "ultracode_home", + "DEFAULTS", "demo_pool", "MockUltracodeClient", +] diff --git a/src/agent_ultra/ultracode/engine.py b/src/agent_ultra/ultracode/engine.py new file mode 100644 index 0000000..56fc2e4 --- /dev/null +++ b/src/agent_ultra/ultracode/engine.py @@ -0,0 +1,1099 @@ +"""Ultracode — deterministic multi-agent workflow engine. + +An ultracode workflow is a plain Python module with a ``META`` dict and an +``async def run(wf)`` coroutine. The ``wf`` object fans work across many +bounded agents — each agent is ONE model call (optionally schema-validated) — +through ``parallel`` (barrier) and ``pipeline`` (no barrier) combinators, under +hard call/token budgets, with a resumable journal, broker-gated host checks, +and an anti-fraud receipt. + + META = {"name": "smoke", "description": "...", "phases": ["Ping"]} + + async def run(wf): + wf.phase("Ping") + a, b = await wf.parallel([ + lambda: wf.agent("Reply with exactly: pong-1", label="ping-1"), + lambda: wf.agent("Reply with exactly: pong-2", label="ping-2"), + ]) + return {"replies": [a, b]} + +The pattern this proves: multi-agent fan-out -> journal -> resume -> receipt +-> status. It is stdlib-only at the core; the model endpoint is any +OpenAI-compatible URL, and the offline mock route needs no key. + +The journal (``events.jsonl`` + ``journal.jsonl``) and the receipt +(``receipt.json``, checksummed) are the source of truth. A status card is a +pure display over the journal — a workflow's own text output never moves it. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import inspect +import json +import logging +import os +import re +import sys +import threading +import time +import traceback +from dataclasses import dataclass, field, asdict +from pathlib import Path + +from ..routes.pool import RoutePool +from ..routes.client import RouteError + +logger = logging.getLogger(__name__) + +_PKG_DIR = Path(__file__).resolve().parent + + +# -------------------------------------------------------------------------- +# Home + config (env -> defaults; no committed paths) +# -------------------------------------------------------------------------- + +def ultracode_home() -> Path: + env = os.environ.get("AGENT_ULTRA_HOME", "").strip() + base = Path(env) if env else Path.home() / ".agent-ultra" + return base / "ultracode" + + +DEFAULTS: dict = { + "max_workers": 6, # concurrent in-flight agents + "max_calls": 200, # model calls per run; exceeded -> HOLD + "token_budget": 0, # prompt+completion tokens per run; 0 = off + "max_tokens_agent": 4000, # per-call completion cap (agent may override) + "schema_retries": 2, # re-asks when JSON is invalid against a schema + "evidence_max_files": 6, + "evidence_max_chars": 8000, + "check_timeout": 30, + "report_agents_cap": 200, + "report_text_cap": 2000, +} + + +# -------------------------------------------------------------------------- +# Errors +# -------------------------------------------------------------------------- + +class WorkflowError(Exception): + """A run could not start or the script is malformed.""" + + +class BudgetExceeded(WorkflowError): + """The run hit its hard call/token ceiling; everything so far is journaled + so a resume continues from the paid-for prefix.""" + + +class AgentCallError(Exception): + """Every candidate route failed for one agent call (internal).""" + + +# -------------------------------------------------------------------------- +# JSON extraction + minimal schema validation (models wrap JSON in prose) +# -------------------------------------------------------------------------- + +_FENCE_RE = re.compile(r"```(?:json)?\s*(.*?)```", re.DOTALL) + + +def _scan_balanced(cand: str, start: int, opener: str, closer: str): + depth, in_str, esc = 0, False, False + for i in range(start, len(cand)): + ch = cand[i] + if in_str: + if esc: + esc = False + elif ch == "\\": + esc = True + elif ch == '"': + in_str = False + continue + if ch == '"': + in_str = True + elif ch == opener: + depth += 1 + elif ch == closer: + depth -= 1 + if depth == 0: + try: + return json.loads(cand[start:i + 1]) + except Exception: + return None + return None + + +def extract_json(text: str): + if not text: + return None + for cand in _FENCE_RE.findall(text) + [text]: + cand = cand.strip() + starts = sorted( + (cand.find(o), o, c) + for o, c in (("[", "]"), ("{", "}")) if cand.find(o) >= 0) + for start, opener, closer in starts: + parsed = _scan_balanced(cand, start, opener, closer) + if parsed is not None: + return parsed + return None + + +_TYPE_CHECKS = { + "object": lambda v: isinstance(v, dict), + "array": lambda v: isinstance(v, list), + "string": lambda v: isinstance(v, str), + "number": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool), + "integer": lambda v: isinstance(v, int) and not isinstance(v, bool), + "boolean": lambda v: isinstance(v, bool), + "null": lambda v: v is None, +} + + +def validate_schema(value, schema: dict, path: str = "$") -> list[str]: + errs: list[str] = [] + if not isinstance(schema, dict): + return errs + typ = schema.get("type") + if typ: + types = typ if isinstance(typ, list) else [typ] + if not any(_TYPE_CHECKS.get(t, lambda v: True)(value) for t in types): + return [f"{path}: expected {typ}, got {type(value).__name__}"] + if "enum" in schema and value not in schema["enum"]: + errs.append(f"{path}: {value!r} not in enum {schema['enum']}") + if isinstance(value, dict): + for req in schema.get("required", []): + if req not in value: + errs.append(f"{path}: missing required key {req!r}") + for k, sub in (schema.get("properties") or {}).items(): + if k in value: + errs.extend(validate_schema(value[k], sub, f"{path}.{k}")) + if isinstance(value, list) and isinstance(schema.get("items"), dict): + for i, item in enumerate(value): + errs.extend(validate_schema(item, schema["items"], f"{path}[{i}]")) + return errs + + +# -------------------------------------------------------------------------- +# Helpers +# -------------------------------------------------------------------------- + +def _sha(text: str) -> str: + return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest() + + +def _slug(text: str, limit: int = 40) -> str: + s = re.sub(r"[^a-zA-Z0-9]+", "-", (text or "run")).strip("-").lower() + return s[:limit] or "run" + + +def _truncate(text: str, limit: int) -> str: + text = text or "" + if len(text) <= limit: + return text + return text[:limit] + f"\n...(truncated, {len(text)} chars total)" + + +def _utc() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +# -------------------------------------------------------------------------- +# Terminal rendering — DISPLAY ONLY. Plain ASCII unless the terminal provably +# supports better; any failure degrades and never raises into the run. +# -------------------------------------------------------------------------- + +_GLYPHS_RICH = {"completed": "●", "deployed": "○", + "failed": "⚠", "blocked": "⚠"} +_GLYPHS_PLAIN = {"completed": "#", "deployed": ".", "failed": "!", "blocked": "!"} + + +def supports_rich(stream=None, env=None) -> bool: + try: + env = os.environ if env is None else env + if str(env.get("AGENT_ULTRA_PLAIN", "")).strip() not in ("", "0"): + return False + if env.get("NO_COLOR") or env.get("CI"): + return False + stream = sys.stdout if stream is None else stream + if not (hasattr(stream, "isatty") and stream.isatty()): + return False + term = str(env.get("TERM", "")).strip().lower() + if term in ("dumb", "unknown"): + return False + if os.name == "nt" and not (env.get("WT_SESSION") or env.get("ANSICON") + or env.get("TERM_PROGRAM") or term): + return False + enc = str(getattr(stream, "encoding", "") or "").lower() + if enc and "utf" not in enc: + return False + return True + except Exception: + return False + + +def _ascii(text: str) -> str: + return str(text).encode("ascii", "replace").decode("ascii") + + +def _card_text(s: dict, glyphs: dict, plain: bool) -> str: + counts = s.get("counts") if isinstance(s.get("counts"), dict) else {} + + def n(key): + try: + return int(counts.get(key, 0) or 0) + except Exception: + return 0 + + marks = "" + for d in (s.get("dots") or []): + st = str((d or {}).get("state", "")) if isinstance(d, dict) else "" + marks += glyphs.get(st, glyphs["deployed"]) + status = str(s.get("final_state") or "") or "RUNNING" + lines = ["ULTRACODE"] + if s.get("run_id"): + lines.append(f"Run: {s['run_id']}") + if s.get("name"): + lines.append(f"Name: {s['name']}") + if s.get("phase"): + lines.append(f"Phase: {s['phase']}") + lines.append(f"Agents: {n('completed')}/{n('deployed')} complete") + lines.append(f"Failed: {n('failed')}") + lines.append(f"Blocked: {n('blocked')}") + queued = n("planned") - n("deployed") + if queued > 0: + lines.append(f"Queued: {queued}") + if marks: + lines.append("Progress: [" + marks + "]" if plain + else "Progress: " + " ".join(marks)) + lines.append(f"Status: {status}") + if s.get("error"): + lines.append("Error: " + " ".join(str(s["error"]).split())[:200]) + if s.get("receipt_path"): + lines.append(f"Receipt: {s['receipt_path']}") + text = "\n".join(lines) + return _ascii(text) if plain else text + + +def render_card(summary, rich: bool | None = None, stream=None) -> str: + """Format a summarize_events() dict for a terminal. Never raises.""" + try: + if rich is None: + rich = supports_rich(stream) + s = summary if isinstance(summary, dict) else {} + if rich: + try: + return _card_text(s, _GLYPHS_RICH, plain=False) + except Exception: + pass + return _card_text(s, _GLYPHS_PLAIN, plain=True) + except Exception: + return ("ULTRACODE\nStatus: UNKNOWN (render failed; the journal " + "events.jsonl and receipt.json remain the source of truth)") + + +def print_card(summary, stream=None) -> None: + try: + out = stream or sys.stdout + try: + print(render_card(summary, stream=out), file=out) + except Exception: + print(render_card(summary, rich=False), file=out) + except Exception: + pass + + +# -------------------------------------------------------------------------- +# Event stream — typed runtime events (the card replays this) +# -------------------------------------------------------------------------- + +FINAL_STATES = {"completed": ("ultracode_completed", "COMPLETE"), + "aborted_budget": ("ultracode_hold", "HOLD"), + "failed": ("ultracode_failed", "FAILED"), + "cancelled": ("ultracode_cancelled", "CANCELLED")} + +_PHASE_ALIASES = ( + ("verification_started", re.compile(r"(?i)verif|skeptic|adversar")), + ("synthesis_started", re.compile(r"(?i)synth|summar")), + ("fan_in_started", re.compile(r"(?i)fan[-_ ]?in|merge|dedup|reconcil")), +) +_PHASE_KIND_ALIASES = {"verification": "verification_started", + "synthesis": "synthesis_started", + "fan_in": "fan_in_started"} + + +class EventLog: + def __init__(self, path: Path, run_id: str): + self.path = path + self.run_id = run_id + self._seq = 0 + self._lock = threading.Lock() + self.path.parent.mkdir(parents=True, exist_ok=True) + + def emit(self, event: str, **fields) -> None: + with self._lock: + self._seq += 1 + entry = {"v": 1, "event": event, "ultracode_run_id": self.run_id, + "seq": self._seq, "ts": _utc(), **fields} + try: + with open(self.path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(entry, ensure_ascii=False, + default=str) + "\n") + except OSError: + logger.warning("event write failed: %s", self.path) + + +def summarize_events(events_path: str | Path) -> dict: + """Replay events.jsonl into the card model. Single source of truth for + rendering — what the card shows is what the journal proves.""" + order: list[str] = [] + state: dict[str, str] = {} + roles: dict[str, str] = {} + planned = 0 + phase = name = final_state = run_id = started = last_ts = "" + error = receipt = artifact = "" + try: + lines = Path(events_path).read_text(encoding="utf-8").splitlines() + except OSError: + return {"ok": False, "error": f"no events at {events_path}"} + for line in lines: + try: + e = json.loads(line) + except ValueError: + continue + ev = e.get("event", "") + last_ts = e.get("ts", last_ts) + aid = e.get("agent_id", "") + run_id = run_id or e.get("ultracode_run_id", "") + if ev == "ultracode_started": + name = e.get("name", "") + started = e.get("ts", "") + elif ev == "phase_started": + phase = e.get("phase", phase) + elif ev == "agent_planned": + planned += 1 + roles[aid] = e.get("agent_role", "") + elif ev in ("agent_deployed", "agent_completed", "agent_failed", + "agent_blocked"): + if aid not in state: + order.append(aid) + state[aid] = ev.replace("agent_", "") + elif ev in ("ultracode_completed", "ultracode_hold", + "ultracode_failed", "ultracode_cancelled"): + final_state = e.get("final_state", "") + error = e.get("error", "") + receipt = e.get("receipt_path", "") + artifact = e.get("artifact_json", "") + counts = {"planned": planned, "deployed": len(order), + "completed": sum(1 for s in state.values() if s == "completed"), + "failed": sum(1 for s in state.values() if s == "failed"), + "blocked": sum(1 for s in state.values() if s == "blocked")} + return {"ok": True, "run_id": run_id, "name": name, "phase": phase, + "final_state": final_state, "counts": counts, + "dots": [{"agent_id": a, "role": roles.get(a, ""), + "state": state[a]} for a in order], + "started_utc": started, "last_event_utc": last_ts, + "error": error, "receipt_path": receipt, "artifact_json": artifact} + + +# -------------------------------------------------------------------------- +# Journal (replay cache for resume) + Budget +# -------------------------------------------------------------------------- + +class Journal: + def __init__(self, path: Path): + self.path = path + self.path.parent.mkdir(parents=True, exist_ok=True) + + def append(self, entry: dict) -> None: + try: + with open(self.path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(entry, ensure_ascii=False, + default=str) + "\n") + except OSError: + logger.warning("journal write failed: %s", self.path) + + @staticmethod + def load_replay_cache(path: Path) -> dict: + cache: dict[str, list] = {} + try: + for line in path.read_text(encoding="utf-8").splitlines(): + try: + e = json.loads(line) + except ValueError: + continue + if e.get("type") == "agent" and e.get("status") == "ok": + cache.setdefault(e["key"], []).append(e) + except OSError: + pass + return cache + + +class Budget: + def __init__(self, max_calls: int, token_budget: int, usage_source=None): + self.max_calls = int(max_calls) + self.token_budget = int(token_budget) + self.calls = 0 + self._src = usage_source + self._baseline = self._usage_total() + + def _usage_total(self) -> int: + snap = getattr(self._src, "usage_snapshot", None) + if snap is None: + return 0 + try: + u = snap() + return int(u.get("prompt_tokens", 0)) + int(u.get("completion_tokens", 0)) + except Exception: + return 0 + + def tokens_spent(self) -> int: + return max(0, self._usage_total() - self._baseline) + + def remaining_calls(self) -> int: + return max(0, self.max_calls - self.calls) + + def check_or_raise(self) -> None: + if self.max_calls and self.calls >= self.max_calls: + raise BudgetExceeded( + f"model-call budget exhausted ({self.calls}/{self.max_calls})") + if self.token_budget and self.tokens_spent() >= self.token_budget: + raise BudgetExceeded( + f"token budget exhausted ({self.tokens_spent()}/{self.token_budget})") + + +# -------------------------------------------------------------------------- +# Data model +# -------------------------------------------------------------------------- + +@dataclass +class AgentRecord: + seq: int + label: str + phase: str + prompt_sha: str + status: str = "ok" # ok | error | invalid_schema + route: str = "" + text: str = "" + parsed: object = None + error: str = "" + calls: int = 0 + cached: bool = False + duration_s: float = 0.0 + + +@dataclass +class WorkflowReport: + name: str + run_id: str + script_path: str + script_sha: str + args: object = None + status: str = "completed" # completed | failed | aborted_budget | cancelled + final_state: str = "" # COMPLETE | HOLD | FAILED | CANCELLED + result: object = None + error: str = "" + counts: dict = field(default_factory=dict) + events_path: str = "" + phases: list = field(default_factory=list) + model_calls: int = 0 + cached_hits: int = 0 + agents_run: int = 0 + tokens: dict = field(default_factory=dict) + duration_s: float = 0.0 + started_utc: str = "" + artifact_dir: str = "" + artifact_json: str = "" + journal_path: str = "" + receipt_path: str = "" + resumed_from: str = "" + agents: list = field(default_factory=list) + log: list = field(default_factory=list) + + +# -------------------------------------------------------------------------- +# Receipt — structural proof a real run happened (checksummed) +# -------------------------------------------------------------------------- + +def build_receipt(report: WorkflowReport, records: list) -> dict: + receipt = { + "schema_version": 1, + "run_id": report.run_id, + "name": report.name, + "script_sha256": report.script_sha, + "started_utc": report.started_utc, + "status": report.status, + "model_calls": report.model_calls, + "cached_hits": report.cached_hits, + "tokens": report.tokens, + "agents": [ + {"seq": r.seq, "label": r.label, "phase": r.phase, "route": r.route, + "status": r.status, "cached": r.cached, "prompt_sha256": r.prompt_sha, + "output_sha256": _sha(r.text or "")} + for r in records], + } + receipt["receipt_sha256"] = _receipt_checksum(receipt) + return receipt + + +def _receipt_checksum(receipt: dict) -> str: + body = {k: v for k, v in receipt.items() if k != "receipt_sha256"} + return _sha(json.dumps(body, sort_keys=True, ensure_ascii=False)) + + +def verify_receipt(receipt: dict) -> bool: + return (isinstance(receipt, dict) + and receipt.get("receipt_sha256") == _receipt_checksum(receipt)) + + +# -------------------------------------------------------------------------- +# The `wf` surface +# -------------------------------------------------------------------------- + +class Workflow: + def __init__(self, *, cfg: dict, pool: RoutePool, journal: Journal, + events: EventLog, replay: dict, budget: Budget, + broker_factory, args, run_dir: Path, status_path: Path, + run_id: str, name: str): + self.args = args + self.budget = budget + self.phases: list[str] = [] + self.records: list[AgentRecord] = [] + self.log_lines: list[str] = [] + self.counts = {"planned": 0, "deployed": 0, "completed": 0, + "failed": 0, "blocked": 0} + self._cfg = cfg + self._pool = pool + self._journal = journal + self._events = events + self._replay = replay + self._broker_factory = broker_factory + self._broker = None + self._run_dir = run_dir + self._status_path = status_path + self._run_id = run_id + self._name = name + self._seq = 0 + self._cached_hits = 0 + self._current_phase = "" + self._started = time.time() + self._sem: asyncio.Semaphore | None = None + + # -- progress ---------------------------------------------------------- + + def phase(self, title: str, kind: str = "") -> None: + self._current_phase = str(title) + self.phases.append(self._current_phase) + self._events.emit("phase_started", phase=self._current_phase) + alias = _PHASE_KIND_ALIASES.get((kind or "").strip().lower()) + if not alias: + for name, rx in _PHASE_ALIASES: + if rx.search(self._current_phase): + alias = name + break + if alias: + self._events.emit(alias, phase=self._current_phase) + self.log(f"PHASE: {title}") + + def log(self, message: str) -> None: + line = f"{_utc()} {message}" + self.log_lines.append(line) + logger.info("ultracode[%s]: %s", self._name, message) + self._journal.append({"type": "log", "ts": _utc(), "message": str(message)}) + self._write_status(str(message)) + + def _write_status(self, last: str = "") -> None: + try: + self._status_path.parent.mkdir(parents=True, exist_ok=True) + self._status_path.write_text(json.dumps({ + "run_id": self._run_id, "name": self._name, + "phase": self._current_phase, "phases": self.phases, + "model_calls": self.budget.calls, "cached_hits": self._cached_hits, + "agents_run": self._seq, "counts": dict(self.counts), + "events_path": str(self._events.path), + "elapsed_s": round(time.time() - self._started, 1), + "started_utc": _utc(), "last_log": last[:200], + }, ensure_ascii=False), encoding="utf-8") + except OSError: + pass + + # -- evidence ---------------------------------------------------------- + + def evidence(self, paths, max_files: int | None = None, + max_chars: int | None = None) -> str: + max_files = max_files or self._cfg["evidence_max_files"] + max_chars = max_chars or self._cfg["evidence_max_chars"] + chunks = [] + for p in list(paths)[:max_files]: + path = Path(p) + try: + body = path.read_text(encoding="utf-8", errors="replace") + chunks.append(f"--- {path} ---\n{_truncate(body, max_chars)}") + except OSError as e: + chunks.append(f"--- {path} --- (unreadable: {e})") + return "\n\n".join(chunks) + + # -- broker-gated host checks ----------------------------------------- + + def run_check(self, command: str, reason: str = "") -> dict: + """Execute a host command through the risk-tiered broker in CRITIC + mode: only SAFE (pure-read) commands auto-run; everything else is + classified, ledgered, and returned as requires_approval.""" + if self._broker is None: + self._broker = self._broker_factory() + res = self._broker.run(command, reason=reason or f"ultracode {self._name}") + entry = asdict(res) if hasattr(res, "__dataclass_fields__") else dict(res) + self._journal.append({"type": "check", "ts": _utc(), **entry}) + self._events.emit("check_executed", command=entry.get("command", ""), + risk_tier=entry.get("risk_tier", ""), + status=entry.get("status", "")) + return entry + + # -- the agent primitive ---------------------------------------------- + + async def agent(self, prompt: str, *, schema: dict | None = None, + route: str = "", label: str = "", phase: str = "", + system: str = "", evidence=None, + max_tokens: int | None = None): + if self._sem is None: + self._sem = asyncio.Semaphore(int(self._cfg["max_workers"])) + self._seq += 1 + seq = self._seq + label = label or f"agent-{seq}" + phase = phase or self._current_phase + max_tokens = int(max_tokens or self._cfg["max_tokens_agent"]) + + parts = [] + if system: + parts.append(system.strip()) + if evidence: + parts.append("EVIDENCE (read-only excerpts):\n" + self.evidence(evidence)) + parts.append(str(prompt)) + if schema: + parts.append("Respond with ONLY a JSON value matching this schema " + "-- no prose, no markdown fences:\n" + json.dumps(schema)) + full_prompt = "\n\n".join(parts) + key = _sha(f"{full_prompt}\x00{route}\x00{max_tokens}") + + rec = AgentRecord(seq=seq, label=label, phase=phase, + prompt_sha=_sha(full_prompt)) + agent_id = self._aid(rec) + task_summary = " ".join(str(prompt).split())[:140] + self.counts["planned"] += 1 + self._events.emit("agent_planned", agent_id=agent_id, agent_role=label, + phase=phase, task_summary=task_summary) + + cached = self._replay_take(key) + if cached is not None: + rec.status, rec.cached = "ok", True + rec.text = cached.get("text", "") + rec.parsed = cached.get("parsed") + rec.route = cached.get("route", "") + self._cached_hits += 1 + self.counts["deployed"] += 1 + self.counts["completed"] += 1 + artifact = self._write_agent_artifact(rec) + self._events.emit("agent_deployed", agent_id=agent_id, + agent_role=label, phase=phase, cached=True) + self._events.emit("agent_completed", agent_id=agent_id, + agent_role=label, phase=phase, status="completed", + route=rec.route, cached=True, + output_artifact_path=artifact) + self.records.append(rec) + self._journal.append(self._entry(key, rec)) + self._write_status(f"replayed {label}") + return rec.parsed if schema else rec.text + + start = time.time() + async with self._sem: + self.counts["deployed"] += 1 + self._events.emit("agent_deployed", agent_id=agent_id, + agent_role=label, phase=phase, cached=False) + try: + await asyncio.to_thread(self._attempts_sync, rec, full_prompt, + schema, route, max_tokens, seq) + except BudgetExceeded as e: + rec.status, rec.error = "error", "budget exceeded" + rec.duration_s = round(time.time() - start, 1) + self.counts["blocked"] += 1 + self._events.emit("agent_blocked", agent_id=agent_id, + agent_role=label, phase=phase, + status="blocked", error=str(e)) + self.records.append(rec) + self._journal.append(self._entry(key, rec)) + raise + rec.duration_s = round(time.time() - start, 1) + self.records.append(rec) + self._journal.append(self._entry(key, rec)) + artifact = self._write_agent_artifact(rec) + if rec.status == "ok": + self.counts["completed"] += 1 + self._events.emit("agent_completed", agent_id=agent_id, + agent_role=label, phase=phase, status="completed", + route=rec.route, calls=rec.calls, + duration_s=rec.duration_s, cached=False, + output_artifact_path=artifact) + else: + self.counts["failed"] += 1 + self._events.emit("agent_failed", agent_id=agent_id, + agent_role=label, phase=phase, status="failed", + error=rec.error, output_artifact_path=artifact) + self._write_status(f"{rec.status}: {label}") + if rec.status != "ok": + self.log(f"agent {label} {rec.status}: {rec.error[:200]}") + return None + return rec.parsed if schema else rec.text + + @staticmethod + def _aid(rec: AgentRecord) -> str: + return f"a{rec.seq:03d}" + + def _write_agent_artifact(self, rec: AgentRecord) -> str: + try: + d = self._run_dir / "agents" + d.mkdir(parents=True, exist_ok=True) + p = d / f"{self._aid(rec)}.json" + p.write_text(json.dumps(asdict(rec), ensure_ascii=False, indent=1, + default=str), encoding="utf-8") + return str(p) + except OSError: + return "" + + def _attempts_sync(self, rec, full_prompt, schema, route, max_tokens, seq): + attempts = (int(self._cfg["schema_retries"]) + 1) if schema else 1 + prompt = full_prompt + last_errs: list[str] = [] + for attempt in range(attempts): + try: + text, used = self._chat_with_routes(prompt, route, max_tokens, + seq, rec) + except AgentCallError as e: + rec.status, rec.error = "error", str(e) + return + rec.text, rec.route = text, used + if not schema: + rec.status = "ok" + return + parsed = extract_json(text) + last_errs = (["no JSON found in response"] if parsed is None + else validate_schema(parsed, schema)) + if not last_errs: + rec.parsed, rec.status = parsed, "ok" + return + if attempt < attempts - 1: + self._events.emit("agent_heartbeat", agent_id=self._aid(rec), + agent_role=rec.label, phase=rec.phase, + status="retrying_schema", + detail="; ".join(last_errs[:3])) + prompt = (full_prompt + + "\n\nYour previous response was INVALID: " + + "; ".join(last_errs[:5]) + + "\nReturn ONLY corrected JSON matching the schema.") + rec.status = "invalid_schema" + rec.error = "; ".join(last_errs[:5]) + + def _chat_with_routes(self, prompt, route, max_tokens, seq, rec): + candidates = [route] if route else self._pool.assign(seq) + if not candidates: + raise AgentCallError("no routes available") + last = None + for cand in candidates: + self.budget.check_or_raise() + self.budget.calls += 1 + rec.calls += 1 + try: + return self._pool.chat(cand, prompt, max_tokens), cand + except RouteError as e: + last = e + if not route: + self._pool.mark_dead(cand) + self._events.emit("agent_heartbeat", agent_id=self._aid(rec), + agent_role=rec.label, phase=rec.phase, + status="route_fallthrough", + detail=f"{cand}: {str(e)[:160]}") + raise AgentCallError(f"all routes failed (last: {last})") + + def _replay_take(self, key: str): + entries = self._replay.get(key) + return entries.pop(0) if entries else None + + def _entry(self, key: str, rec: AgentRecord) -> dict: + return {"type": "agent", "ts": _utc(), "key": key, **asdict(rec)} + + # -- combinators ------------------------------------------------------- + + async def parallel(self, thunks) -> list: + """Run zero-arg async thunks concurrently and WAIT FOR ALL (barrier). + A thunk that raises resolves to None; only BudgetExceeded propagates.""" + async def one(t): + try: + return await t() + except BudgetExceeded: + raise + except Exception as e: + self.log(f"parallel task failed: {e.__class__.__name__}: {e}") + return None + return list(await asyncio.gather(*(one(t) for t in thunks))) + + async def pipeline(self, items, *stages) -> list: + """Run each item through all stages independently -- NO barrier + between stages. Stage callables receive (prev[, item[, index]]) and + may be sync or async. A stage that raises drops its item to None.""" + items = list(items) + + async def chain(item, idx): + val = item + for si, stage in enumerate(stages): + try: + val = await self._apply_stage(stage, val, item, idx) + except BudgetExceeded: + raise + except Exception as e: + self.log(f"pipeline item {idx} dropped at stage {si}: " + f"{e.__class__.__name__}: {e}") + return None + return val + return list(await asyncio.gather( + *(chain(it, i) for i, it in enumerate(items)))) + + async def _apply_stage(self, stage, prev, item, idx): + try: + n_params = len([ + p for p in inspect.signature(stage).parameters.values() + if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD) + and p.default is p.empty]) + except (TypeError, ValueError): + n_params = 1 + out = stage(*(prev, item, idx)[:max(1, min(3, n_params))]) + if inspect.isawaitable(out): + out = await out + return out + + +# -------------------------------------------------------------------------- +# Engine +# -------------------------------------------------------------------------- + +class UltracodeEngine: + """Loads and runs one ultracode workflow. Construct with an injected + RoutePool (mock or live) and optional broker/home for offline tests.""" + + def __init__(self, pool: RoutePool, config: dict | None = None, + broker=None, home: Path | None = None, usage_source=None): + self.pool = pool + self.home = Path(home) if home else ultracode_home() + self.cfg = dict(DEFAULTS) + if config: + self.cfg.update({k: v for k, v in config.items() if k in DEFAULTS}) + self._broker = broker + self._usage_source = usage_source + + # -- script resolution ------------------------------------------------ + + def scripts_dir(self) -> Path: + return self.home / "scripts" + + def examples_dir(self) -> Path: + return _PKG_DIR / "examples" + + def resolve(self, name_or_path: str) -> Path: + cand = Path(name_or_path) + if cand.is_file(): + return cand + stem = name_or_path if name_or_path.endswith(".py") else name_or_path + ".py" + for base in (self.scripts_dir(), self.examples_dir()): + p = base / stem + if p.is_file(): + return p + raise WorkflowError( + f"no workflow named {name_or_path!r} -- looked in " + f"{self.scripts_dir()} and the bundled examples. " + "Run `agent-ultra ultracode list` to see what exists.") + + def list_scripts(self) -> dict: + out = {"saved": [], "examples": []} + for kind, base in (("saved", self.scripts_dir()), + ("examples", self.examples_dir())): + if base.is_dir(): + for p in sorted(base.glob("*.py")): + if p.name.startswith("_"): + continue + meta = self._peek_meta(p) + out[kind].append({"name": p.stem, "path": str(p), + "description": meta.get("description", "")[:160]}) + return out + + @staticmethod + def _peek_meta(path: Path) -> dict: + import ast + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in tree.body: + if (isinstance(node, ast.Assign) + and any(getattr(t, "id", "") == "META" for t in node.targets)): + val = ast.literal_eval(node.value) + return val if isinstance(val, dict) else {} + except Exception: + pass + return {} + + # -- run -------------------------------------------------------------- + + def run_script(self, script_path, args=None, resume_run_id: str = "") -> WorkflowReport: + path = Path(script_path).resolve() + source = path.read_text(encoding="utf-8") + script_sha = _sha(source) + module = self._load_module(path, script_sha) + + meta = getattr(module, "META", None) + if not isinstance(meta, dict) or not str(meta.get("name", "")).strip(): + raise WorkflowError(f"{path.name}: META must be a dict with a 'name'") + run_fn = getattr(module, "run", None) + if not inspect.iscoroutinefunction(run_fn): + raise WorkflowError(f"{path.name}: must define `async def run(wf)`") + + name = _slug(str(meta["name"])) + run_id = (time.strftime("%Y%m%dT%H%M%S") + + f"-{name}-{_sha(os.urandom(8).hex())[:6]}") + run_dir = self.home / "runs" / run_id + run_dir.mkdir(parents=True, exist_ok=True) + status_path = self.home / "runs" / f".status-{run_id}.json" + + replay: dict = {} + if resume_run_id: + old = self.home / "runs" / resume_run_id / "journal.jsonl" + replay = Journal.load_replay_cache(old) + if not replay: + logger.warning("resume: no replayable journal at %s", old) + + budget = Budget(self.cfg["max_calls"], self.cfg["token_budget"], + usage_source=self._usage_source) + journal = Journal(run_dir / "journal.jsonl") + events = EventLog(run_dir / "events.jsonl", run_id) + broker_factory = ((lambda: self._broker) if self._broker is not None + else (lambda: self._default_broker())) + + wf = Workflow(cfg=self.cfg, pool=self.pool, journal=journal, + events=events, replay=replay, budget=budget, + broker_factory=broker_factory, args=args, run_dir=run_dir, + status_path=status_path, run_id=run_id, name=name) + + report = WorkflowReport( + name=name, run_id=run_id, script_path=str(path), + script_sha=script_sha, args=args, started_utc=_utc(), + resumed_from=resume_run_id or "", artifact_dir=str(run_dir), + journal_path=str(journal.path), events_path=str(events.path)) + journal.append({"type": "run_start", "ts": report.started_utc, + "run_id": run_id, "name": name, "script": str(path), + "script_sha": script_sha, "args": args, + "resumed_from": resume_run_id or ""}) + events.emit("ultracode_started", name=name, script_sha=script_sha, + task_summary=" ".join(str(meta.get("description", "")).split())[:140], + resumed_from=resume_run_id or "") + + start = time.time() + try: + report.result = asyncio.run(run_fn(wf)) + report.status = "completed" + except BudgetExceeded as e: + report.status, report.error = "aborted_budget", str(e) + except WorkflowError as e: + report.status, report.error = "failed", str(e) + except (KeyboardInterrupt, asyncio.CancelledError) as e: + report.status = "cancelled" + report.error = f"cancelled ({e.__class__.__name__})" + except Exception: + report.status = "failed" + report.error = _truncate(traceback.format_exc(), 4000) + finally: + try: + status_path.unlink() + except OSError: + pass + + report.duration_s = round(time.time() - start, 1) + report.model_calls = budget.calls + report.cached_hits = wf._cached_hits + report.agents_run = len(wf.records) + report.tokens = {"total": budget.tokens_spent()} + report.phases = wf.phases + report.counts = dict(wf.counts) + report.log = wf.log_lines[-200:] + cap = int(self.cfg["report_agents_cap"]) + tcap = int(self.cfg["report_text_cap"]) + report.agents = [{**asdict(r), "text": _truncate(r.text, tcap)} + for r in wf.records[:cap]] + + event_name, final_state = FINAL_STATES.get( + report.status, ("ultracode_failed", "FAILED")) + report.final_state = final_state + self._finalize(report, wf, journal, run_dir) + events.emit(event_name, final_state=final_state, counts=dict(wf.counts), + error=report.error[:500], receipt_path=report.receipt_path, + artifact_json=report.artifact_json) + return report + + def _finalize(self, report, wf, journal, run_dir) -> None: + receipt = build_receipt(report, wf.records) + try: + (run_dir / "receipt.json").write_text( + json.dumps(receipt, ensure_ascii=False, indent=1), encoding="utf-8") + report.receipt_path = str(run_dir / "receipt.json") + except OSError: + logger.warning("could not write receipt") + try: + (run_dir / "report.json").write_text( + json.dumps(asdict(report), ensure_ascii=False, indent=1, + default=str), encoding="utf-8") + report.artifact_json = str(run_dir / "report.json") + except OSError: + logger.warning("could not write report artifact") + journal.append({"type": "run_end", "ts": _utc(), "status": report.status, + "model_calls": report.model_calls, "error": report.error[:500]}) + self._append_log(report) + + def _append_log(self, report) -> None: + log = self.home / "ultracode-log.jsonl" + try: + log.parent.mkdir(parents=True, exist_ok=True) + with open(log, "a", encoding="utf-8") as fh: + fh.write(json.dumps({ + "ts": _utc(), "run_id": report.run_id, "name": report.name, + "status": report.status, "final_state": report.final_state, + "counts": report.counts, "events_path": report.events_path, + "model_calls": report.model_calls, + "cached_hits": report.cached_hits, "tokens": report.tokens, + "duration_s": report.duration_s, + "artifact_json": report.artifact_json, + "receipt": report.receipt_path, + "result_head": _truncate( + json.dumps(report.result, default=str, ensure_ascii=False) + if report.result is not None else "", 300), + "error": report.error[:300], + }, ensure_ascii=False) + "\n") + except OSError: + pass + + def _default_broker(self): + from ..broker.broker import CommandBroker, CRITIC_TIERS + return CommandBroker(ledger_path=self.home / "broker-ledger.jsonl", + auto_run_tiers=CRITIC_TIERS, + timeout=self.cfg["check_timeout"]) + + @staticmethod + def _load_module(path: Path, script_sha: str): + from importlib.util import spec_from_file_location, module_from_spec + mod_name = f"agent_ultra_workflow_{path.stem}_{script_sha[:12]}" + sys.modules.pop(mod_name, None) + spec = spec_from_file_location(mod_name, str(path)) + if spec is None or spec.loader is None: + raise WorkflowError(f"cannot load workflow script: {path}") + mod = module_from_spec(spec) + sys.modules[mod_name] = mod + try: + spec.loader.exec_module(mod) + except Exception as e: + sys.modules.pop(mod_name, None) + raise WorkflowError( + f"{path.name} failed to import: {e.__class__.__name__}: {e}") + return mod diff --git a/src/agent_ultra/ultracode/examples/__init__.py b/src/agent_ultra/ultracode/examples/__init__.py new file mode 100644 index 0000000..b60699c --- /dev/null +++ b/src/agent_ultra/ultracode/examples/__init__.py @@ -0,0 +1,2 @@ +# Bundled ultracode example workflows. Loaded by path (not imported); this +# file just makes the directory a package so the examples ship in the wheel. diff --git a/src/agent_ultra/ultracode/examples/review.py b/src/agent_ultra/ultracode/examples/review.py new file mode 100644 index 0000000..9b684d9 --- /dev/null +++ b/src/agent_ultra/ultracode/examples/review.py @@ -0,0 +1,152 @@ +"""Adversarial review workflow: parallel finders (one per lens) hunt problems +in a target file/dir, findings are deduped, then each faces independent +skeptic votes -- only findings a majority fail to refute survive. The +canonical fan-out -> verify -> synthesize shape. + + agent-ultra ultracode run review --mock --args '{"target": "./src"}' + +args (object): + target (required) file or directory to review + question (optional) what to review for + lenses (optional) finder lenses (default 4 below) + votes (optional) skeptics per finding (default 3, majority rules) + max_files (optional) evidence cap (default 8) +""" + +from pathlib import Path + +META = { + "name": "review", + "description": "fan-out finders -> dedupe -> per-finding skeptic votes -> synthesis", + "phases": ["Find", "Verify", "Synthesize"], +} + +DEFAULT_LENSES = ["correctness", "security", "failure-modes", "data-integrity"] + +FINDINGS_SCHEMA = { + "type": "object", + "properties": { + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "claim": {"type": "string"}, + "severity": {"type": "string", + "enum": ["critical", "high", "medium", "low"]}, + "anchor": {"type": "string"}, + }, + "required": ["claim", "severity"], + }, + }, + }, + "required": ["findings"], +} + +VERDICT_SCHEMA = { + "type": "object", + "properties": { + "refuted": {"type": "boolean"}, + "reasoning": {"type": "string"}, + }, + "required": ["refuted", "reasoning"], +} + +SKEPTIC_ANGLES = [ + "Assume the claim is WRONG and find the evidence that disproves it.", + "Refute it if the impact is theoretical or already mitigated.", + "Refute it if the cited anchor does not actually support the claim.", +] + + +def _pick_files(target: Path, cap: int) -> list: + if target.is_file(): + return [target] + exts = {".py", ".js", ".ts", ".md", ".yaml", ".yml", ".json", ".toml"} + picked = [p for p in sorted(target.rglob("*")) + if p.is_file() and p.suffix.lower() in exts + and "node_modules" not in p.parts and ".git" not in p.parts] + picked.sort(key=lambda p: p.stat().st_size, reverse=True) + return picked[:cap] + + +def _dedupe(findings: list) -> list: + kept = [] + for f in findings: + words = set(f["claim"].lower().split()) + dup = any(len(words & set(k["claim"].lower().split())) + / max(1, len(words | set(k["claim"].lower().split()))) > 0.6 + for k in kept) + if not dup: + kept.append(f) + return kept + + +async def run(wf): + a = wf.args if isinstance(wf.args, dict) else {} + target = Path(str(a.get("target", ".") or ".")).expanduser() + if not target.exists(): + raise ValueError(f"args.target does not exist: {target}") + question = a.get("question") or ( + "Review this material for real defects: correctness bugs, security " + "holes, data-loss paths, and operational failure modes.") + lenses = list(a.get("lenses") or DEFAULT_LENSES) + votes = max(1, int(a.get("votes", 3))) + cap = int(a.get("max_files", 8)) + + files = _pick_files(target, cap) + if not files: + return {"confirmed": [], "note": f"no reviewable files under {target}"} + evidence = wf.evidence(files, max_files=cap) + + wf.phase("Find") + finder_results = await wf.parallel([ + (lambda lens=lens: wf.agent( + f"You are a {lens} critic. {question}\n" + f"Report at most 4 findings through the {lens} lens ONLY, each with " + "a short verbatim anchor from the evidence.\n\nEVIDENCE:\n" + evidence, + schema=FINDINGS_SCHEMA, label=f"find:{lens}", phase="Find")) + for lens in lenses]) + all_findings = [] + for lens, res in zip(lenses, finder_results): + for f in (res or {}).get("findings", []): + all_findings.append({**f, "lens": lens}) + findings = _dedupe(all_findings) + if not findings: + return {"confirmed": [], "note": "no findings survived dedupe"} + + wf.phase("Verify") + + async def verify(finding, _item, idx): + ballots = await wf.parallel([ + (lambda k=k: wf.agent( + f"A reviewer claims ({finding['lens']}, {finding['severity']}): " + f"{finding['claim']}\nSkeptic angle: " + f"{SKEPTIC_ANGLES[k % len(SKEPTIC_ANGLES)]}\n" + "Try hard to REFUTE it. If you cannot, refuted=false.\n\n" + "EVIDENCE:\n" + evidence, + schema=VERDICT_SCHEMA, label=f"verify:{idx}:{k}", phase="Verify")) + for k in range(votes)]) + real = [b for b in ballots if b] + refutes = sum(1 for b in real if b.get("refuted")) + return {**finding, "votes": len(real), "refuted_by": refutes, + "confirmed": bool(real) and refutes <= len(real) // 2} + + judged = [j for j in await wf.pipeline(findings, verify) if j] + confirmed = [j for j in judged if j["confirmed"]] + + wf.phase("Synthesize") + synthesis = None + if confirmed: + listing = "\n".join(f"- ({c['lens']}/{c['severity']}) {c['claim']}" + for c in confirmed) + synthesis = await wf.agent( + "These findings each survived independent skeptic votes:\n" + f"{listing}\n\nRank them by urgency; one line each, most urgent first.", + label="synthesis") + + return {"target": str(target), + "files_reviewed": [str(f) for f in files], + "confirmed": confirmed, + "rejected": [j for j in judged if not j["confirmed"]], + "synthesis": synthesis} diff --git a/src/agent_ultra/ultracode/examples/smoke.py b/src/agent_ultra/ultracode/examples/smoke.py new file mode 100644 index 0000000..50835af --- /dev/null +++ b/src/agent_ultra/ultracode/examples/smoke.py @@ -0,0 +1,36 @@ +"""Smoke workflow: two parallel agents + a two-stage pipeline. Proves +fan-out, journaling, and the receipt end to end. Runs offline under --mock. + + agent-ultra ultracode run smoke --mock +""" + +META = { + "name": "smoke", + "description": "2 parallel pings + a 2-stage pipeline; end-to-end check", + "phases": ["Ping", "Chain"], +} + + +async def run(wf): + wf.phase("Ping") + replies = await wf.parallel([ + lambda: wf.agent("Reply with exactly: pong-one", label="ping-1"), + lambda: wf.agent("Reply with exactly: pong-two", label="ping-2"), + ]) + + wf.phase("Chain") + chained = await wf.pipeline( + ["alpha", "bravo"], + lambda word: wf.agent( + f"Return a JSON object with one key 'upper' whose value is " + f"'{word}' uppercased.", + schema={"type": "object", + "properties": {"upper": {"type": "string"}}, + "required": ["upper"]}, + label=f"upper:{word}"), + lambda parsed, word: {"word": word, "upper": (parsed or {}).get("upper")}, + ) + + return {"pings": replies, + "chained": [c for c in chained if c], + "calls_spent": wf.budget.calls} diff --git a/src/agent_ultra/ultracode/mock.py b/src/agent_ultra/ultracode/mock.py new file mode 100644 index 0000000..12af22f --- /dev/null +++ b/src/agent_ultra/ultracode/mock.py @@ -0,0 +1,107 @@ +"""Offline, deterministic, keyless mock route for ultracode. + +No network, no API key. The client answers by inspecting the prompt: + + * "Reply with exactly: X" -> X + * a schema request -> a synthesized schema-valid JSON value + * "uppercase " -> the word uppercased (as JSON if asked) + * otherwise -> a short deterministic echo + +This lets every bundled example and the demo run with zero setup, and makes +tests fully offline. It is intentionally simple — a stand-in that exercises +the fan-out/journal/resume/receipt machinery, not a real model. +""" + +from __future__ import annotations + +import json +import re +import threading + +from ..routes.pool import RoutePool +from ..routes.client import RouteError + +_SCHEMA_RE = re.compile(r"matching this schema[^\n]*\n(\{.*\})\s*$", re.DOTALL) +_EXACT_RE = re.compile(r"[Rr]eply with exactly:?\s*(.+)") +# match "uppercase X", "'X' uppercased", or "X uppercased" +_UPPER_RE = re.compile( + r"uppercas\w*\s+'?([A-Za-z0-9_-]+)'?|'?([A-Za-z0-9_-]+)'?\s+uppercas") + + +def _synthesize(schema): + """Build a minimal value that satisfies a (subset) JSON schema.""" + if not isinstance(schema, dict): + return "mock" + if "enum" in schema and schema["enum"]: + return schema["enum"][0] + typ = schema.get("type") + if isinstance(typ, list): + typ = typ[0] if typ else "string" + if typ == "object": + out = {} + props = schema.get("properties") or {} + for key in schema.get("required", list(props.keys())): + out[key] = _synthesize(props.get(key, {"type": "string"})) + return out + if typ == "array": + return [_synthesize(schema.get("items", {"type": "string"}))] + if typ == "integer": + return 1 + if typ == "number": + return 1.0 + if typ == "boolean": + return False + if typ == "null": + return None + return "mock" + + +class MockUltracodeClient: + def __init__(self, default: str = "ok", fail_routes=None): + self.default = default + self.fail_routes = set(fail_routes or ()) + self.calls: list[tuple[str, str]] = [] + self._lock = threading.Lock() + self._usage = {"prompt_tokens": 0, "completion_tokens": 0} + + def usage_snapshot(self) -> dict: + with self._lock: + return dict(self._usage) + + def complete(self, model: str, prompt: str, max_tokens: int) -> str: + with self._lock: + self.calls.append((model, prompt[:60])) + self._usage["prompt_tokens"] += max(1, len(prompt) // 4) + self._usage["completion_tokens"] += 8 + if model in self.fail_routes: + raise RouteError(f"{model}: mock route configured to fail") + m = _SCHEMA_RE.search(prompt) + if m: + try: + schema = json.loads(m.group(1)) + except ValueError: + schema = {} + up = _UPPER_RE.search(prompt) + val = _synthesize(schema) + # if the schema wants a single string and the prompt asked to + # uppercase a word, honor it so examples produce meaningful output + if up and isinstance(val, dict): + word = up.group(1) or up.group(2) + for k, v in val.items(): + if isinstance(v, str): + val[k] = word.upper() + break + return json.dumps(val) + m = _EXACT_RE.search(prompt) + if m: + return m.group(1).strip().splitlines()[0].strip() + m = _UPPER_RE.search(prompt) + if m: + return (m.group(1) or m.group(2)).upper() + return f"{self.default}: " + " ".join(prompt.split())[:40] + + +def demo_pool(routes=None, fail_routes=None) -> RoutePool: + """A RoutePool wired to the deterministic mock client.""" + client = MockUltracodeClient(fail_routes=fail_routes) + return RoutePool(list(routes or ["mock-a", "mock-b"]), client=client) diff --git a/src/agent_ultra/verifier/__init__.py b/src/agent_ultra/verifier/__init__.py new file mode 100644 index 0000000..2132c00 --- /dev/null +++ b/src/agent_ultra/verifier/__init__.py @@ -0,0 +1,44 @@ +"""verifier — refute-first claim verification. + +The verifier's job is to try to REFUTE a completion claim, not to confirm it. +It defaults to *refuted* whenever evidence is insufficient, so a claim only +survives if something concrete backs it: + + * **Engine re-check.** If the claim declares a verify command and a + ``run_check`` runner is injected, run it: exit 0 confirms, non-zero + refutes. No LLM is consulted — a real command is the strongest evidence. + + * **LLM refute.** With no declared command, an injected ``judge`` is asked, + with a refute-first prompt that defaults to refuted on insufficient + evidence. + + * **Neither channel => refuted** (the honest default). + +A verifier receipt is stamped with the claim's ``claim_sha256`` so it can only +ever resolve the claim it actually checked. A refuted verdict uses a +non-acceptable receipt verdict, so it can never satisfy a gate. + +Both the command runner and the judge are INJECTED — the kit runs no commands +and makes no network calls by default. +""" + +from __future__ import annotations + +from .verify import ( # noqa: F401 + verify_claim, + claim_sha256, + Verifier, + VerifierResult, + REFUTE_FIRST_PROMPT, +) +from .budget import EscalationBudget, BudgetExceeded # noqa: F401 + +__all__ = [ + "verify_claim", + "claim_sha256", + "Verifier", + "VerifierResult", + "REFUTE_FIRST_PROMPT", + "EscalationBudget", + "BudgetExceeded", +] diff --git a/src/agent_ultra/verifier/budget.py b/src/agent_ultra/verifier/budget.py new file mode 100644 index 0000000..6dd04f7 --- /dev/null +++ b/src/agent_ultra/verifier/budget.py @@ -0,0 +1,58 @@ +"""Escalation budget — bound how often the verifier may escalate. + +Verification (especially the LLM channel) costs something. The budget caps +escalations per session and per rolling window, and dedups on identical +``claim_sha256`` so re-asking the same claim is free after the first answer. +All limits are configurable with neutral defaults. +""" + +from __future__ import annotations + +import time +from collections import deque +from dataclasses import dataclass, field + + +class BudgetExceeded(RuntimeError): + """An escalation was requested past the configured cap.""" + + +@dataclass +class EscalationBudget: + per_session: int = 20 # max escalations for one session + per_window: int = 5 # max escalations inside window_seconds + window_seconds: int = 60 + + _session_counts: dict = field(default_factory=dict, init=False) + _window: deque = field(default_factory=deque, init=False) + _dedup: dict = field(default_factory=dict, init=False) # claim_sha -> result + + def cached(self, claim_sha: str): + """Return a previously computed result for this exact claim, or None.""" + return self._dedup.get(claim_sha) + + def remember(self, claim_sha: str, result) -> None: + self._dedup[claim_sha] = result + + def check(self, session_id: str, claim_sha: str) -> None: + """Raise BudgetExceeded if a fresh escalation would breach a cap. A + deduped (already-seen) claim never counts against the budget.""" + if claim_sha in self._dedup: + return + now = time.time() + while self._window and now - self._window[0] > self.window_seconds: + self._window.popleft() + if len(self._window) >= self.per_window: + raise BudgetExceeded( + f"per-window cap {self.per_window}/{self.window_seconds}s reached") + if self._session_counts.get(session_id, 0) >= self.per_session: + raise BudgetExceeded( + f"per-session cap {self.per_session} reached for {session_id!r}") + + def charge(self, session_id: str, claim_sha: str) -> None: + """Record a fresh escalation against the caps (skip if deduped).""" + if claim_sha in self._dedup: + return + self._window.append(time.time()) + self._session_counts[session_id] = \ + self._session_counts.get(session_id, 0) + 1 diff --git a/src/agent_ultra/verifier/verify.py b/src/agent_ultra/verifier/verify.py new file mode 100644 index 0000000..0e42176 --- /dev/null +++ b/src/agent_ultra/verifier/verify.py @@ -0,0 +1,162 @@ +"""verify_claim — the refute-first entry point, plus the budgeted Verifier. + +``verify_claim`` is a pure function over injected collaborators; ``Verifier`` +wraps it with an escalation budget and (optionally) a receipts bus to emit a +``kind=verifier`` receipt. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass, field +from typing import Callable + +from .budget import EscalationBudget, BudgetExceeded + +# The refute-first instruction handed to an injected judge. It is deliberately +# biased toward refutation: absence of evidence is treated as refutation. +REFUTE_FIRST_PROMPT = ( + "You are a REFUTING verifier. Your default answer is REFUTED. " + "Only return refuted=false if the provided evidence CONCLUSIVELY supports " + "the claim. If evidence is missing, ambiguous, or merely plausible, return " + "refuted=true. Respond with refuted (bool), confidence (0..1), and a short " + "reason.\n\nCLAIM:\n{claim}\n\nEVIDENCE:\n{evidence}\n" +) + + +def claim_sha256(claim_text: str) -> str: + """Stable hash of the claim text — the binding key for verifier receipts.""" + return hashlib.sha256((claim_text or "").encode("utf-8")).hexdigest() + + +# injected collaborator signatures: +# run_check(cmd, cwd) -> {"exit_code": int, "ledger_ref": str} +# judge(prompt) -> {"refuted": bool, "confidence": float, "reason": str} +# emit_receipt(env_fields: dict) -> receipt_id +RunCheck = Callable[[str, str], dict] +Judge = Callable[[str], dict] +EmitReceipt = Callable[[dict], str] + + +@dataclass +class VerifierResult: + refuted: bool + confidence: float + reason: str + checks_run: list = field(default_factory=list) + claim_sha256: str = "" + receipt_id: str = "" + + def as_dict(self) -> dict: + d = {"refuted": self.refuted, "confidence": self.confidence, + "reason": self.reason, "checks_run": self.checks_run, + "claim_sha256": self.claim_sha256} + if self.receipt_id: + d["receipt_id"] = self.receipt_id + return d + + +def verify_claim( + claim_text: str, + *, + workspace: str = "", + session_id: str = "", + refs: list | None = None, + declared_verify_cmd: str = "", + run_check: RunCheck | None = None, + judge: Judge | None = None, + emit_receipt: EmitReceipt | None = None, + actor: str = "reviewer", +) -> dict: + """Refute-first verification of ``claim_text``. + + Channels, in priority order: + 1. engine re-check — ``declared_verify_cmd`` + ``run_check``: exit 0 + confirms (refuted=False), non-zero refutes. No LLM. + 2. LLM refute — no command: ``judge`` with the refute-first prompt. + 3. neither — refuted (honest default). + + Emits a ``kind=verifier`` receipt (if ``emit_receipt`` given) stamped with + ``claim_sha256``; a REFUTED verdict maps to a non-acceptable receipt + verdict (``failed``) so it can never satisfy a gate. + """ + csha = claim_sha256(claim_text) + checks_run: list = [] + + if declared_verify_cmd and run_check is not None: + res = run_check(declared_verify_cmd, workspace) + exit_code = int(res.get("exit_code", 1)) + checks_run.append({"cmd": declared_verify_cmd, + "exit_code": exit_code, + "ledger_ref": res.get("ledger_ref", "")}) + refuted = exit_code != 0 + confidence = 0.99 if not refuted else 0.95 + reason = (f"engine re-check exit {exit_code}: " + + ("confirmed" if not refuted else "refuted")) + result = VerifierResult(refuted, confidence, reason, checks_run, csha) + + elif judge is not None: + evidence = "\n".join(str(r) for r in (refs or [])) or "(none provided)" + prompt = REFUTE_FIRST_PROMPT.format(claim=claim_text, evidence=evidence) + j = judge(prompt) or {} + # refute-first: default to refuted if the judge is silent/ambiguous + refuted = bool(j.get("refuted", True)) + confidence = float(j.get("confidence", 0.5)) + reason = j.get("reason", "judge did not justify; defaulting to refuted") + checks_run.append({"judge": True, "refuted": refuted}) + result = VerifierResult(refuted, confidence, reason, checks_run, csha) + + else: + result = VerifierResult( + True, 0.5, + "no declared command and no judge: insufficient evidence -> refuted", + checks_run, csha) + + if emit_receipt is not None: + verdict = "failed" if result.refuted else "completed" + fields = { + "kind": "verifier", + "actor": actor, + "verdict": verdict, + "workspace": workspace, + "session_id": session_id, + "claim_sha256": csha, + "canonical_command": declared_verify_cmd, + "evidence": [{"type": "check", "value": c} for c in checks_run], + } + result.receipt_id = emit_receipt(fields) or "" + + return result.as_dict() + + +@dataclass +class Verifier: + """Budgeted, dedup-aware wrapper over :func:`verify_claim`. + + ``escalate`` is the budgeted entry point: identical claims (same + ``claim_sha256``) are answered from cache and cost nothing; fresh claims + are charged against the per-session / per-window caps and raise + ``BudgetExceeded`` past a cap. + """ + + run_check: RunCheck | None = None + judge: Judge | None = None + emit_receipt: EmitReceipt | None = None + budget: EscalationBudget = field(default_factory=EscalationBudget) + actor: str = "reviewer" + + def escalate(self, claim_text: str, *, workspace: str = "", + session_id: str = "", refs: list | None = None, + declared_verify_cmd: str = "") -> dict: + csha = claim_sha256(claim_text) + cached = self.budget.cached(csha) + if cached is not None: + return cached + self.budget.check(session_id, csha) # raises BudgetExceeded past a cap + result = verify_claim( + claim_text, workspace=workspace, session_id=session_id, refs=refs, + declared_verify_cmd=declared_verify_cmd, run_check=self.run_check, + judge=self.judge, emit_receipt=self.emit_receipt, actor=self.actor) + self.budget.charge(session_id, csha) + self.budget.remember(csha, result) + return result diff --git a/tests/private_strings.txt b/tests/private_strings.txt new file mode 100644 index 0000000..1b0ff12 --- /dev/null +++ b/tests/private_strings.txt @@ -0,0 +1,26 @@ +# Base64-encoded private-string denylist for the leak gate. +# One base64 token per line; '#' lines and blanks are ignored. +# The gate decodes these at runtime so this file itself never contains +# the plaintext tokens (and so does not trip its own scan). +# To add a token: base64-encode it (UTF-8) and append a line. +YWxldGhlb24= +bXl0aG9z +U09VTA== +c2staGVybWVz +QzpcQUlcYnJhaW4= +V2F4aWxsaWFt +Z2xtLTUuMg== +bWltby12Mi41LXBybw== +ZGVlcHNlZWstdjQtcHJv +Z2VtaW5pLTMtZmxhc2g= +a2ltaQ== +bG9jYWwtZmFzdA== +dmlzaW9u +aGVybWVzLWFjdA== +aGVybWVzLXZlcmlmeQ== +aGVybWVzLXBsYW4= +bW5lbWU= +QmlnIEE= +QmlnIEQ= +QXBwRGF0YVxMb2NhbFxoZXJtZXM= +RG9jdW1lbnRzXEFJLVdvcmtmbG93 diff --git a/tests/test_leakgate.py b/tests/test_leakgate.py new file mode 100644 index 0000000..0be099f --- /dev/null +++ b/tests/test_leakgate.py @@ -0,0 +1,84 @@ +"""Leak gate — the tree must be clean, the gate must not trip on its own +denylist, and its matching rules must behave as specified. + +The denylist is base64-encoded in tests/private_strings.txt and decoded at +runtime, so neither the denylist file nor this test embeds any plaintext +private token. +""" + +from pathlib import Path + +from agent_ultra.leakgate import ( + scan_tree, scan_text, scan_message, load_denylist, DENYLIST_RELPATH, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] +DENYLIST = REPO_ROOT / DENYLIST_RELPATH + + +def test_whole_tree_is_clean(): + hits = scan_tree(REPO_ROOT) + assert hits == [], f"private strings found in the tree: {hits}" + + +def test_denylist_decodes_to_tokens(): + tokens = load_denylist(DENYLIST) + assert len(tokens) >= 20 # the mandated minimum set + # every token round-trips to non-empty plaintext + assert all(t for t in tokens) + + +def test_denylist_file_does_not_trip_itself(): + # The base64 denylist file is exempt and holds no plaintext, so scanning + # the tree (which includes it) yields no hits from it. + hits = [h for h in scan_tree(REPO_ROOT) if h[0] == DENYLIST_RELPATH] + assert hits == [] + + +# The case-sensitive whole-word token, reconstructed from base64 so this test +# file embeds no plaintext private string (and so stays clean under the gate). +import base64 as _b64 +_CS_WORD = _b64.b64decode("U09VTA==").decode("utf-8") # the all-caps collision + + +def test_case_insensitive_match(): + tokens = load_denylist(DENYLIST) + # pick a lowercase-safe token (not the case-sensitive whole-word one) + ci_tok = next(t for t in tokens if t != _CS_WORD and t.isascii()) + assert scan_text(ci_tok.upper(), tokens) # uppercased still matches + assert scan_text(ci_tok.lower(), tokens) + + +def test_soul_is_case_sensitive_whole_word(): + tokens = load_denylist(DENYLIST) + assert _CS_WORD in tokens + # exact all-caps whole word trips + assert any(tok == _CS_WORD + for _, tok in scan_text(f"the {_CS_WORD} of it", tokens)) + # lowercase prose does NOT trip (common English word) + assert not any(tok == _CS_WORD + for _, tok in scan_text(f"a gentle {_CS_WORD.lower()} by", + tokens)) + # substring inside another word does NOT trip + assert not any(tok == _CS_WORD + for _, tok in scan_text(f"{_CS_WORD}FUL music", tokens)) + + +def test_path_separator_normalised(): + tokens = load_denylist(DENYLIST) + # a windows-path token written with forward slashes still matches + win_tok = next((t for t in tokens if "\\" in t), None) + assert win_tok is not None + posix_form = win_tok.replace("\\", "/") + assert scan_text(posix_form, tokens) + + +def test_scan_message_catches_token(): + tokens = load_denylist(DENYLIST) + a_tok = next(t for t in tokens if t != _CS_WORD) + hits = scan_message(f"fix: touched {a_tok} config", DENYLIST) + assert any(tok == a_tok for _, tok in hits) + + +def test_clean_message_passes(): + assert scan_message("phase8: add receipts_bus and verifier", DENYLIST) == [] diff --git a/tests/test_receipts_bus.py b/tests/test_receipts_bus.py new file mode 100644 index 0000000..9413cbe --- /dev/null +++ b/tests/test_receipts_bus.py @@ -0,0 +1,256 @@ +"""receipts_bus — authenticity, tamper, binding, availability, gate audit. + +All offline: the store is SQLite on disk (tmp_path) and the HMAC key is a +tmp file. No network, no injected LLM here (the bus makes none). +""" + +import json +import sqlite3 + +import pytest + +from agent_ultra.receipts_bus import ( + ReceiptsBus, BusUnavailable, build_envelope, EnvelopeError, +) +from agent_ultra.receipts_bus.envelope import verify_hashes, receipt_sha256 + + +def _bus(tmp_path, **kw): + key = tmp_path / "install.key" + key.write_bytes(b"per-install-secret-key-0123456789") + return ReceiptsBus(db_path=tmp_path / "receipts.db", key_path=key, **kw) + + +# -- enums ----------------------------------------------------------------- + +def test_closed_enums_reject_unknown_members(): + with pytest.raises(EnvelopeError): + build_envelope(kind="not-a-kind", actor="broker", verdict="shipped") + with pytest.raises(EnvelopeError): + build_envelope(kind="panel", actor="nobody", verdict="shipped") + with pytest.raises(EnvelopeError): + build_envelope(kind="panel", actor="broker", verdict="maybe") + + +# -- authenticity vs integrity (the forgery case) -------------------------- + +def test_authentic_separate_from_integrity_forgery_rejected(tmp_path): + bus = _bus(tmp_path) + # A hand-authored envelope with a CORRECT sha256 but NO hmac. + forged = build_envelope(kind="panel", actor="engine_a", verdict="shipped") + forged["receipt_hmac"] = "" # strip authenticity + forged["receipt_sha256"] = receipt_sha256(forged) # but keep integrity valid + bus.put(forged) + res = bus.verify(forged["receipt_id"]) + assert res["integrity"] is True # sha256 checks out + assert res["authentic"] is False # but it is NOT authentic + assert res["ok"] is False # enforce-mode rejects it + + +def test_forged_hmac_rejected(tmp_path): + bus = _bus(tmp_path) + env = bus.append(kind="panel", actor="engine_a", verdict="shipped") + env["receipt_hmac"] = "deadbeef" * 8 # wrong key / made up + bus.put(env) + res = bus.verify(env["receipt_id"]) + assert res["authentic"] is False + + +def test_genuine_receipt_is_authentic(tmp_path): + bus = _bus(tmp_path) + env = bus.append(kind="panel", actor="engine_a", verdict="shipped") + res = bus.verify(env["receipt_id"]) + assert res["integrity"] and res["authentic"] and res["ok"] + + +# -- tamper detection ------------------------------------------------------ + +def test_tamper_breaks_integrity(tmp_path): + bus = _bus(tmp_path) + env = bus.append(kind="panel", actor="engine_a", verdict="hold") + env["verdict"] = "shipped" # flip the body, leave the old hashes + bus.put(env) + res = bus.verify(env["receipt_id"]) + assert res["integrity"] is False + assert res["ok"] is False + + +# -- BusUnavailable never empty -------------------------------------------- + +def test_corrupt_store_raises_not_empty(tmp_path): + bus = _bus(tmp_path) + bus.append(kind="panel", actor="engine_a", verdict="shipped") + # Corrupt the DB file so a read errors. + (tmp_path / "receipts.db").write_bytes(b"not a sqlite database at all") + with pytest.raises(BusUnavailable): + bus.list() + + +def test_missing_receipt_is_none_not_error(tmp_path): + bus = _bus(tmp_path) + assert bus.get("does-not-exist") is None + + +def test_concurrent_writers_both_resolve(tmp_path): + # Two bus instances over the same on-disk store both persist. + bus_a = _bus(tmp_path) + bus_b = ReceiptsBus(db_path=tmp_path / "receipts.db", + key_path=tmp_path / "install.key") + a = bus_a.append(kind="panel", actor="engine_a", verdict="shipped", + session_id="s1") + b = bus_b.append(kind="loop", actor="engine_b", verdict="completed", + session_id="s1") + ids = {r["receipt_id"] for r in bus_a.list(session_id="s1")} + assert a["receipt_id"] in ids and b["receipt_id"] in ids + + +# -- binding rule ---------------------------------------------------------- + +def test_cited_binds(tmp_path): + bus = _bus(tmp_path) + env = bus.append(kind="panel", actor="engine_a", verdict="shipped", + session_id="s1") + ctx = {"session_id": "s1", "cited": env["receipt_id"]} + cand = bus.resolve(ctx)[0] + ok, clause = bus.binds(ctx, cand) + assert ok and clause == "cited" + + +def test_command_never_satisfies_completion(tmp_path): + bus = _bus(tmp_path) + env = bus.append(kind="command", actor="broker", verdict="completed", + session_id="s1", canonical_command="pytest -q") + ctx = {"session_id": "s1", "verify_command": "pytest -q"} + from agent_ultra.receipts_bus import Candidate + ok, clause = bus.binds(ctx, Candidate(receipt=env)) + assert ok is False + assert clause == "command-not-completion" + + +def test_verifier_binds_only_on_claim_hash_match(tmp_path): + bus = _bus(tmp_path) + env = bus.append(kind="verifier", actor="reviewer", verdict="completed", + session_id="s1", claim_sha256="AAAA") + from agent_ultra.receipts_bus import Candidate + ok_match, cl_match = bus.binds( + {"session_id": "s1", "claim_sha256": "AAAA"}, Candidate(receipt=env)) + ok_other, cl_other = bus.binds( + {"session_id": "s1", "claim_sha256": "BBBB"}, Candidate(receipt=env)) + assert ok_match and cl_match == "verifier-claim-match" + assert not ok_other and cl_other == "verifier-claim-mismatch" + + +def test_verify_command_scope_and_freshness(tmp_path): + bus = _bus(tmp_path) + fresh = bus.append(kind="verification", actor="engine_a", verdict="completed", + session_id="s1", workspace="/w", + canonical_command="pytest -q") + from agent_ultra.receipts_bus import Candidate + ok, clause = bus.binds( + {"session_id": "s1", "workspace": "/w", "verify_command": "pytest -q"}, + Candidate(receipt=fresh)) + assert ok and clause == "verify-command" + # scope mismatch -> weak, not a pass + ok2, clause2 = bus.binds( + {"session_id": "s1", "workspace": "/other", "verify_command": "pytest -q"}, + Candidate(receipt=fresh)) + assert not ok2 and clause2 == "ledger-weak" + + +def test_stale_verify_command_is_ledger_weak(tmp_path): + bus = _bus(tmp_path, freshness_seconds=0) + env = bus.append(kind="verification", actor="engine_a", verdict="completed", + session_id="s1", canonical_command="pytest -q") + import time as _t + _t.sleep(1.1) + from agent_ultra.receipts_bus import Candidate + ok, clause = bus.binds( + {"session_id": "s1", "verify_command": "pytest -q"}, + Candidate(receipt=env)) + assert not ok and clause == "ledger-weak" + + +def test_task_ref_binds(tmp_path): + bus = _bus(tmp_path) + env = bus.append(kind="loop", actor="engine_a", verdict="completed", + session_id="s1", task_ref={"issue": "ABC-1"}) + from agent_ultra.receipts_bus import Candidate + ok, clause = bus.binds( + {"session_id": "s1", "task_ref": {"issue": "ABC-1"}}, + Candidate(receipt=env)) + assert ok and clause == "task-ref" + + +def test_manual_never_completes(tmp_path): + bus = _bus(tmp_path) + env = bus.append(kind="manual", actor="operator", verdict="approved", + session_id="s1", task_ref={"issue": "ABC-1"}) + from agent_ultra.receipts_bus import Candidate + ok, clause = bus.binds( + {"session_id": "s1", "task_ref": {"issue": "ABC-1"}}, + Candidate(receipt=env)) + assert ok is False + assert clause == "kind-cannot-complete" + + +def test_non_acceptable_verdict_never_binds(tmp_path): + bus = _bus(tmp_path) + env = bus.append(kind="verifier", actor="reviewer", verdict="failed", + session_id="s1", claim_sha256="AAAA") + from agent_ultra.receipts_bus import Candidate + ok, clause = bus.binds( + {"session_id": "s1", "claim_sha256": "AAAA"}, Candidate(receipt=env)) + assert ok is False and clause == "non-acceptable-verdict" + + +def test_unauthenticated_candidate_never_binds(tmp_path): + bus = _bus(tmp_path) + env = build_envelope(kind="panel", actor="engine_a", verdict="shipped", + session_id="s1") # no key -> no hmac + env["receipt_sha256"] = receipt_sha256(env) + bus.put(env) + from agent_ultra.receipts_bus import Candidate + ok, clause = bus.binds({"session_id": "s1", "cited": env["receipt_id"]}, + Candidate(receipt=env)) + assert ok is False and clause == "unauthenticated" + + +def test_federated_reader_annotates_origin(tmp_path): + def external(ctx): + env = build_envelope(kind="panel", actor="engine_a", verdict="shipped") + return [env] + bus = _bus(tmp_path) + bus.federated_readers["external"] = external + origins = {c.origin for c in bus.resolve({"session_id": "s1"})} + assert "external" in origins + + +# -- gate_audit hash chain ------------------------------------------------- + +def test_gate_audit_chain_verifies(tmp_path): + bus = _bus(tmp_path) + bus.append_audit("ship", "enforce", "allow", "engine_a", "tests green") + bus.append_audit("ship", "enforce", "block", "reviewer", "stale evidence") + out = bus.verify_audit("ship") + assert out["ok"] is True and out["first_broken"] is None + + +def test_gate_audit_tamper_detected(tmp_path): + bus = _bus(tmp_path) + e1 = bus.append_audit("ship", "enforce", "allow", "engine_a", "green") + bus.append_audit("ship", "enforce", "block", "reviewer", "stale") + # Tamper with the first row's reason directly in the DB. + conn = sqlite3.connect(str(tmp_path / "receipts.db")) + conn.execute("UPDATE gate_audit SET reason='forged' WHERE event_id=?", (e1,)) + conn.commit() + conn.close() + out = bus.verify_audit("ship") + assert out["ok"] is False + assert out["first_broken"] == e1 + + +def test_get_audit_event(tmp_path): + bus = _bus(tmp_path) + eid = bus.append_audit("ship", "enforce", "allow", "operator", "ok") + ev = bus.get_audit_event(eid) + assert ev["decision"] == "allow" and ev["gate"] == "ship" diff --git a/tests/test_ultracode.py b/tests/test_ultracode.py new file mode 100644 index 0000000..bd7e0c3 --- /dev/null +++ b/tests/test_ultracode.py @@ -0,0 +1,297 @@ +"""Offline tests for the ultracode engine — stdlib + pytest, no network. + +Everything runs against the deterministic mock route; the journal and receipt +are asserted as the source of truth, and the terminal card is asserted to be +ASCII-safe and crash-proof. +""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from agent_ultra.ultracode import ( + UltracodeEngine, demo_pool, MockUltracodeClient, + summarize_events, render_card, print_card, supports_rich, verify_receipt, + DEFAULTS, +) +from agent_ultra.ultracode.engine import _sha # noqa: for parity checks +from agent_ultra.routes.pool import RoutePool + + +SMOKE = ''' +META = {"name": "t-smoke", "description": "test", "phases": ["P"]} +async def run(wf): + wf.phase("P") + a, b = await wf.parallel([ + lambda: wf.agent("Reply with exactly: one", label="one"), + lambda: wf.agent("Reply with exactly: two", label="two"), + ]) + return {"a": a, "b": b} +''' + + +def write(tmp_path, body, name="script.py"): + p = tmp_path / name + p.write_text(body, encoding="utf-8") + return p + + +def make_engine(tmp_path, pool=None, **cfg): + return UltracodeEngine(pool or demo_pool(), config=cfg, + home=tmp_path / "home", + usage_source=(pool or demo_pool()).client_for("mock-a") + if pool is None else None) + + +def read_events(rep): + return [json.loads(l) for l in + Path(rep.events_path).read_text(encoding="utf-8").splitlines()] + + +# -------------------------------------------------------------------------- + +def test_smoke_completes_with_receipt_and_journal(tmp_path): + eng = UltracodeEngine(demo_pool(), home=tmp_path / "h") + rep = eng.run_script(write(tmp_path, SMOKE)) + assert rep.status == "completed" and rep.final_state == "COMPLETE" + assert rep.result == {"a": "one", "b": "two"} + assert rep.model_calls == 2 + journal = Path(rep.journal_path).read_text(encoding="utf-8").splitlines() + kinds = [json.loads(l)["type"] for l in journal] + assert kinds[0] == "run_start" and kinds[-1] == "run_end" + assert kinds.count("agent") == 2 + receipt = json.loads(Path(rep.receipt_path).read_text(encoding="utf-8")) + assert verify_receipt(receipt) and len(receipt["agents"]) == 2 + receipt["model_calls"] = 999 + assert not verify_receipt(receipt) + + +def test_events_match_reality_not_prose(tmp_path): + # a mock that CLAIMS success for many agents; the stream must show 2 + class Liar(MockUltracodeClient): + def complete(self, model, prompt, max_tokens): + super().complete(model, prompt, max_tokens) + return "All done! 8 agents completed successfully." + pool = RoutePool(["mock-a"], client=Liar()) + eng = UltracodeEngine(pool, home=tmp_path / "h") + rep = eng.run_script(write(tmp_path, SMOKE)) + names = [e["event"] for e in read_events(rep)] + assert names[0] == "ultracode_started" and names[-1] == "ultracode_completed" + assert names.count("agent_completed") == 2 # NOT 8 + card = summarize_events(rep.events_path) + assert card["counts"] == {"planned": 2, "deployed": 2, "completed": 2, + "failed": 0, "blocked": 0} + assert [d["state"] for d in card["dots"]] == ["completed", "completed"] + + +def test_schema_retry_then_valid(tmp_path): + class Flaky(MockUltracodeClient): + def __init__(self): + super().__init__() + self.n = 0 + def complete(self, model, prompt, max_tokens): + self.n += 1 + return "not json" if self.n == 1 else '{"k": "v"}' + pool = RoutePool(["mock-a"], client=Flaky()) + eng = UltracodeEngine(pool, home=tmp_path / "h") + script = ''' +META = {"name": "t-schema", "description": "t"} +async def run(wf): + return await wf.agent("x", schema={"type": "object", + "properties": {"k": {"type": "string"}}, "required": ["k"]}) +''' + rep = eng.run_script(write(tmp_path, script)) + assert rep.result == {"k": "v"} and rep.model_calls == 2 + + +def test_budget_hold_blocks_and_holds(tmp_path): + eng = UltracodeEngine(demo_pool(), home=tmp_path / "h", config={"max_calls": 1}) + script = ''' +META = {"name": "t-hold", "description": "t"} +async def run(wf): + await wf.agent("one") + await wf.agent("two") + return "unreachable" +''' + rep = eng.run_script(write(tmp_path, script)) + assert rep.status == "aborted_budget" and rep.final_state == "HOLD" + card = summarize_events(rep.events_path) + assert card["counts"]["completed"] == 1 and card["counts"]["blocked"] == 1 + + +def test_failed_agent_is_failed_dot(tmp_path): + pool = demo_pool(routes=["good", "bad"], fail_routes={"bad"}) + eng = UltracodeEngine(pool, home=tmp_path / "h") + script = ''' +META = {"name": "t-fail", "description": "t"} +async def run(wf): + ok = await wf.agent("hi", route="good", label="ok") + bad = await wf.agent("no", route="bad", label="bad") + return {"ok": ok, "bad": bad} +''' + rep = eng.run_script(write(tmp_path, script)) + card = summarize_events(rep.events_path) + assert card["counts"]["completed"] == 1 and card["counts"]["failed"] == 1 + assert sorted(d["state"] for d in card["dots"]) == ["completed", "failed"] + + +def test_pipeline_drops_failed_item(tmp_path): + eng = UltracodeEngine(demo_pool(), home=tmp_path / "h") + script = ''' +META = {"name": "t-pipe", "description": "t"} +async def run(wf): + def boom(prev, item): + if item == "b": + raise ValueError("nope") + return prev + return await wf.pipeline(["a", "b"], + lambda item: wf.agent("x " + item), boom) +''' + rep = eng.run_script(write(tmp_path, script)) + assert rep.status == "completed" + assert rep.result[1] is None and rep.result[0] is not None + + +def test_resume_replays_without_calls(tmp_path): + eng = UltracodeEngine(demo_pool(), home=tmp_path / "h") + path = write(tmp_path, SMOKE) + first = eng.run_script(path) + assert first.model_calls == 2 + + class Explode(MockUltracodeClient): + def complete(self, *a): + raise AssertionError("resume must not call the model") + eng2 = UltracodeEngine(RoutePool(["mock-a"], client=Explode()), + home=tmp_path / "h") + second = eng2.run_script(path, resume_run_id=first.run_id) + assert second.status == "completed" + assert second.model_calls == 0 and second.cached_hits == 2 + assert second.result == first.result + + +def test_resume_runs_only_new_calls(tmp_path): + a = ''' +META = {"name": "t-grow", "description": "t"} +async def run(wf): + return {"x": await wf.agent("stage one")} +''' + b = ''' +META = {"name": "t-grow", "description": "t"} +async def run(wf): + x = await wf.agent("stage one") + y = await wf.agent("stage two: " + (x or "")) + return {"x": x, "y": y} +''' + eng = UltracodeEngine(demo_pool(), home=tmp_path / "h") + first = eng.run_script(write(tmp_path, a, "grow.py")) + assert first.model_calls == 1 + eng2 = UltracodeEngine(demo_pool(), home=tmp_path / "h") + second = eng2.run_script(write(tmp_path, b, "grow.py"), + resume_run_id=first.run_id) + assert second.cached_hits == 1 and second.model_calls == 1 + + +def test_malformed_scripts_fail_loudly(tmp_path): + from agent_ultra.ultracode import WorkflowError + eng = UltracodeEngine(demo_pool(), home=tmp_path / "h") + with pytest.raises(WorkflowError, match="META"): + eng.run_script(write(tmp_path, "async def run(wf):\n return 1\n", "a.py")) + with pytest.raises(WorkflowError, match="async def run"): + eng.run_script(write(tmp_path, 'META={"name":"x"}\n', "b.py")) + with pytest.raises(WorkflowError, match="failed to import"): + eng.run_script(write(tmp_path, "def def def\n", "c.py")) + + +# -- terminal safety -------------------------------------------------------- + +class _TTY: + def __init__(self, tty=True, encoding="utf-8"): + self._tty, self.encoding = tty, encoding + def isatty(self): + return self._tty + + +def test_supports_rich_detection(): + good = {"TERM": "xterm-256color"} + assert supports_rich(_TTY(), good) is True + assert supports_rich(_TTY(tty=False), good) is False + assert supports_rich(_TTY(), {"TERM": "dumb"}) is False + assert supports_rich(_TTY(), {"TERM": "xterm", "CI": "1"}) is False + assert supports_rich(_TTY(), {"TERM": "xterm", "NO_COLOR": "1"}) is False + assert supports_rich(_TTY(), {"TERM": "xterm", "AGENT_ULTRA_PLAIN": "1"}) is False + assert supports_rich(_TTY(encoding="cp1252"), good) is False + + class Evil: + def isatty(self): + raise RuntimeError("boom") + assert supports_rich(Evil(), good) is False + + +def test_render_card_plain_is_ascii(tmp_path): + pool = demo_pool(routes=["good", "bad"], fail_routes={"bad"}) + eng = UltracodeEngine(pool, home=tmp_path / "h") + script = ''' +META = {"name": "t-card", "description": "t"} +async def run(wf): + await wf.agent("a", route="good", label="ok") + await wf.agent("b", route="bad", label="bad") +''' + rep = eng.run_script(write(tmp_path, script)) + card = render_card(summarize_events(rep.events_path), rich=False) + card.encode("ascii") # pure ASCII + lines = card.splitlines() + assert lines[0] == "ULTRACODE" + assert "Agents: 1/2 complete" in lines and "Failed: 1" in lines + bar = [l for l in lines if l.startswith("Progress: ")][0] + assert bar.startswith("Progress: [") and sorted(bar[11:-1]) == ["!", "#"] + + +def test_render_card_never_raises_on_garbage(): + for g in (None, 42, "text", [], {"counts": "x", "dots": 7}, + {"dots": [None, 3, {"state": object()}]}): + assert render_card(g, rich=False).startswith("ULTRACODE") + assert render_card(g, rich=True).startswith("ULTRACODE") + + +def test_print_card_survives_broken_stream(tmp_path): + eng = UltracodeEngine(demo_pool(), home=tmp_path / "h") + rep = eng.run_script(write(tmp_path, SMOKE)) + + class Broken: + encoding = "utf-8" + def isatty(self): + return True + def write(self, *a): + raise OSError("pipe closed") + def flush(self): + raise OSError("pipe closed") + print_card(summarize_events(rep.events_path), stream=Broken()) # no raise + + +# -- CLI end to end (subprocess, redirected -> plain ASCII) ----------------- + +def test_cli_run_and_resume(tmp_path): + import os + env = dict(os.environ) + env["AGENT_ULTRA_HOME"] = str(tmp_path / "clihome") + env["AGENT_ULTRA_PLAIN"] = "1" + run = subprocess.run( + [sys.executable, "-m", "agent_ultra.adapters.cli", + "ultracode", "run", "smoke", "--mock"], + capture_output=True, text=True, env=env, timeout=120) + assert run.returncode == 0, run.stderr + run.stdout.encode("ascii") # plain fallback engaged + assert "ULTRACODE" in run.stdout and "Status: COMPLETE" in run.stdout + run_id = [l.split()[-1] for l in run.stdout.splitlines() + if l.startswith("Resume:")][0] + + res = subprocess.run( + [sys.executable, "-m", "agent_ultra.adapters.cli", + "ultracode", "resume", run_id, "--mock"], + capture_output=True, text=True, env=env, timeout=120) + assert res.returncode == 0, res.stderr + assert "Status: COMPLETE" in res.stdout + assert '"calls_spent": 0' in res.stdout # resume proved cached calls diff --git a/tests/test_verifier.py b/tests/test_verifier.py new file mode 100644 index 0000000..50c1f20 --- /dev/null +++ b/tests/test_verifier.py @@ -0,0 +1,177 @@ +"""verifier — refute-first, engine re-check, judge path, budget, binding. + +Offline: run_check and judge are injected fakes; no network, no host commands. +""" + +import pytest + +from agent_ultra.verifier import ( + verify_claim, Verifier, EscalationBudget, BudgetExceeded, claim_sha256, +) +from agent_ultra.verifier.verify import claim_sha256 as csha + + +# -- injected fakes -------------------------------------------------------- + +def make_run_check(exit_code): + calls = [] + + def run_check(cmd, cwd): + calls.append((cmd, cwd)) + return {"exit_code": exit_code, "ledger_ref": "ledger://1"} + run_check.calls = calls + return run_check + + +class FakeJudge: + """Deterministic judge: returns whatever verdict it was constructed with.""" + def __init__(self, refuted, confidence=0.8, reason="fake"): + self.payload = {"refuted": refuted, "confidence": confidence, + "reason": reason} + self.prompts = [] + + def __call__(self, prompt): + self.prompts.append(prompt) + return dict(self.payload) + + +# -- engine re-check (no LLM) ---------------------------------------------- + +def test_engine_recheck_pass_confirms(): + rc = make_run_check(0) + out = verify_claim("tests pass", declared_verify_cmd="pytest -q", + run_check=rc) + assert out["refuted"] is False + assert out["checks_run"][0]["exit_code"] == 0 + assert len(rc.calls) == 1 + + +def test_engine_recheck_fail_refutes(): + rc = make_run_check(1) + out = verify_claim("tests pass", declared_verify_cmd="pytest -q", + run_check=rc) + assert out["refuted"] is True + + +def test_engine_recheck_takes_priority_over_judge(): + rc = make_run_check(0) + judge = FakeJudge(refuted=True) + out = verify_claim("tests pass", declared_verify_cmd="pytest -q", + run_check=rc, judge=judge) + assert out["refuted"] is False # command won + assert judge.prompts == [] # judge never consulted + + +# -- judge path ------------------------------------------------------------ + +def test_judge_can_refute(): + judge = FakeJudge(refuted=True, reason="no evidence") + out = verify_claim("it works", judge=judge, refs=["log line"]) + assert out["refuted"] is True + assert "REFUTED" in judge.prompts[0] # refute-first prompt used + + +def test_judge_can_confirm(): + judge = FakeJudge(refuted=False, confidence=0.95) + out = verify_claim("it works", judge=judge, refs=["proof"]) + assert out["refuted"] is False + + +def test_judge_silence_defaults_refuted(): + def silent_judge(prompt): + return {} # no verdict + out = verify_claim("it works", judge=silent_judge) + assert out["refuted"] is True + + +# -- neither channel ------------------------------------------------------- + +def test_no_channel_defaults_refuted(): + out = verify_claim("it works") + assert out["refuted"] is True + assert "insufficient evidence" in out["reason"] + + +# -- receipt emission + claim binding -------------------------------------- + +def test_refuted_receipt_uses_non_acceptable_verdict(tmp_path): + from agent_ultra.receipts_bus import ReceiptsBus, Candidate + key = tmp_path / "k"; key.write_bytes(b"k" * 32) + bus = ReceiptsBus(db_path=tmp_path / "r.db", key_path=key) + + def emit(fields): + return bus.append(**fields)["receipt_id"] + + out = verify_claim("bogus claim", judge=FakeJudge(refuted=True), + emit_receipt=emit, session_id="s1") + rid = out["receipt_id"] + receipt = bus.get(rid) + assert receipt["verdict"] == "failed" # non-acceptable + # and it cannot satisfy a gate for its own claim + ok, clause = bus.binds( + {"session_id": "s1", "claim_sha256": out["claim_sha256"]}, + Candidate(receipt=receipt)) + assert ok is False and clause == "non-acceptable-verdict" + + +def test_verifier_receipt_binds_only_its_own_claim(tmp_path): + from agent_ultra.receipts_bus import ReceiptsBus, Candidate + key = tmp_path / "k"; key.write_bytes(b"k" * 32) + bus = ReceiptsBus(db_path=tmp_path / "r.db", key_path=key) + + def emit(fields): + return bus.append(**fields)["receipt_id"] + + # confirm claim A + out = verify_claim("claim A", declared_verify_cmd="pytest -q", + run_check=make_run_check(0), emit_receipt=emit, + session_id="s1") + receipt = bus.get(out["receipt_id"]) + # binds A + okA, _ = bus.binds({"session_id": "s1", "claim_sha256": csha("claim A")}, + Candidate(receipt=receipt)) + # never binds B + okB, clauseB = bus.binds( + {"session_id": "s1", "claim_sha256": csha("claim B")}, + Candidate(receipt=receipt)) + assert okA is True + assert okB is False and clauseB == "verifier-claim-mismatch" + + +# -- budget: dedup + caps -------------------------------------------------- + +def test_budget_dedup_same_claim_is_free(): + calls = {"n": 0} + + def judge(prompt): + calls["n"] += 1 + return {"refuted": True, "confidence": 0.5, "reason": "x"} + + v = Verifier(judge=judge, budget=EscalationBudget(per_window=1, per_session=1)) + a = v.escalate("same claim", session_id="s1") + b = v.escalate("same claim", session_id="s1") # deduped, no charge + assert a == b + assert calls["n"] == 1 # judge only called once + + +def test_budget_per_window_cap_raises(): + v = Verifier(judge=FakeJudge(refuted=True), + budget=EscalationBudget(per_window=2, window_seconds=1000, + per_session=100)) + v.escalate("claim 1", session_id="s1") + v.escalate("claim 2", session_id="s1") + with pytest.raises(BudgetExceeded): + v.escalate("claim 3", session_id="s1") + + +def test_budget_per_session_cap_raises(): + v = Verifier(judge=FakeJudge(refuted=True), + budget=EscalationBudget(per_window=100, per_session=1)) + v.escalate("claim 1", session_id="s1") + with pytest.raises(BudgetExceeded): + v.escalate("claim 2", session_id="s1") + + +def test_claim_sha256_stable(): + assert claim_sha256("x") == claim_sha256("x") + assert claim_sha256("x") != claim_sha256("y") diff --git a/tests/test_workers.py b/tests/test_workers.py index 02ac9e8..cfb2c16 100644 --- a/tests/test_workers.py +++ b/tests/test_workers.py @@ -143,15 +143,14 @@ def tests_fail(ws, cmd, timeout): assert "reverted" in fx.results[0].error -# 11. No private Aletheon/FABLE.5 paths in the worker layer. +# 11. No private host identifiers in the worker layer. The denylist is the +# shared base64 file, decoded at runtime, so this test embeds no plaintext. def test_no_private_leakage_in_workers(): - root = Path(__file__).resolve().parents[1] / "src" / "agent_ultra" / "workers" - banned = ["aletheon", "waxilliam", "mneme", "mythos-", "sk-hermes", - "hermes_home", "AppData", "fable_", "/opt/data"] - for p in root.glob("*.py"): - text = p.read_text(encoding="utf-8").lower() - for b in banned: - assert b.lower() not in text, f"{b!r} leaked into {p.name}" + from agent_ultra.leakgate import scan_tree + root = Path(__file__).resolve().parents[1] + hits = [(path, line, tok) for (path, line, tok) in scan_tree(root) + if path.startswith("src/agent_ultra/workers/")] + assert hits == [], f"private identifiers leaked into the worker layer: {hits}" def test_auto_policy_is_documented():