diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..e4b9311 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,30 @@ +name: test + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: test-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + pytest: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + - name: Install development dependencies + run: python -m pip install --upgrade pip && python -m pip install -e '.[dev]' + - name: Compile Python sources + run: python -m py_compile bridge_core/*.py scripts/*.py + - name: Run tests + run: python -m pytest -q diff --git a/README.md b/README.md index 3c3791b..944ccf8 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,10 @@ It gives you a bounded place to create, receive, track, and close agent-to-agent ## What it does - creates explicit handoff records between agents -- stores sender and recipient copies on disk for auditability +- stores an authoritative canonical record plus recoverable sender and recipient queue views - enforces route policy and status transitions - exposes a local HTTP API for wrappers and automation -- supports immediate lifecycle notifications (`create`, `close`, `block`) +- supports durable at-least-once lifecycle events plus immediate compatibility notifications - supports read-only event consumers for surfacing lifecycle events into chat/UI adapters - includes patrol tooling for reminders and unresolved-handoff detection @@ -28,7 +28,7 @@ Route policy is optional: You do **not** need to edit `bridge_core/models.py` or `bridge_core/policy.py` to adopt your own ecosystem. -For deeper docs, start with `docs/README.md`. +For deeper docs, start with `docs/README.md` and `docs/bridge-v2-reliability.md`. ## Repository layout @@ -76,7 +76,8 @@ python3 scripts/bridge_cli.py create \ --issue-type task \ --subject "Demo handoff" \ --requested-action "Inspect the repo and report back" \ - --minimal-context "Started from the README quickstart" + --minimal-context "Started from the README quickstart" \ + --idempotency-key "demo-create-1" ``` You should get JSON like: @@ -187,9 +188,23 @@ python3 scripts/bridge_agent.py --agent agent-b close HND-... --outcome "Verifie - `POST /v1/handoffs/{handoff_id}/ack` - `POST /v1/handoffs/{handoff_id}/block` - `POST /v1/handoffs/{handoff_id}/close` +- `POST /v1/handoffs/{handoff_id}/cancel` - `POST /v1/handoffs/{handoff_id}/status` - `POST /v1/handoffs/{handoff_id}/archive` +Create accepts `Idempotency-Key` or `idempotency_key`. Lifecycle writes accept `expected_revision` or an `If-Match` header and return HTTP 409 for stale revisions. + +Durable outbox and reconciliation commands: + +```bash +python3 scripts/bridge_cli.py outbox-list --state pending +python3 scripts/bridge_cli.py outbox-deliver --limit 100 +python3 scripts/bridge_cli.py outbox-replay EVT-... +python3 scripts/bridge_cli.py outbox-recover +python3 scripts/bridge_cli.py outbox-resolve-transaction EVT-... --action enqueue --reason "verified missing delivery" +python3 scripts/bridge_cli.py reconcile --check-only +``` + ## Lifecycle notifications Bridge supports two layers of notification behavior: diff --git a/bridge_core/events.py b/bridge_core/events.py new file mode 100644 index 0000000..0a375bb --- /dev/null +++ b/bridge_core/events.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from typing import Any +from uuid import uuid4 + +from .models import CURRENT_SCHEMA_VERSION, HandoffRecord, now_iso + +DELIVERY_STATES = {"pending", "retrying", "delivered", "dead_letter"} + + +@dataclass(slots=True) +class LifecycleEvent: + event_id: str + handoff_id: str + event_type: str + revision: int + occurred_at: str + target_agent: str + sender: str + recipient: str + actor: str + status: str + subject: str + resolution_summary: str + correlation_id: str | None = None + causation_id: str | None = None + schema_version: int = CURRENT_SCHEMA_VERSION + attempts: int = 0 + next_attempt_at: str = field(default_factory=now_iso) + last_error: str = "" + delivery_state: str = "pending" + external_correlation: dict[str, str] = field(default_factory=dict) + + @classmethod + def from_handoff( + cls, + handoff: HandoffRecord, + *, + event_type: str, + actor: str, + target_agent: str, + event_id: str | None = None, + ) -> "LifecycleEvent": + return cls( + event_id=event_id or f"EVT-{uuid4().hex}", + handoff_id=handoff.handoff_id, + event_type=event_type, + revision=handoff.revision, + occurred_at=now_iso(), + target_agent=target_agent, + sender=handoff.sender, + recipient=handoff.recipient, + actor=actor, + status=handoff.status, + subject=handoff.subject, + resolution_summary=handoff.resolution_summary, + correlation_id=handoff.correlation_id, + causation_id=handoff.causation_id, + ) + + @classmethod + def from_mapping(cls, mapping: dict[str, Any]) -> "LifecycleEvent": + allowed = {field_name for field_name in cls.__dataclass_fields__} + values = {key: value for key, value in mapping.items() if key in allowed} + return cls(**values) + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + def due(self, now: datetime | None = None) -> bool: + current = now or datetime.now(timezone.utc) + due_at = datetime.fromisoformat(self.next_attempt_at.replace("Z", "+00:00")) + return self.delivery_state in {"pending", "retrying"} and due_at <= current \ No newline at end of file diff --git a/bridge_core/file_repository.py b/bridge_core/file_repository.py index 6f92ea1..7192837 100644 --- a/bridge_core/file_repository.py +++ b/bridge_core/file_repository.py @@ -1,114 +1,575 @@ from __future__ import annotations +from contextlib import contextmanager from pathlib import Path -import shutil +from typing import Callable, Iterator +import fcntl +import hashlib +import json +import os +import re +import tempfile +import threading +from .events import LifecycleEvent from .frontmatter import dump_document, parse_frontmatter -from .models import HandoffRecord, StoredHandoff, now_iso -from .repository import BridgeRepository, HandoffNotFoundError +from .models import CURRENT_SCHEMA_VERSION, HandoffRecord, StoredHandoff, now_iso +from .repository import ( + BridgeRepository, + HandoffNotFoundError, + IdempotencyConflictError, + RepositoryConflictError, +) class FileBridgeRepository(BridgeRepository): - def __init__(self, bridge_root: Path | str): + """Crash-safe canonical records with backward-compatible queue views.""" + + def __init__( + self, + bridge_root: Path | str, + *, + fault_injector: Callable[[str], None] | None = None, + initialize: bool = True, + ): self.bridge_root = Path(bridge_root) self.audit_file = self.bridge_root / "audit" / "handoff-log.md" - self._ensure_layout() + self.records_root = self.bridge_root / "records" + self.locks_root = self.bridge_root / ".locks" + self.idempotency_root = self.bridge_root / "idempotency" + self.events_root = self.bridge_root / "outbox" / "events" + self.transactions_root = self.bridge_root / "transactions" + self.resolved_transactions_root = self.bridge_root / "audit" / "resolved-transactions" + self.conflicts_root = self.bridge_root / "audit" / "conflicts" + self._fault_injector = fault_injector + self._thread_locks: dict[str, threading.RLock] = {} + self._thread_locks_guard = threading.Lock() + self._local = threading.local() + if initialize: + self._ensure_layout() def _ensure_layout(self) -> None: for kind in ("incoming", "outgoing"): (self.bridge_root / kind).mkdir(parents=True, exist_ok=True) - (self.bridge_root / "archive").mkdir(parents=True, exist_ok=True) - self.audit_file.parent.mkdir(parents=True, exist_ok=True) + for path in ( + self.bridge_root / "archive", + self.audit_file.parent, + self.records_root, + self.locks_root, + self.idempotency_root, + self.events_root, + self.transactions_root, + self.resolved_transactions_root, + self.conflicts_root, + ): + path.mkdir(parents=True, exist_ok=True) + path.chmod(0o700) if not self.audit_file.exists(): - self.audit_file.write_text("", encoding="utf-8") + self._atomic_write_bytes(self.audit_file, b"") def _queue_path(self, kind: str, agent: str, handoff_id: str) -> Path: return self.bridge_root / kind / agent / f"{handoff_id}.md" - def _write_record(self, path: Path, handoff: HandoffRecord) -> None: + def _canonical_path(self, handoff_id: str) -> Path: + return self.records_root / f"{handoff_id}.md" + + def _record_bytes(self, handoff: HandoffRecord) -> bytes: + return dump_document(handoff.to_frontmatter(), handoff.body).encode("utf-8") + + def _fsync_dir(self, path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + def _atomic_write_bytes(self, path: Path, content: bytes, *, mode: int = 0o600) -> None: path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(dump_document(handoff.to_frontmatter(), handoff.body), encoding="utf-8") - path.chmod(0o600) - - def create(self, handoff: HandoffRecord) -> HandoffRecord: - outgoing = self._queue_path("outgoing", handoff.sender, handoff.handoff_id) - incoming = self._queue_path("incoming", handoff.recipient, handoff.handoff_id) - self._write_record(outgoing, handoff) - incoming.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(outgoing, incoming) - incoming.chmod(0o600) - self.append_audit(handoff, handoff.status) - return handoff + path.parent.chmod(0o700) + descriptor, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) + temp_path = Path(temp_name) + try: + os.fchmod(descriptor, mode) + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temp_path, path) + path.chmod(mode) + self._fsync_dir(path.parent) + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + temp_path.unlink(missing_ok=True) + raise + + def _atomic_write_json(self, path: Path, payload: dict[str, object]) -> None: + self._atomic_write_bytes(path, (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode("utf-8")) + + def _write_record(self, path: Path, handoff: HandoffRecord) -> None: + self._atomic_write_bytes(path, self._record_bytes(handoff)) + + def _fault(self, stage: str) -> None: + if self._fault_injector is not None: + self._fault_injector(stage) + + def _lock_for(self, key: str) -> threading.RLock: + with self._thread_locks_guard: + return self._thread_locks.setdefault(key, threading.RLock()) + + @contextmanager + def _file_lock(self, key: str) -> Iterator[None]: + safe_key = hashlib.sha256(key.encode("utf-8")).hexdigest() + thread_lock = self._lock_for(safe_key) + with thread_lock: + depths = getattr(self._local, "depths", {}) + depth = depths.get(safe_key, 0) + depths[safe_key] = depth + 1 + self._local.depths = depths + lock_handle = None + try: + if depth == 0: + if self.locks_root.exists(): + lock_path = self.locks_root / f"{safe_key}.lock" + lock_handle = lock_path.open("a+b") + lock_path.chmod(0o600) + fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) + yield + finally: + depths[safe_key] -= 1 + if depths[safe_key] == 0: + depths.pop(safe_key, None) + if lock_handle is not None: + fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN) + lock_handle.close() + + def handoff_lock(self, handoff_id: str): + return self._file_lock(f"handoff:{handoff_id}") + + def _idempotency_index_path(self, sender: str, key_hash: str) -> Path: + return self.idempotency_root / sender / f"{key_hash}.json" + + def _write_idempotency_index(self, handoff: HandoffRecord, key_hash: str) -> None: + self._atomic_write_json( + self._idempotency_index_path(handoff.sender, key_hash), + { + "schema_version": CURRENT_SCHEMA_VERSION, + "sender": handoff.sender, + "idempotency_key_hash": key_hash, + "request_hash": handoff.canonical_request_hash, + "handoff_id": handoff.handoff_id, + "created_at": handoff.created_at, + }, + ) + + def _recover_idempotency_index( + self, + *, + sender: str, + key_hash: str, + request_hash: str, + ) -> HandoffRecord | None: + matches: list[HandoffRecord] = [] + for path in sorted(self.records_root.glob("*.md")): + if key_hash not in path.read_text(encoding="utf-8", errors="replace"): + continue + record = self._load_path(path).record + if record.sender == sender and record.idempotency_key_hash == key_hash: + matches.append(record) + if len(matches) > 1: + raise RepositoryConflictError( + f"multiple canonical handoffs claim idempotency key for {sender}: {key_hash}" + ) + if not matches: + return None + recovered = matches[0] + if recovered.canonical_request_hash != request_hash: + raise IdempotencyConflictError( + f"idempotency key already used by {sender} with a different request payload" + ) + self._write_idempotency_index(recovered, key_hash) + self.reconcile_handoff(recovered.handoff_id, repair=True) + return self.load_records(recovered.handoff_id)[0].record + + def create(self, handoff: HandoffRecord, *, idempotency_key: str | None = None) -> HandoffRecord: + key_hash = hashlib.sha256(idempotency_key.encode("utf-8")).hexdigest() if idempotency_key else "" + create_lock = f"idempotency:{handoff.sender}:{key_hash}" if key_hash else f"create:{handoff.handoff_id}" + with self._file_lock(create_lock): + if key_hash: + index_path = self._idempotency_index_path(handoff.sender, key_hash) + if index_path.exists(): + index = json.loads(index_path.read_text(encoding="utf-8")) + if index.get("request_hash") != handoff.canonical_request_hash: + raise IdempotencyConflictError( + f"idempotency key already used by {handoff.sender} with a different request payload" + ) + original_id = str(index["handoff_id"]) + self.reconcile_handoff(original_id) + return self.load_records(original_id)[0].record + recovered = self._recover_idempotency_index( + sender=handoff.sender, + key_hash=key_hash, + request_hash=handoff.canonical_request_hash, + ) + if recovered is not None: + return recovered + handoff.idempotency_key_hash = key_hash + + with self.handoff_lock(handoff.handoff_id): + canonical = self._canonical_path(handoff.handoff_id) + if canonical.exists(): + raise RepositoryConflictError(f"handoff already exists: {handoff.handoff_id}") + self._write_record(canonical, handoff) + if key_hash: + self._write_idempotency_index(handoff, key_hash) + outgoing = self._queue_path("outgoing", handoff.sender, handoff.handoff_id) + incoming = self._queue_path("incoming", handoff.recipient, handoff.handoff_id) + self._write_record(outgoing, handoff) + self._fault("after_outgoing_write") + self._write_record(incoming, handoff) + self._fault("after_incoming_write") + self.append_audit(handoff, handoff.status) + return handoff + + def _load_path(self, path: Path) -> StoredHandoff: + data, body = parse_frontmatter(path.read_text(encoding="utf-8", errors="replace")) + return StoredHandoff(path=path, record=HandoffRecord.from_mapping(data, body)) def load_records(self, handoff_id: str) -> list[StoredHandoff]: - def sort_key(path: Path) -> tuple[int, str]: - parts = path.parts - if "outgoing" in parts: - rank = 0 - elif "incoming" in parts: - rank = 1 - elif "archive" in parts: - rank = 2 - else: - rank = 3 - return rank, str(path) - - matches = sorted(self.bridge_root.glob(f"**/{handoff_id}.md"), key=sort_key) + canonical = self._canonical_path(handoff_id) + matches: list[Path] = [] + if canonical.exists(): + matches.append(canonical) + queue_matches = sorted(self.bridge_root.glob(f"incoming/*/{handoff_id}.md")) + queue_matches += sorted(self.bridge_root.glob(f"outgoing/*/{handoff_id}.md")) + archive_matches = sorted((self.bridge_root / "archive" / handoff_id).glob("*.md")) + matches.extend(queue_matches) + matches.extend(archive_matches) if not matches: raise HandoffNotFoundError("handoff not found") - records: list[StoredHandoff] = [] - for path in matches: - data, body = parse_frontmatter(path.read_text(encoding="utf-8", errors="replace")) - records.append(StoredHandoff(path=path, record=HandoffRecord.from_mapping(data, body))) - return records + return [self._load_path(path) for path in matches] def list_incoming(self, agent: str) -> list[HandoffRecord]: - records: list[HandoffRecord] = [] - for path in sorted((self.bridge_root / "incoming" / agent).glob("*.md")): - data, body = parse_frontmatter(path.read_text(encoding="utf-8", errors="replace")) - records.append(HandoffRecord.from_mapping(data, body)) - return records + return [self._load_path(path).record for path in sorted((self.bridge_root / "incoming" / agent).glob("*.md"))] def save_records(self, records: list[StoredHandoff]) -> None: - for stored in records: - self._write_record(stored.path, stored.record) + if not records: + return + handoff_id = records[0].record.handoff_id + with self.handoff_lock(handoff_id): + canonical_record = next( + (item.record for item in records if item.path == self._canonical_path(handoff_id)), + records[0].record, + ) + canonical_record.schema_version = CURRENT_SCHEMA_VERSION + self._write_record(self._canonical_path(handoff_id), canonical_record) + for stored in records: + if stored.path == self._canonical_path(handoff_id) or "archive" in stored.path.parts: + continue + stored.record.schema_version = CURRENT_SCHEMA_VERSION + self._write_record(stored.path, stored.record) def archive_records(self, records: list[StoredHandoff]) -> Path: handoff_id = records[0].record.handoff_id - archive_dir = self.bridge_root / "archive" / handoff_id - archive_dir.mkdir(parents=True, exist_ok=True) - archive_dir.chmod(0o700) - canonical_path = archive_dir / f"{handoff_id}.md" - for stored in records: - path = stored.path - record_text = dump_document(stored.record.to_frontmatter(), stored.record.body) - if not canonical_path.exists(): - canonical_path.write_text(record_text, encoding="utf-8") - canonical_path.chmod(0o600) - if path.exists(): - path.unlink() - continue - canonical_text = canonical_path.read_text(encoding="utf-8") - if canonical_text == record_text: - if path.exists(): - path.unlink() - continue - queue_label = path.parent.parent.name - suffix = stored.record.sender if queue_label == "outgoing" else stored.record.recipient - alternate_path = archive_dir / f"{handoff_id}.{queue_label}.{suffix}.md" - alternate_path.write_text(record_text, encoding="utf-8") - alternate_path.chmod(0o600) - if path.exists(): - path.unlink() - return archive_dir + with self.handoff_lock(handoff_id): + archive_dir = self.bridge_root / "archive" / handoff_id + archive_dir.mkdir(parents=True, exist_ok=True) + archive_dir.chmod(0o700) + canonical_record = next( + (item.record for item in records if item.path == self._canonical_path(handoff_id)), + records[0].record, + ) + canonical_path = archive_dir / f"{handoff_id}.md" + canonical_bytes = self._record_bytes(canonical_record) + self._atomic_write_bytes(canonical_path, canonical_bytes) + for stored in records: + path = stored.path + if path == self._canonical_path(handoff_id) or "archive" in path.parts: + continue + record_bytes = self._record_bytes(stored.record) + if record_bytes != canonical_bytes: + queue_label = path.parent.parent.name + suffix = stored.record.sender if queue_label == "outgoing" else stored.record.recipient + alternate = archive_dir / f"{handoff_id}.{queue_label}.{suffix}.md" + self._atomic_write_bytes(alternate, record_bytes) + path.unlink(missing_ok=True) + self._fsync_dir(path.parent) + self._canonical_path(handoff_id).unlink(missing_ok=True) + self._fsync_dir(self.records_root) + return archive_dir def append_audit(self, handoff: HandoffRecord, status: str) -> None: metadata = "" if status == "acknowledged" and handoff.acknowledged_at != "none": metadata = f" | ack_source={handoff.acknowledgment_source} acknowledged_at={handoff.acknowledged_at}" + metadata += f" | schema={handoff.schema_version} revision={handoff.revision}" line = ( f"- {now_iso()} | {handoff.handoff_id} | {handoff.sender} -> {handoff.recipient} | " f"{status} | {handoff.subject}{metadata}\n" - ) - with self.audit_file.open("a", encoding="utf-8") as handle: - handle.write(line) + ).encode("utf-8") + with self._file_lock("audit"): + descriptor = os.open(self.audit_file, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) + try: + os.write(descriptor, line) + os.fsync(descriptor) + finally: + os.close(descriptor) + + def append_diagnostic_audit(self, handoff_id: str, event: str, detail: str) -> None: + line = f"- {now_iso()} | {handoff_id} | system | {event} | {detail}\n".encode("utf-8") + with self._file_lock("audit"): + descriptor = os.open(self.audit_file, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) + try: + os.write(descriptor, line) + os.fsync(descriptor) + finally: + os.close(descriptor) + + @staticmethod + def _safe_event_id(event_id: str) -> str: + if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,199}", event_id) is None: + raise ValueError("invalid event_id") + return event_id + + def _event_path(self, event_id: str) -> Path: + return self.events_root / f"{self._safe_event_id(event_id)}.json" + + def _transaction_path(self, event_id: str) -> Path: + return self.transactions_root / f"{self._safe_event_id(event_id)}.json" + + def _resolved_transaction_path(self, event_id: str) -> Path: + return self.resolved_transactions_root / f"{self._safe_event_id(event_id)}.json" + + def prepare_event(self, event: LifecycleEvent) -> None: + payload = {"phase": "prepared", "event": event.to_dict(), "prepared_at": now_iso()} + self._atomic_write_json(self._transaction_path(event.event_id), payload) + + def complete_event(self, event_id: str) -> None: + path = self._transaction_path(event_id) + path.unlink(missing_ok=True) + self._fsync_dir(self.transactions_root) + + def enqueue_event(self, event: LifecycleEvent) -> LifecycleEvent: + with self._file_lock(f"event:{event.event_id}"): + path = self._event_path(event.event_id) + if path.exists(): + return LifecycleEvent.from_mapping(json.loads(path.read_text(encoding="utf-8"))) + self._atomic_write_json(path, event.to_dict()) + return event + + def list_events(self, *, states: set[str] | None = None) -> list[LifecycleEvent]: + events = [] + for path in sorted(self.events_root.glob("*.json")): + event = LifecycleEvent.from_mapping(json.loads(path.read_text(encoding="utf-8"))) + if states is None or event.delivery_state in states: + events.append(event) + return events + + def update_event(self, event: LifecycleEvent) -> LifecycleEvent: + with self._file_lock(f"event:{event.event_id}"): + path = self._event_path(event.event_id) + if path.exists(): + current = LifecycleEvent.from_mapping(json.loads(path.read_text(encoding="utf-8"))) + if current.delivery_state == "delivered" and event.delivery_state != "delivered": + return current + if current.delivery_state == "dead_letter" and event.delivery_state in {"pending", "retrying"}: + return current + self._atomic_write_json(path, event.to_dict()) + return event + + def replay_event(self, event_id: str) -> LifecycleEvent: + with self._file_lock(f"event:{event_id}"): + path = self._event_path(event_id) + if not path.exists(): + raise HandoffNotFoundError(f"event not found: {event_id}") + event = LifecycleEvent.from_mapping(json.loads(path.read_text(encoding="utf-8"))) + event.delivery_state = "pending" + event.attempts = 0 + event.next_attempt_at = now_iso() + event.last_error = "" + self._atomic_write_json(path, event.to_dict()) + return event + + def recover_transactions(self) -> list[str]: + recovered: list[str] = [] + for path in sorted(self.transactions_root.glob("*.json")): + payload = json.loads(path.read_text(encoding="utf-8")) + event = LifecycleEvent.from_mapping(dict(payload["event"])) + if event.event_id != path.stem: + raise RepositoryConflictError( + f"prepared transaction event_id mismatch: path={path.stem} payload={event.event_id}" + ) + try: + record = self.load_records(event.handoff_id)[0].record + except HandoffNotFoundError: + continue + if record.revision != event.revision or record.status != event.status: + continue + self.enqueue_event(event) + self.complete_event(event.event_id) + recovered.append(event.event_id) + return recovered + + def resolve_transaction( + self, + event_id: str, + *, + action: str, + reason: str, + force_recoverable_discard: bool = False, + ) -> LifecycleEvent: + normalized_action = action.strip().lower() + if normalized_action not in {"enqueue", "discard"}: + raise ValueError("transaction action must be enqueue or discard") + if force_recoverable_discard and normalized_action != "discard": + raise ValueError("force_recoverable_discard is valid only for discard") + normalized_reason = reason.strip() + if not normalized_reason: + raise ValueError("transaction resolution requires a reason") + with self._file_lock(f"event:{event_id}"): + path = self._transaction_path(event_id) + if not path.exists(): + raise HandoffNotFoundError(f"prepared transaction not found: {event_id}") + payload = json.loads(path.read_text(encoding="utf-8")) + event = LifecycleEvent.from_mapping(dict(payload["event"])) + if event.event_id != event_id: + raise RepositoryConflictError( + f"prepared transaction event_id mismatch: path={event_id} payload={event.event_id}" + ) + canonical_revision: int | None = None + canonical_status: str | None = None + try: + canonical = self.load_records(event.handoff_id)[0].record + canonical_revision = canonical.revision + canonical_status = canonical.status + except HandoffNotFoundError: + pass + canonical_match = canonical_revision == event.revision and canonical_status == event.status + if normalized_action == "discard" and canonical_match and not force_recoverable_discard: + raise ValueError( + "prepared transaction is still recoverable; enqueue it or explicitly force recoverable discard" + ) + if normalized_action == "enqueue": + self.enqueue_event(event) + self._atomic_write_json( + self._resolved_transaction_path(event_id), + { + "schema_version": CURRENT_SCHEMA_VERSION, + "event": event.to_dict(), + "action": normalized_action, + "reason": normalized_reason[:1000], + "resolved_at": now_iso(), + "canonical_revision": canonical_revision, + "canonical_status": canonical_status, + "canonical_match": canonical_match, + "force_recoverable_discard": bool(force_recoverable_discard), + }, + ) + self.complete_event(event_id) + audit_reason = normalized_reason[:500].replace("\r", " ").replace("\n", " ") + self.append_diagnostic_audit( + event.handoff_id, + "transaction_resolved", + f"event_id={event_id} action={normalized_action} reason={audit_reason}", + ) + return event + + def _preserve_conflict(self, path: Path, *, reason: str) -> Path: + digest = hashlib.sha256(path.read_bytes()).hexdigest()[:12] + destination = self.conflicts_root / f"{path.stem}.{digest}.{path.parent.parent.name}.md" + if not destination.exists(): + self._atomic_write_bytes(destination, path.read_bytes()) + self.append_diagnostic_audit(path.stem, "mirror_conflict", f"{reason}; evidence={destination}") + return destination + + def reconcile_handoff(self, handoff_id: str, *, repair: bool = True) -> list[str]: + issues: list[str] = [] + with self.handoff_lock(handoff_id): + records = self.load_records(handoff_id) + canonical_path = self._canonical_path(handoff_id) + canonical = next((item for item in records if item.path == canonical_path), None) + queue_records = [item for item in records if "incoming" in item.path.parts or "outgoing" in item.path.parts] + legacy_without_canonical = canonical is None + if canonical is None: + if not queue_records: + return [f"{handoff_id}: missing canonical record"] + byte_variants = {item.path.read_bytes() for item in queue_records} + if len(byte_variants) > 1: + if repair: + for item in queue_records: + self._preserve_conflict(item.path, reason="no canonical record and divergent queue copies") + return [f"{handoff_id}: ambiguous divergent v1 queue copies"] + canonical = queue_records[0] + if repair: + issues.append(f"{handoff_id}: missing canonical record") + canonical.record.schema_version = CURRENT_SCHEMA_VERSION + self._write_record(canonical_path, canonical.record) + + expected = self._record_bytes(canonical.record) + expected_paths = [ + self._queue_path("outgoing", canonical.record.sender, handoff_id), + self._queue_path("incoming", canonical.record.recipient, handoff_id), + ] + for path in expected_paths: + if not path.exists(): + issues.append(f"{handoff_id}: missing mirror {path.relative_to(self.bridge_root)}") + if repair: + self._atomic_write_bytes(path, expected) + continue + if legacy_without_canonical: + if repair: + self._atomic_write_bytes(path, expected) + continue + item = self._load_path(path) + if item.record.revision != canonical.record.revision: + issues.append( + f"{handoff_id}: stale revision {path.relative_to(self.bridge_root)}=" + f"{item.record.revision} canonical={canonical.record.revision}" + ) + if repair: + self._preserve_conflict(path, reason="stale mirror revision") + self._atomic_write_bytes(path, expected) + elif path.read_bytes() != expected: + issues.append(f"{handoff_id}: divergent mirror {path.relative_to(self.bridge_root)}") + if repair: + self._preserve_conflict(path, reason="divergent mirror content") + self._atomic_write_bytes(path, expected) + if issues and repair: + self.append_diagnostic_audit(handoff_id, "reconciled", "; ".join(issues)) + return issues + + def diagnose(self, *, pending_age_seconds: float = 300.0) -> list[str]: + issues: list[str] = [] + ids = {path.stem for path in self.records_root.glob("*.md")} + ids.update(path.stem for path in self.bridge_root.glob("incoming/*/*.md")) + ids.update(path.stem for path in self.bridge_root.glob("outgoing/*/*.md")) + for handoff_id in sorted(ids): + issues.extend(self.reconcile_handoff(handoff_id, repair=False)) + now = __import__("time").time() + for event in self.list_events(): + path = self._event_path(event.event_id) + if event.delivery_state in {"pending", "retrying"} and now - path.stat().st_mtime > pending_age_seconds: + issues.append(f"{event.event_id}: pending outbox event beyond threshold") + if event.delivery_state == "dead_letter": + issues.append(f"{event.event_id}: dead-letter event") + for path in sorted(self.idempotency_root.glob("*/*.json")): + try: + index = json.loads(path.read_text(encoding="utf-8")) + record = self.load_records(str(index["handoff_id"]))[0].record + if record.sender != index.get("sender") or record.canonical_request_hash != index.get("request_hash"): + issues.append(f"{path.relative_to(self.bridge_root)}: idempotency index inconsistency") + except (OSError, ValueError, KeyError, HandoffNotFoundError, json.JSONDecodeError): + issues.append(f"{path.relative_to(self.bridge_root)}: idempotency index inconsistency") + for path in sorted(self.transactions_root.glob("*.json")): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + event = LifecycleEvent.from_mapping(dict(payload["event"])) + if event.event_id != path.stem: + issues.append(f"{path.stem}: prepared transaction event_id mismatch") + else: + issues.append(f"{path.stem}: incomplete event transaction") + except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError): + issues.append(f"{path.stem}: malformed event transaction") + return issues \ No newline at end of file diff --git a/bridge_core/models.py b/bridge_core/models.py index 89b7338..09a89d7 100644 --- a/bridge_core/models.py +++ b/bridge_core/models.py @@ -4,10 +4,13 @@ from datetime import datetime, timezone from pathlib import Path from typing import Sequence +import hashlib +import json import random +CURRENT_SCHEMA_VERSION = 2 ACTIVE_STATUSES = {"open", "acknowledged", "in_progress", "blocked"} -ALL_STATUSES = ACTIVE_STATUSES | {"closed", "archived"} +ALL_STATUSES = ACTIVE_STATUSES | {"closed", "cancelled", "archived"} HANDOFF_KINDS = {"incident", "request", "question", "result"} PRIORITIES = {"low", "medium", "high", "urgent"} RISK_LEVELS = {"low", "medium", "high"} @@ -41,6 +44,9 @@ class CreateHandoffInput: response_format: str = DEFAULT_RESPONSE_FORMAT related_paths: Sequence[str] = field(default_factory=list) constraints: str = "- none" + idempotency_key: str | None = None + correlation_id: str | None = None + causation_id: str | None = None def validate(self) -> None: if self.handoff_kind not in HANDOFF_KINDS: @@ -49,6 +55,31 @@ def validate(self) -> None: raise ValueError(f"invalid priority: {self.priority}") if self.risk_level not in RISK_LEVELS: raise ValueError(f"invalid risk_level: {self.risk_level}") + if self.idempotency_key is not None and not self.idempotency_key.strip(): + raise ValueError("idempotency_key must not be empty") + + def canonical_request_hash(self) -> str: + payload = { + "sender": self.sender, + "recipient": self.recipient, + "issue_type": self.issue_type, + "subject": self.subject, + "requested_action": self.requested_action, + "minimal_context": self.minimal_context, + "handoff_kind": self.handoff_kind, + "priority": self.priority, + "risk_level": self.risk_level, + "due_at": self.due_at or "none", + "approval_needed": bool(self.approval_needed), + "approval_context": self.approval_context or "none", + "response_format": self.response_format, + "related_paths": list(self.related_paths), + "constraints": self.constraints or "- none", + "correlation_id": self.correlation_id, + "causation_id": self.causation_id, + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() @dataclass(slots=True) @@ -73,6 +104,12 @@ class HandoffRecord: response_format: str = DEFAULT_RESPONSE_FORMAT related_paths: list[str] = field(default_factory=list) body: str = "" + schema_version: int = CURRENT_SCHEMA_VERSION + revision: int = 0 + correlation_id: str | None = None + causation_id: str | None = None + canonical_request_hash: str = "" + idempotency_key_hash: str = "" @classmethod def from_create(cls, data: CreateHandoffInput) -> "HandoffRecord": @@ -99,6 +136,11 @@ def from_create(cls, data: CreateHandoffInput) -> "HandoffRecord": response_format=data.response_format, related_paths=list(data.related_paths), body=render_body(data.requested_action, data.minimal_context, data.constraints), + schema_version=CURRENT_SCHEMA_VERSION, + revision=0, + correlation_id=data.correlation_id, + causation_id=data.causation_id, + canonical_request_hash=data.canonical_request_hash(), ) @classmethod @@ -124,10 +166,18 @@ def from_mapping(cls, mapping: dict[str, object], body: str) -> "HandoffRecord": response_format=str(mapping.get("response_format", DEFAULT_RESPONSE_FORMAT) or DEFAULT_RESPONSE_FORMAT), related_paths=[str(item) for item in mapping.get("related_paths", [])], body=body, + schema_version=int(str(mapping.get("schema_version", 1) or 1)), + revision=int(str(mapping.get("revision", 0) or 0)), + correlation_id=_optional_string(mapping.get("correlation_id")), + causation_id=_optional_string(mapping.get("causation_id")), + canonical_request_hash=str(mapping.get("canonical_request_hash", "") or ""), + idempotency_key_hash=str(mapping.get("idempotency_key_hash", "") or ""), ) def to_frontmatter(self) -> dict[str, object]: return { + "schema_version": self.schema_version, + "revision": self.revision, "handoff_id": self.handoff_id, "status": self.status, "created_at": self.created_at, @@ -147,6 +197,10 @@ def to_frontmatter(self) -> dict[str, object]: "subject": self.subject, "response_format": self.response_format, "related_paths": list(self.related_paths), + "correlation_id": self.correlation_id or "none", + "causation_id": self.causation_id or "none", + "canonical_request_hash": self.canonical_request_hash, + "idempotency_key_hash": self.idempotency_key_hash, } @@ -156,6 +210,11 @@ class StoredHandoff: record: HandoffRecord +def _optional_string(value: object) -> str | None: + normalized = str(value or "").strip() + return None if normalized in {"", "none"} else normalized + + def render_body(requested_action: str, minimal_context: str, constraints: str) -> str: return "\n".join( [ diff --git a/bridge_core/policy.py b/bridge_core/policy.py index 230489f..92e8141 100644 --- a/bridge_core/policy.py +++ b/bridge_core/policy.py @@ -5,11 +5,12 @@ STATUS_ALIASES = {"acked": "acknowledged"} STATUS_TRANSITIONS = { - "open": {"open", "acknowledged", "in_progress", "blocked", "closed"}, - "acknowledged": {"acknowledged", "in_progress", "blocked", "closed"}, - "in_progress": {"in_progress", "blocked", "closed", "acknowledged"}, - "blocked": {"blocked", "in_progress", "closed", "acknowledged"}, + "open": {"open", "acknowledged", "in_progress", "blocked", "closed", "cancelled"}, + "acknowledged": {"acknowledged", "in_progress", "blocked", "closed", "cancelled"}, + "in_progress": {"in_progress", "blocked", "closed", "acknowledged", "cancelled"}, + "blocked": {"blocked", "in_progress", "closed", "acknowledged", "cancelled"}, "closed": {"closed", "archived"}, + "cancelled": {"cancelled", "archived"}, "archived": {"archived"}, } @@ -77,6 +78,20 @@ def require_status_transition(current: str, new: str, *, has_resolution_summary: return new_normalized +def require_lifecycle_role(*, actor: str, sender: str, recipient: str, new_status: str, maintenance: bool = False) -> None: + if maintenance: + return + require_actor_access(actor, sender, recipient) + normalized = normalize_status(new_status) + if actor == recipient and normalized in {"acknowledged", "in_progress", "blocked", "closed", "archived"}: + return + if actor == sender and normalized == "cancelled": + return + if normalized in {"open"} and actor == sender: + return + raise AccessPolicyError(f"actor {actor} is not authorized to set {normalized} for this handoff") + + def is_active_status(status: str) -> bool: return normalize_status(status) in ACTIVE_STATUSES diff --git a/bridge_core/repository.py b/bridge_core/repository.py index 774bb71..6e25f46 100644 --- a/bridge_core/repository.py +++ b/bridge_core/repository.py @@ -2,7 +2,9 @@ from abc import ABC, abstractmethod from pathlib import Path +from typing import ContextManager +from .events import LifecycleEvent from .models import HandoffRecord, StoredHandoff @@ -10,9 +12,25 @@ class HandoffNotFoundError(FileNotFoundError): pass +class IdempotencyConflictError(ValueError): + pass + + +class RevisionConflictError(ValueError): + pass + + +class RepositoryConflictError(RuntimeError): + pass + + class BridgeRepository(ABC): @abstractmethod - def create(self, handoff: HandoffRecord) -> HandoffRecord: + def create(self, handoff: HandoffRecord, *, idempotency_key: str | None = None) -> HandoffRecord: + raise NotImplementedError + + @abstractmethod + def handoff_lock(self, handoff_id: str) -> ContextManager[None]: raise NotImplementedError @abstractmethod @@ -34,3 +52,38 @@ def archive_records(self, records: list[StoredHandoff]) -> Path: @abstractmethod def append_audit(self, handoff: HandoffRecord, status: str) -> None: raise NotImplementedError + + @abstractmethod + def enqueue_event(self, event: LifecycleEvent) -> LifecycleEvent: + raise NotImplementedError + + @abstractmethod + def prepare_event(self, event: LifecycleEvent) -> None: + raise NotImplementedError + + @abstractmethod + def complete_event(self, event_id: str) -> None: + raise NotImplementedError + + @abstractmethod + def list_events(self, *, states: set[str] | None = None) -> list[LifecycleEvent]: + raise NotImplementedError + + @abstractmethod + def update_event(self, event: LifecycleEvent) -> LifecycleEvent: + raise NotImplementedError + + @abstractmethod + def replay_event(self, event_id: str) -> LifecycleEvent: + raise NotImplementedError + + @abstractmethod + def resolve_transaction( + self, + event_id: str, + *, + action: str, + reason: str, + force_recoverable_discard: bool = False, + ) -> LifecycleEvent: + raise NotImplementedError diff --git a/bridge_core/service.py b/bridge_core/service.py index 6d63c3c..f9b8fc7 100644 --- a/bridge_core/service.py +++ b/bridge_core/service.py @@ -1,18 +1,24 @@ from __future__ import annotations +from datetime import datetime, timedelta, timezone from pathlib import Path +from typing import Callable +import random import re -from .models import CreateHandoffInput, HandoffRecord, StoredHandoff, now_iso +from .events import LifecycleEvent +from .models import CURRENT_SCHEMA_VERSION, CreateHandoffInput, HandoffRecord, StoredHandoff, now_iso from .policy import ( + AccessPolicyError, StatusPolicyError, is_active_status, normalize_status, require_actor_access, + require_lifecycle_role, require_route, require_status_transition, ) -from .repository import BridgeRepository +from .repository import BridgeRepository, IdempotencyConflictError, RepositoryConflictError, RevisionConflictError class BridgeService: @@ -21,8 +27,35 @@ def __init__(self, repository: BridgeRepository): def create_handoff(self, request: CreateHandoffInput) -> HandoffRecord: require_route(request.sender, request.recipient) - handoff = HandoffRecord.from_create(request) - return self.repository.create(handoff) + request.validate() + proposed = HandoffRecord.from_create(request) + event = LifecycleEvent.from_handoff( + proposed, + event_type="handoff_created", + actor=request.sender, + target_agent=request.recipient, + ) + self.repository.prepare_event(event) + try: + created = self.repository.create(proposed, idempotency_key=request.idempotency_key) + except IdempotencyConflictError: + self.repository.complete_event(event.event_id) + raise + except RepositoryConflictError: + # Repository conflicts occur before this proposed handoff becomes + # canonical, so its prepared create event is not recoverable. + self.repository.complete_event(event.event_id) + raise + except BaseException: + # A prepared transaction is intentionally retained if canonical state was + # committed before a crash; recovery can decide whether to enqueue it. + raise + if created.handoff_id != proposed.handoff_id: + self.repository.complete_event(event.event_id) + return created + self.repository.enqueue_event(event) + self.repository.complete_event(event.event_id) + return created def get_handoff(self, handoff_id: str, *, actor: str | None = None) -> HandoffRecord: records = self.repository.load_records(handoff_id) @@ -48,29 +81,234 @@ def set_status( status: str, outcome: str = "", acknowledgment_source: str | None = None, + expected_revision: int | None = None, + maintenance: bool = False, ) -> HandoffRecord: - records = self.repository.load_records(handoff_id) - primary = records[0].record - require_actor_access(actor, primary.sender, primary.recipient) - outcome_text = outcome.strip() - has_resolution_summary = bool(outcome_text or primary.resolution_summary != "pending") - normalized_status = require_status_transition(primary.status, status, has_resolution_summary=has_resolution_summary) - ack_source = _normalize_acknowledgment_source(acknowledgment_source, status=normalized_status) - updated = self._mutate_records(records, normalized_status, outcome_text, acknowledgment_source=ack_source) - self.repository.save_records(records) - self.repository.append_audit(updated, updated.status) - return updated + with self.repository.handoff_lock(handoff_id): + records = self.repository.load_records(handoff_id) + primary = records[0].record + normalized_requested = normalize_status(status) + require_lifecycle_role( + actor=actor, + sender=primary.sender, + recipient=primary.recipient, + new_status=normalized_requested, + maintenance=maintenance, + ) + outcome_text = outcome.strip() + if expected_revision is not None and expected_revision != primary.revision: + if self._is_safe_same_state_retry(primary, normalized_requested, outcome_text): + return primary + raise RevisionConflictError( + f"stale revision for {handoff_id}: expected {expected_revision}, current {primary.revision}" + ) + if self._is_safe_same_state_retry(primary, normalized_requested, outcome_text): + return primary + has_resolution_summary = bool(outcome_text or primary.resolution_summary != "pending") + normalized_status = require_status_transition( + primary.status, + normalized_requested, + has_resolution_summary=has_resolution_summary, + ) + ack_source = _normalize_acknowledgment_source(acknowledgment_source, status=normalized_status) + next_revision = primary.revision + 1 + updated = self._mutate_records( + records, + normalized_status, + outcome_text, + acknowledgment_source=ack_source, + revision=next_revision, + ) + target_agent = updated.recipient if normalized_status == "cancelled" else updated.sender + event = LifecycleEvent.from_handoff( + updated, + event_type=f"handoff_{normalized_status}", + actor=actor, + target_agent=target_agent, + ) + self.repository.prepare_event(event) + self.repository.save_records(records) + self.repository.append_audit(updated, updated.status) + self.repository.enqueue_event(event) + self.repository.complete_event(event.event_id) + return updated - def archive_handoff(self, handoff_id: str, *, actor: str) -> Path: - records = self.repository.load_records(handoff_id) - primary = records[0].record - require_actor_access(actor, primary.sender, primary.recipient) - if {normalize_status(item.record.status) for item in records} != {"closed"}: - raise StatusPolicyError("only closed handoffs can be archived") - archived = self._mutate_records(records, "archived", "", acknowledgment_source=None) - archive_dir = self.repository.archive_records(records) - self.repository.append_audit(archived, archived.status) - return archive_dir + def cancel_handoff( + self, + handoff_id: str, + *, + actor: str, + reason: str, + expected_revision: int | None = None, + ) -> HandoffRecord: + if not reason.strip(): + raise StatusPolicyError("cancelling a handoff requires a reason") + return self.set_status( + handoff_id, + actor=actor, + status="cancelled", + outcome=reason, + expected_revision=expected_revision, + ) + + def archive_handoff( + self, + handoff_id: str, + *, + actor: str, + expected_revision: int | None = None, + maintenance: bool = False, + ) -> Path: + with self.repository.handoff_lock(handoff_id): + records = self.repository.load_records(handoff_id) + primary = records[0].record + require_lifecycle_role( + actor=actor, + sender=primary.sender, + recipient=primary.recipient, + new_status="archived", + maintenance=maintenance, + ) + if expected_revision is not None and expected_revision != primary.revision: + raise RevisionConflictError( + f"stale revision for {handoff_id}: expected {expected_revision}, current {primary.revision}" + ) + statuses = {normalize_status(item.record.status) for item in records} + if not statuses <= {"closed", "cancelled"}: + raise StatusPolicyError("only closed or cancelled handoffs can be archived") + archived = self._mutate_records( + records, + "archived", + "", + acknowledgment_source=None, + revision=primary.revision + 1, + ) + event = LifecycleEvent.from_handoff( + archived, + event_type="handoff_archived", + actor=actor, + target_agent=archived.sender, + ) + self.repository.prepare_event(event) + archive_dir = self.repository.archive_records(records) + self.repository.append_audit(archived, archived.status) + self.repository.enqueue_event(event) + self.repository.complete_event(event.event_id) + return archive_dir + + def list_outbox(self, *, states: set[str] | None = None) -> list[LifecycleEvent]: + return self.repository.list_events(states=states) + + def replay_event(self, event_id: str) -> LifecycleEvent: + return self.repository.replay_event(event_id) + + def resolve_transaction( + self, + event_id: str, + *, + action: str, + reason: str, + force_recoverable_discard: bool = False, + ) -> LifecycleEvent: + return self.repository.resolve_transaction( + event_id, + action=action, + reason=reason, + force_recoverable_discard=force_recoverable_discard, + ) + + def process_outbox( + self, + deliver: Callable[[LifecycleEvent], tuple[bool, str]], + *, + limit: int = 100, + max_attempts: int = 8, + base_delay_seconds: float = 1.0, + max_delay_seconds: float = 300.0, + ) -> list[LifecycleEvent]: + processed: list[LifecycleEvent] = [] + due = [event for event in self.list_outbox(states={"pending", "retrying"}) if event.due()] + for event in due[: max(0, limit)]: + ok, detail = deliver(event) + if ok: + processed.append(self._mark_event_delivered(event)) + else: + processed.append( + self._mark_event_failed( + event, + detail, + max_attempts=max_attempts, + base_delay_seconds=base_delay_seconds, + max_delay_seconds=max_delay_seconds, + ) + ) + return processed + + def mark_event_delivered(self, event_id: str) -> LifecycleEvent: + event = self._event(event_id) + return self._mark_event_delivered(event) + + def _mark_event_delivered(self, event: LifecycleEvent) -> LifecycleEvent: + event.delivery_state = "delivered" + event.last_error = "" + return self.repository.update_event(event) + + def mark_event_failed( + self, + event_id: str, + error: str, + *, + max_attempts: int = 8, + base_delay_seconds: float = 1.0, + max_delay_seconds: float = 300.0, + jitter_ratio: float = 0.2, + ) -> LifecycleEvent: + event = self._event(event_id) + return self._mark_event_failed( + event, + error, + max_attempts=max_attempts, + base_delay_seconds=base_delay_seconds, + max_delay_seconds=max_delay_seconds, + jitter_ratio=jitter_ratio, + ) + + def _mark_event_failed( + self, + event: LifecycleEvent, + error: str, + *, + max_attempts: int = 8, + base_delay_seconds: float = 1.0, + max_delay_seconds: float = 300.0, + jitter_ratio: float = 0.2, + ) -> LifecycleEvent: + event.attempts += 1 + event.last_error = error.strip()[:1000] + if event.attempts >= max(1, max_attempts): + event.delivery_state = "dead_letter" + event.next_attempt_at = now_iso() + else: + event.delivery_state = "retrying" + base = min(max_delay_seconds, base_delay_seconds * (2 ** (event.attempts - 1))) + jitter = random.uniform(0.0, max(0.0, base * jitter_ratio)) + next_attempt = datetime.now(timezone.utc) + timedelta(seconds=base + jitter) + event.next_attempt_at = next_attempt.isoformat().replace("+00:00", "Z") + return self.repository.update_event(event) + + def _event(self, event_id: str) -> LifecycleEvent: + for event in self.repository.list_events(): + if event.event_id == event_id: + return event + raise FileNotFoundError(f"event not found: {event_id}") + + @staticmethod + def _is_safe_same_state_retry(primary: HandoffRecord, status: str, outcome: str) -> bool: + if normalize_status(primary.status) != status: + return False + if not outcome: + return True + return primary.resolution_summary == outcome def _mutate_records( self, @@ -79,11 +317,14 @@ def _mutate_records( outcome: str, *, acknowledgment_source: str | None, + revision: int, ) -> HandoffRecord: updated: HandoffRecord | None = None transition_time = now_iso() for stored in records: stored.record.status = status + stored.record.schema_version = CURRENT_SCHEMA_VERSION + stored.record.revision = revision stored.record.updated_at = transition_time if status == "acknowledged" and stored.record.acknowledged_at == "none": stored.record.acknowledged_at = transition_time @@ -91,7 +332,8 @@ def _mutate_records( if outcome: stored.record.resolution_summary = outcome stored.record.body = self._replace_outcome(stored.record.body, outcome) - updated = stored.record + if updated is None: + updated = stored.record assert updated is not None return updated @@ -110,4 +352,4 @@ def _normalize_acknowledgment_source(value: str | None, *, status: str) -> str | normalized = str(value or "manual").strip().lower() if normalized not in {"manual", "auto"}: raise StatusPolicyError("acknowledgment_source must be auto or manual") - return normalized + return normalized \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index d4aede9..7d48c0e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -3,6 +3,7 @@ ## Start here - [`../README.md`](../README.md) — project overview and quick start - [`architecture.md`](architecture.md) — component boundaries and data flow +- [`bridge-v2-reliability.md`](bridge-v2-reliability.md) — reliability contracts, recovery, migration, deployment, and rollback - [`deployment.md`](deployment.md) — deployment model and systemd template usage - [`runtime-layout.md`](runtime-layout.md) — runtime directories under `BRIDGE_ROOT` diff --git a/docs/architecture.md b/docs/architecture.md index 7781f0a..5bf623d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -14,10 +14,11 @@ Bridge provides explicit, auditable handoffs between autonomous agents without s ## Data flow 1. sender creates handoff through CLI or API-backed wrapper 2. Bridge validates auth, route policy, and status rules -3. Bridge writes sender outbox copy and recipient inbox copy under `BRIDGE_ROOT` -4. optional recipient notify URL receives immediate lifecycle event -5. recipient acknowledges, blocks, closes, or archives handoff -6. optional read-only lifecycle consumer receives `closed`/`blocked` style events +3. Bridge v2 atomically writes a canonical record, then recoverable sender/recipient queue views +4. Bridge durably enqueues a versioned lifecycle event with at-least-once delivery semantics +5. optional recipient notify URL receives an immediate compatibility fast path +6. recipient acknowledges, progresses, blocks, closes, or archives; sender may explicitly cancel/withdraw +7. outbox delivery retries consumer outages with stable event IDs and dead-letter evidence ## Boundaries - Bridge ships with no built-in agent IDs @@ -25,6 +26,11 @@ Bridge provides explicit, auditable handoffs between autonomous agents without s - lifecycle consumers are adapters outside core logic - runtime state stays under `BRIDGE_ROOT`, outside tracked source - agent-private memory stays outside Bridge entirely +- durable work state and worker dispatch stay in external work queues, not Bridge + +## Reliability v2 + +See [`bridge-v2-reliability.md`](bridge-v2-reliability.md) for the canonical-record transaction model, optimistic revisions, lifecycle authorization matrix, durable outbox, v1 compatibility, adapter contract, migration, deployment, and rollback procedures. ## Why package name stays `bridge_core/` `bridge_core/` is importable Python package name used across scripts and tests. Renaming it to bare `src/` would not be a like-for-like cleanup; `src/` is usually container directory for package layout, not package name itself. diff --git a/docs/bridge-v2-reliability.md b/docs/bridge-v2-reliability.md new file mode 100644 index 0000000..c5d6526 --- /dev/null +++ b/docs/bridge-v2-reliability.md @@ -0,0 +1,201 @@ +# Bridge v2 reliability foundation + +## Scope and non-goals + +Bridge v2 is a backward-compatible reliability layer for explicit, scoped, redacted agent handoffs. It is an at-least-once event bus with idempotent producers and consumers. It does **not** provide exactly-once delivery, worker dispatch, shared memory, or durable work-state ownership. External work queues remain independent systems. + +Bridge core stays generic: `bridge_core/` imports no Hermes, Kanban, profile, vault, or messaging-platform code. + +## Transaction and recovery model + +### Canonical state and queue views + +A v2 handoff has one canonical Markdown record at: + +```text +records/.md +``` + +The existing files remain as backward-compatible queue views: + +```text +outgoing//.md +incoming//.md +``` + +The canonical record is authoritative. Record writes use a same-directory temporary file, flush, file `fsync`, mode `0600`, atomic rename, and parent-directory `fsync`. Creation and lifecycle mutation use an exclusive per-handoff `flock` plus an in-process reentrant lock. + +A create writes canonical state before queue views. If the process stops after one view, `reconcile` rebuilds the missing view from canonical evidence. A conflicting view is copied to `audit/conflicts/`, an audit event is appended, and only then may the view be replaced. Ambiguous v1 conflicts without canonical evidence are never auto-selected. + +Two byte-identical v1 queue copies without a canonical record are unambiguous legacy state, not divergence. Check-only reconciliation leaves them quiet. Repair mode materializes the v2 canonical record and rewrites both queue views to the canonical v2 representation without creating false conflict evidence. Non-identical v1 copies remain an operator decision. + +### Idempotent create + +Clients may send `Idempotency-Key` or `idempotency_key`. The key is scoped to the authenticated sender. Bridge stores only its SHA-256 digest in `idempotency//`; it stores a canonical request SHA-256 over material request fields. The raw key is not written. + +- same sender + key + canonical payload: return the original handoff; +- same sender + key + materially different payload: conflict (`HTTP 409`); +- concurrent identical requests: serialized by the sender/key lock and produce one handoff. + +Canonical state is written before its idempotency index. If a process stops in that interval, a retry under the same sender/key lock scans canonical records for the stored key digest, validates the canonical request hash, rebuilds the missing index, repairs queue views, and returns the original handoff. Multiple canonical claims for one sender/key digest fail closed as a repository conflict. + +### Lifecycle transactions and outbox + +Before a lifecycle state commit, Bridge writes a prepared transaction journal containing the immutable event envelope. It then: + +1. atomically commits canonical handoff state and queue views; +2. fsync-appends audit evidence; +3. atomically writes the outbox event; +4. removes the prepared transaction. + +If a process stops after state commit but before outbox enqueue, `outbox-recover` compares the prepared event with canonical revision/status and enqueues the same `event_id`. Consumer failure never rolls back handoff state. + +Automatic recovery deliberately leaves a prepared event untouched when canonical state has advanced or disappeared. An operator must inspect the transaction, canonical audit history, and downstream consumer state, then explicitly resolve it with `outbox-resolve-transaction`: `enqueue` preserves at-least-once historical delivery with the original `event_id`; `discard` is only for proven duplicate/invalid transactions. Bridge refuses to discard an event that still exactly matches canonical state unless the operator also supplies `--force-recoverable-discard`. Both actions require a reason and preserve canonical match evidence under `audit/resolved-transactions/` before removing the journal. + +Events live at `outbox/events/.json`. State transitions are `pending` → `retrying` → `delivered` or `dead_letter`. Backoff is bounded exponential delay plus jitter. Manual replay resets attempts and scheduling but retains `event_id`. + +Immediate notify remains a compatibility fast path. Its body preserves the v1 create/block/close shape and carries `X-Bridge-Event-ID`. Durable `outbox-deliver` sends the full v2 event body and the same header. + +## Record contract + +New records use: + +- `schema_version: 2` +- `revision: 0` at create, incremented once per lifecycle mutation +- optional `correlation_id` +- optional `causation_id` +- `canonical_request_hash` +- `idempotency_key_hash` when create idempotency is used + +Missing v2 fields mean a v1 record. A v1 record loads as `schema_version=1`, `revision=0` without an eager migration. Its first successful lifecycle mutation writes v2 canonical state and v2 queue views at revision 1. + +## Lifecycle event contract + +Every durable lifecycle event has: + +- `schema_version` +- unique stable `event_id` +- `handoff_id` +- `event_type` +- `revision` +- `occurred_at` +- `target_agent`, `sender`, `recipient`, `actor` +- `status`, `subject`, `resolution_summary` +- optional `correlation_id`, `causation_id` +- optional generic `external_correlation` mapping +- `attempts`, `next_attempt_at`, `last_error`, `delivery_state` + +Lifecycle event types are: + +- `handoff_created` +- `handoff_acknowledged` +- `handoff_in_progress` +- `handoff_blocked` +- `handoff_closed` +- `handoff_cancelled` +- `handoff_archived` + +`retrying` and `dead_letter` are durable delivery states for the same immutable event. Diagnostics/adapters may surface these as `handoff_retrying` and `handoff_dead_lettered` operational notifications without creating a second recursively delivered outbox item. + +## Optimistic concurrency + +API callers may send `expected_revision` or `If-Match: ""`. CLI/wrapper callers use `--expected-revision`. A stale update returns `HTTP 409 revision_conflict`. + +A repeated same-state request is a no-op when its outcome is empty or exactly matches the stored outcome. It does not increment revision or create another event. Authorization is checked before this no-op rule. + +## Lifecycle authorization matrix + +| Action | Sender | Recipient | Host maintenance | +|---|---:|---:|---:| +| create | yes | no | explicit tooling only | +| acknowledge | no | yes | recovery override | +| in progress | no | yes | recovery override | +| block | no | yes | recovery override | +| close with verified outcome | no | yes | recovery override | +| cancel/withdraw with reason | yes | no | recovery override | +| archive closed or sender-cancelled handoff | no | yes | recovery override | + +The API exposes no maintenance override. The service-layer `maintenance=True` path is deliberately explicit for offline host recovery tooling and must not be granted through participant tokens. + +## Generic work-queue adapter contract + +An external adapter consumes full v2 outbox events. It must persist processed `event_id` values and maintain a mapping keyed by `handoff_id`. Duplicate `event_id` is a no-op. + +Recommended reconciliation: + +- `handoff_acknowledged`: create or find the durable work item; persist its external ID in adapter-owned state. +- `handoff_in_progress`: update progress evidence, without making Bridge a dispatcher. +- `handoff_blocked`: update the existing work item with blocker evidence. +- `handoff_closed`: close/update the existing work item only after the event contains a verified outcome acceptable to local policy. +- `handoff_cancelled`: mark the work item withdrawn/cancelled under local policy. +- delivery failure: leave the event retrying; exhausted attempts provide dead-letter evidence. +- duplicate event: no-op and return success. + +An adapter may store a Kanban card ID in its own mapping or in generic `external_correlation` metadata. Bridge core neither imports nor calls Kanban. + +## Commands + +```bash +python3 scripts/bridge_cli.py outbox-list --state pending +python3 scripts/bridge_cli.py outbox-list --state dead_letter +python3 scripts/bridge_cli.py outbox-recover +python3 scripts/bridge_cli.py outbox-deliver --limit 100 +python3 scripts/bridge_cli.py outbox-replay EVT-... +python3 scripts/bridge_cli.py outbox-resolve-transaction EVT-... --action enqueue --reason "verified missing delivery" +python3 scripts/bridge_cli.py outbox-resolve-transaction EVT-... --action discard --reason "verified duplicate event" +python3 scripts/bridge_cli.py outbox-resolve-transaction EVT-... --action discard --force-recoverable-discard --reason "verified downstream duplicate; suppress recoverable replay" +python3 scripts/bridge_cli.py reconcile --check-only +python3 scripts/bridge_cli.py reconcile HND-... +``` + +`reconcile --check-only` is read-only. Repair mode restores only from unambiguous canonical evidence; conflict evidence is preserved before replacement. + +## Backward-compatible migration plan + +1. Deploy code without changing live data. +2. Run compile/tests and `reconcile --check-only` against a copied Bridge root. Identical v1 mirror pairs should not be reported as divergent. +3. Back up the live Bridge root with metadata and verify the backup can be listed/read. +4. Stop API/intake writers during the separately approved maintenance window. +5. Run `outbox-recover`, then `reconcile --check-only` on live data. +6. Resolve ambiguous divergent v1 copies manually; do not select by timestamp alone. +7. Run `reconcile` to lazily materialize canonical records and missing views. Verify identical v1 pairs migrate without conflict evidence. +8. Start the API, then run one bounded `outbox-deliver` pass. +9. Verify health, patrol, route/token authorization, open handoff counts, outbox/dead-letter counts, and one isolated synthetic route approved for the maintenance window. +10. Add one separately approved periodic invocation of `outbox-recover` + `outbox-deliver`; do not treat the API fast path as the retry scheduler. Enforce a single active deliverer with the service manager or an external lock. + +No bulk rewrite is required. Untouched v1 handoffs continue to load. + +## Live deployment plan (separate approval required) + +Deployment mutates production code/data and requires a doctor rundown plus explicit Cameron approval. The approved window should include: immutable backup path, exact release artifact, writer quiescence, pre/post patrol captures, service-specific restart scope, and rollback trigger. + +Do not modify token values or `config/bridge_api.env`. Existing route/token configuration remains valid. Optional retry settings are additive environment values: + +- `BRIDGE_OUTBOX_MAX_ATTEMPTS` +- `BRIDGE_OUTBOX_BASE_DELAY_SECONDS` +- `BRIDGE_OUTBOX_MAX_DELAY_SECONDS` +- `BRIDGE_PATROL_OUTBOX_PENDING_SECONDS` + +Adding those values or a periodic scheduler is a separate config/service approval. + +## Rollback + +1. Stop only Bridge API/intake/outbox writers in the approved window. +2. Preserve the current v2 Bridge root as forensic evidence; do not delete it. +3. Restore the pre-deployment Bridge-root backup atomically, including modes/ownership. +4. Restore the prior code release/symlink or package artifact. +5. Restore the prior service definitions only if they changed. +6. Start only the previously running Bridge units. +7. Verify local API health, route/token authorization, open handoff counts, and patrol. +8. Keep v2 outbox/transaction artifacts from the failed attempt for manual reconciliation; do not replay them into v1 automatically. + +Rollback trigger: canonical/queue divergence that cannot be reconciled unambiguously, authorization regression, sustained API errors, or outbox retries causing consumer harm. + +## Remaining operational risks + +- Filesystem atomic rename and `flock` assume a local POSIX filesystem with correct `fsync` behavior; network filesystems need separate validation. +- The API fast path is not a scheduler. Reliable retries require the separately deployed periodic `outbox-deliver` invocation. +- Legacy notify bodies are retained for compatibility; new adapters should consume full v2 outbox events. +- Manual edits to queue views become conflicts against canonical state and are preserved as evidence, not merged automatically. +- Event deduplication is an adapter responsibility keyed by `event_id`; Bridge guarantees at-least-once, not exactly-once. +- Outbox event files are not claimed or leased. A delivered/dead-letter state cannot be regressed by a stale failure update, but multiple deliverers can still duplicate consumer delivery; deploy exactly one deliverer until a claim/lease protocol is added. diff --git a/pyproject.toml b/pyproject.toml index 6af1b5b..2c36d24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,3 +31,6 @@ dev = [ [tool.pytest.ini_options] testpaths = ["tests"] pythonpath = ["."] + +[tool.setuptools] +packages = ["bridge_core"] diff --git a/scripts/bridge_agent.py b/scripts/bridge_agent.py index e7d2c43..096b2d0 100755 --- a/scripts/bridge_agent.py +++ b/scripts/bridge_agent.py @@ -34,6 +34,9 @@ def main() -> None: create_parser.add_argument('--related-path', action='append') create_parser.add_argument('--approval-needed', action='store_true') create_parser.add_argument('--approval-context', default='') + create_parser.add_argument('--idempotency-key') + create_parser.add_argument('--correlation-id') + create_parser.add_argument('--causation-id') sub.add_parser('list-open') @@ -42,22 +45,32 @@ def main() -> None: update_parser = sub.add_parser('set-status') update_parser.add_argument('handoff_id') - update_parser.add_argument('status', choices=['open', 'acknowledged', 'in_progress', 'blocked', 'closed']) + update_parser.add_argument('status', choices=['open', 'acknowledged', 'in_progress', 'blocked', 'closed', 'cancelled']) update_parser.add_argument('--outcome', default='') + update_parser.add_argument('--expected-revision', type=int) ack_parser = sub.add_parser('ack') ack_parser.add_argument('handoff_id') + ack_parser.add_argument('--expected-revision', type=int) block_parser = sub.add_parser('block') block_parser.add_argument('handoff_id') block_parser.add_argument('--outcome', required=True) + block_parser.add_argument('--expected-revision', type=int) close_parser = sub.add_parser('close') close_parser.add_argument('handoff_id') close_parser.add_argument('--outcome', required=True) + close_parser.add_argument('--expected-revision', type=int) + + cancel_parser = sub.add_parser('cancel') + cancel_parser.add_argument('handoff_id') + cancel_parser.add_argument('--reason', required=True) + cancel_parser.add_argument('--expected-revision', type=int) archive_parser = sub.add_parser('archive') archive_parser.add_argument('handoff_id') + archive_parser.add_argument('--expected-revision', type=int) args = parser.parse_args() agent = normalize_agent_id(args.agent) @@ -78,6 +91,9 @@ def main() -> None: 'related_paths': args.related_path or [], 'approval_needed': args.approval_needed, 'approval_context': args.approval_context, + 'idempotency_key': args.idempotency_key, + 'correlation_id': args.correlation_id, + 'causation_id': args.causation_id, } cli_args = [ 'create', '--sender', agent, '--recipient', payload['recipient'], '--issue-type', args.issue_type, @@ -95,25 +111,56 @@ def main() -> None: cli_args += ['--approval-context', args.approval_context] for path in args.related_path or []: cli_args += ['--related-path', path] + for option, value in ( + ('--idempotency-key', args.idempotency_key), + ('--correlation-id', args.correlation_id), + ('--causation-id', args.causation_id), + ): + if value: + cli_args += [option, value] invoke(agent, args, api_call=lambda: create_result(api_request(agent, 'POST', '/v1/handoffs', payload=payload)), cli_args=cli_args) elif args.cmd == 'list-open': invoke(agent, args, api_call=lambda: list_result(agent, api_request(agent, 'GET', '/v1/handoffs?active_only=true')), cli_args=['list-open', '--agent', agent]) elif args.cmd == 'status': invoke(agent, args, api_call=lambda: status_result(agent, args.handoff_id, api_request(agent, 'GET', f'/v1/handoffs/{args.handoff_id}')), cli_args=['status', '--actor', agent, args.handoff_id]) elif args.cmd == 'set-status': - payload = {'status': args.status, 'outcome': args.outcome} + payload = {'status': args.status, 'outcome': args.outcome, 'expected_revision': args.expected_revision} cli_args = ['set-status', '--actor', agent, args.handoff_id, args.status] if args.outcome: cli_args += ['--outcome', args.outcome] + if args.expected_revision is not None: + cli_args += ['--expected-revision', str(args.expected_revision)] invoke(agent, args, api_call=lambda: set_status_result(agent, api_request(agent, 'POST', f'/v1/handoffs/{args.handoff_id}/status', payload=payload)), cli_args=cli_args) elif args.cmd == 'ack': - invoke(agent, args, api_call=lambda: set_status_result(agent, api_request(agent, 'POST', f'/v1/handoffs/{args.handoff_id}/ack', payload={})), cli_args=['set-status', '--actor', agent, args.handoff_id, 'acknowledged']) + payload = {'expected_revision': args.expected_revision} + cli_args = ['set-status', '--actor', agent, args.handoff_id, 'acknowledged'] + if args.expected_revision is not None: + cli_args += ['--expected-revision', str(args.expected_revision)] + invoke(agent, args, api_call=lambda: set_status_result(agent, api_request(agent, 'POST', f'/v1/handoffs/{args.handoff_id}/ack', payload=payload)), cli_args=cli_args) elif args.cmd == 'block': - invoke(agent, args, api_call=lambda: set_status_result(agent, api_request(agent, 'POST', f'/v1/handoffs/{args.handoff_id}/block', payload={'outcome': args.outcome})), cli_args=['set-status', '--actor', agent, args.handoff_id, 'blocked', '--outcome', args.outcome]) + payload = {'outcome': args.outcome, 'expected_revision': args.expected_revision} + cli_args = ['set-status', '--actor', agent, args.handoff_id, 'blocked', '--outcome', args.outcome] + if args.expected_revision is not None: + cli_args += ['--expected-revision', str(args.expected_revision)] + invoke(agent, args, api_call=lambda: set_status_result(agent, api_request(agent, 'POST', f'/v1/handoffs/{args.handoff_id}/block', payload=payload)), cli_args=cli_args) elif args.cmd == 'close': - invoke(agent, args, api_call=lambda: set_status_result(agent, api_request(agent, 'POST', f'/v1/handoffs/{args.handoff_id}/close', payload={'outcome': args.outcome})), cli_args=['set-status', '--actor', agent, args.handoff_id, 'closed', '--outcome', args.outcome]) + payload = {'outcome': args.outcome, 'expected_revision': args.expected_revision} + cli_args = ['set-status', '--actor', agent, args.handoff_id, 'closed', '--outcome', args.outcome] + if args.expected_revision is not None: + cli_args += ['--expected-revision', str(args.expected_revision)] + invoke(agent, args, api_call=lambda: set_status_result(agent, api_request(agent, 'POST', f'/v1/handoffs/{args.handoff_id}/close', payload=payload)), cli_args=cli_args) + elif args.cmd == 'cancel': + payload = {'outcome': args.reason, 'expected_revision': args.expected_revision} + cli_args = ['set-status', '--actor', agent, args.handoff_id, 'cancelled', '--outcome', args.reason] + if args.expected_revision is not None: + cli_args += ['--expected-revision', str(args.expected_revision)] + invoke(agent, args, api_call=lambda: set_status_result(agent, api_request(agent, 'POST', f'/v1/handoffs/{args.handoff_id}/cancel', payload=payload)), cli_args=cli_args) elif args.cmd == 'archive': - invoke(agent, args, api_call=lambda: archive_result(agent, api_request(agent, 'POST', f'/v1/handoffs/{args.handoff_id}/archive', payload={})), cli_args=['archive', '--actor', agent, args.handoff_id]) + payload = {'expected_revision': args.expected_revision} + cli_args = ['archive', '--actor', agent, args.handoff_id] + if args.expected_revision is not None: + cli_args += ['--expected-revision', str(args.expected_revision)] + invoke(agent, args, api_call=lambda: archive_result(agent, api_request(agent, 'POST', f'/v1/handoffs/{args.handoff_id}/archive', payload=payload)), cli_args=cli_args) if __name__ == '__main__': diff --git a/scripts/bridge_api_server.py b/scripts/bridge_api_server.py index 89f03cd..389969a 100755 --- a/scripts/bridge_api_server.py +++ b/scripts/bridge_api_server.py @@ -22,7 +22,7 @@ from bridge_core.frontmatter import parse_frontmatter from bridge_core.models import CreateHandoffInput, DEFAULT_RESPONSE_FORMAT, HandoffRecord from bridge_core.policy import AccessPolicyError, BridgePolicyError, RoutePolicyError, StatusPolicyError, is_active_status, require_actor_access -from bridge_core.repository import HandoffNotFoundError +from bridge_core.repository import HandoffNotFoundError, IdempotencyConflictError, RevisionConflictError from bridge_core.runtime import env_key_for_agent from bridge_core.service import BridgeService @@ -90,7 +90,7 @@ def _dispatch(self, method: str) -> None: if method == "GET" and action is None: self._handle_get(actor, handoff_id) return - if method == "POST" and action in {"ack", "block", "close", "archive", "status"}: + if method == "POST" and action in {"ack", "block", "close", "cancel", "archive", "status"}: body = self._read_json_body(required=False) self._handle_action(actor, handoff_id, action, body) return @@ -158,23 +158,18 @@ def _handle_create(self, actor: str, body: dict[str, Any]) -> None: response_format=str(body.get("response_format") or DEFAULT_RESPONSE_FORMAT), related_paths=_coerce_string_list(body.get("related_paths", []), field_name="related_paths"), constraints=str(body.get("constraints", "- none") or "- none"), + idempotency_key=str(body.get("idempotency_key") or self.headers.get("Idempotency-Key") or "") or None, + correlation_id=str(body.get("correlation_id") or "") or None, + causation_id=str(body.get("causation_id") or "") or None, ) record = self.server.context.service.create_handoff(request) except RoutePolicyError as exc: raise ApiError(HTTPStatus.FORBIDDEN, "forbidden", str(exc)) from exc + except IdempotencyConflictError as exc: + raise ApiError(HTTPStatus.CONFLICT, "idempotency_conflict", str(exc)) from exc except (BridgePolicyError, ValueError, TypeError) as exc: raise ApiError(HTTPStatus.BAD_REQUEST, "bad_request", str(exc)) from exc - _notify_agent( - self.server.context, - target_agent=record.recipient, - payload={ - "handoff_id": record.handoff_id, - "recipient": record.recipient, - "sender": record.sender, - "subject": record.subject, - "trigger": "handoff_created", - }, - ) + _deliver_pending_for_handoff(self.server.context, record.handoff_id, record.revision) self._send_json(HTTPStatus.CREATED, _record_to_dict(record)) def _handle_list(self, actor: str, *, active_only: bool) -> None: @@ -193,10 +188,15 @@ def _handle_get(self, actor: str, handoff_id: str) -> None: def _handle_action(self, actor: str, handoff_id: str, action: str, body: dict[str, Any]) -> None: try: if action == "archive": - archive_dir = self.server.context.service.archive_handoff(handoff_id, actor=actor) + archive_dir = self.server.context.service.archive_handoff( + handoff_id, + actor=actor, + expected_revision=self._expected_revision(body), + ) record = self.server.context.service.get_handoff(handoff_id, actor=actor) payload = _record_to_dict(record) payload["archive_path"] = str(archive_dir) + _deliver_pending_for_handoff(self.server.context, record.handoff_id, record.revision) self._send_json(HTTPStatus.OK, payload) return if action == "status": @@ -206,6 +206,7 @@ def _handle_action(self, actor: str, handoff_id: str, action: str, body: dict[st "ack": "acknowledged", "block": "blocked", "close": "closed", + "cancel": "cancelled", }[action] outcome = str(body.get("outcome", "") or "") acknowledgment_source = None @@ -217,38 +218,43 @@ def _handle_action(self, actor: str, handoff_id: str, action: str, body: dict[st status=status_name, outcome=outcome, acknowledgment_source=acknowledgment_source, + expected_revision=self._expected_revision(body), ) except HandoffNotFoundError as exc: raise ApiError(HTTPStatus.NOT_FOUND, "not_found", str(exc)) from exc except AccessPolicyError as exc: raise ApiError(HTTPStatus.FORBIDDEN, "forbidden", str(exc)) from exc + except RevisionConflictError as exc: + raise ApiError(HTTPStatus.CONFLICT, "revision_conflict", str(exc)) from exc except StatusPolicyError as exc: raise ApiError(HTTPStatus.BAD_REQUEST, "bad_request", str(exc)) from exc - if action in {"close", "block"}: - _notify_agent( - self.server.context, - target_agent=record.sender, - payload={ - "trigger": f"handoff_{record.status}", - "handoff_id": record.handoff_id, - "sender": record.sender, - "recipient": record.recipient, - "actor": actor, - "status": record.status, - "subject": record.subject, - "resolution_summary": record.resolution_summary, - }, - ) + _deliver_pending_for_handoff(self.server.context, record.handoff_id, record.revision) self._send_json(HTTPStatus.OK, _record_to_dict(record)) + def _expected_revision(self, body: dict[str, Any]) -> int | None: + value: object | None = body.get("expected_revision") + if value is None: + raw = self.headers.get("If-Match", "").strip() + if raw: + value = raw.removeprefix("W/").strip().strip('"') + if value in (None, ""): + return None + try: + return int(str(value)) + except ValueError as exc: + raise ApiError(HTTPStatus.BAD_REQUEST, "bad_request", "expected_revision/If-Match must be an integer") from exc + def _send_json(self, status: HTTPStatus, payload: dict[str, Any]) -> None: data = json.dumps(payload, sort_keys=True).encode("utf-8") - self.send_response(int(status)) - self.send_header("Content-Type", "application/json; charset=utf-8") - self.send_header("Content-Length", str(len(data))) - self.send_header("Cache-Control", "no-store") - self.end_headers() - self.wfile.write(data) + try: + self.send_response(int(status)) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(data) + except (BrokenPipeError, ConnectionResetError): + return class ApiError(Exception): @@ -328,13 +334,19 @@ def _record_from_path(path: Path) -> HandoffRecord: return HandoffRecord.from_mapping(data, body) -def _notify_agent(context: ApiContext, *, target_agent: str, payload: dict[str, Any]) -> None: +def _notify_agent( + context: ApiContext, + *, + target_agent: str, + payload: dict[str, Any], + event_id: str = "", +) -> tuple[bool, str]: notify_url = os.environ.get(env_key_for_agent("BRIDGE_NOTIFY_URL_", target_agent), "").strip() if not notify_url: - return + return False, f"no notify endpoint configured for {target_agent}" token = load_agent_tokens(context.config_path).get(target_agent, "") if not token: - return + return False, f"no token configured for {target_agent}" request = Request( notify_url, method="POST", @@ -343,13 +355,64 @@ def _notify_agent(context: ApiContext, *, target_agent: str, payload: dict[str, "Accept": "application/json", "Authorization": f"Bearer {token}", "Content-Type": "application/json", + **({"X-Bridge-Event-ID": event_id} if event_id else {}), }, ) try: with urlopen(request, timeout=DEFAULT_NOTIFY_TIMEOUT_SECONDS): - return - except Exception: - return + return True, "delivered" + except Exception as exc: + return False, str(exc) + + +def _deliver_pending_for_handoff(context: ApiContext, handoff_id: str, revision: int) -> None: + candidates = [ + event + for event in context.service.list_outbox(states={"pending", "retrying"}) + if event.handoff_id == handoff_id and event.revision == revision and event.due() + ] + for event in candidates: + ok, detail = _notify_agent( + context, + target_agent=event.target_agent, + payload=_legacy_notify_payload(event), + event_id=event.event_id, + ) + if ok: + context.service._mark_event_delivered(event) + else: + context.service._mark_event_failed( + event, + detail, + max_attempts=int(os.environ.get("BRIDGE_OUTBOX_MAX_ATTEMPTS", "8")), + base_delay_seconds=float(os.environ.get("BRIDGE_OUTBOX_BASE_DELAY_SECONDS", "1")), + max_delay_seconds=float(os.environ.get("BRIDGE_OUTBOX_MAX_DELAY_SECONDS", "300")), + ) + + +def _legacy_notify_payload(event) -> dict[str, Any]: + if event.event_type == "handoff_created": + return { + "handoff_id": event.handoff_id, + "recipient": event.recipient, + "sender": event.sender, + "subject": event.subject, + "trigger": event.event_type, + } + if event.event_type in {"handoff_closed", "handoff_blocked"}: + return { + "trigger": event.event_type, + "handoff_id": event.handoff_id, + "sender": event.sender, + "recipient": event.recipient, + "actor": event.actor, + "status": event.status, + "subject": event.subject, + "resolution_summary": event.resolution_summary, + } + payload = event.to_dict() + payload["trigger"] = event.event_type + return payload def build_server( @@ -360,6 +423,7 @@ def build_server( config_path: Path | str | None = DEFAULT_CONFIG_PATH, ) -> BridgeApiServer: repository = FileBridgeRepository(bridge_root=Path(bridge_root)) + repository.recover_transactions() context = ApiContext( bridge_root=repository.bridge_root, config_path=Path(config_path) if config_path is not None else None, diff --git a/scripts/bridge_cli.py b/scripts/bridge_cli.py index 63508d2..19cb9f3 100755 --- a/scripts/bridge_cli.py +++ b/scripts/bridge_cli.py @@ -6,6 +6,7 @@ import os import sys from pathlib import Path +from urllib.request import Request, urlopen SCRIPT_ROOT = Path(__file__).resolve().parents[1] if str(SCRIPT_ROOT) not in sys.path: @@ -16,6 +17,8 @@ from bridge_core.models import ALL_STATUSES, ACTIVE_STATUSES, CreateHandoffInput, HandoffRecord from bridge_core.policy import RoutePolicyError, require_actor_access, require_route, visible_queues_for_actor from bridge_core.repository import HandoffNotFoundError +from bridge_core.auth import load_agent_tokens +from bridge_core.runtime import configured_values, env_key_for_agent from bridge_core.service import BridgeService ROOT = Path(os.environ.get('BRIDGE_PROJECT_ROOT', str(SCRIPT_ROOT))) @@ -88,6 +91,9 @@ def create(args): response_format=args.response_format, related_paths=args.related_path or [], constraints=args.constraints or '- none', + idempotency_key=getattr(args, 'idempotency_key', None), + correlation_id=getattr(args, 'correlation_id', None), + causation_id=getattr(args, 'causation_id', None), ) try: record = service().create_handoff(request) @@ -97,6 +103,7 @@ def create(args): 'handoff_id': record.handoff_id, 'outbox': str(_outbox_path(record.handoff_id, record.sender)), 'inbox': str(_inbox_path(record.handoff_id, record.recipient)), + 'revision': record.revision, }, indent=2)) @@ -124,6 +131,7 @@ def status(args): 'actor': args.actor, 'paths': visible_paths, 'statuses': sorted({stored.record.status for stored in records}), + 'revisions': sorted({stored.record.revision for stored in records}), }, indent=2)) @@ -148,20 +156,115 @@ def list_open(args): def set_status(args): try: - record = service().set_status(args.handoff_id, actor=args.actor, status=args.status, outcome=args.outcome) + record = service().set_status( + args.handoff_id, + actor=args.actor, + status=args.status, + outcome=args.outcome, + expected_revision=getattr(args, 'expected_revision', None), + ) except (HandoffNotFoundError, ValueError) as exc: raise SystemExit(str(exc)) from exc - print(json.dumps({'handoff_id': record.handoff_id, 'actor': args.actor, 'status': record.status}, indent=2)) + print(json.dumps({'handoff_id': record.handoff_id, 'actor': args.actor, 'status': record.status, 'revision': record.revision}, indent=2)) def archive(args): try: - archive_dir = service().archive_handoff(args.handoff_id, actor=args.actor) + archive_dir = service().archive_handoff( + args.handoff_id, + actor=args.actor, + expected_revision=getattr(args, 'expected_revision', None), + ) except (HandoffNotFoundError, ValueError) as exc: raise SystemExit(str(exc)) from exc print(json.dumps({'handoff_id': args.handoff_id, 'actor': args.actor, 'archive_dir': str(archive_dir)}, indent=2)) +def outbox_list(args): + states = set(args.state) if args.state else None + print(json.dumps([event.to_dict() for event in service().list_outbox(states=states)], indent=2, sort_keys=True)) + + +def outbox_replay(args: argparse.Namespace) -> None: + event = service().replay_event(args.event_id) + print(json.dumps(event.to_dict(), indent=2, sort_keys=True)) + + +def outbox_resolve_transaction(args: argparse.Namespace) -> None: + event = service().resolve_transaction( + args.event_id, + action=args.action, + reason=args.reason, + force_recoverable_discard=args.force_recoverable_discard, + ) + print( + json.dumps( + {"event": event.to_dict(), "action": args.action, "reason": args.reason}, + indent=2, + sort_keys=True, + ) + ) + + +def reconcile(args): + repo = repository() + if args.handoff_id: + result = {args.handoff_id: repo.reconcile_handoff(args.handoff_id, repair=not args.check_only)} + else: + ids = {path.stem for path in repo.records_root.glob('*.md')} + ids.update(path.stem for path in BRIDGE.glob('incoming/*/*.md')) + ids.update(path.stem for path in BRIDGE.glob('outgoing/*/*.md')) + result = {handoff_id: repo.reconcile_handoff(handoff_id, repair=not args.check_only) for handoff_id in sorted(ids)} + print(json.dumps(result, indent=2, sort_keys=True)) + + +def recover_outbox(_args): + recovered = repository().recover_transactions() + print(json.dumps({'recovered_event_ids': recovered}, indent=2, sort_keys=True)) + + +def deliver_outbox(args): + repo = repository() + bridge_service = BridgeService(repo) + config_path = Path(os.environ['BRIDGE_API_CONFIG']) if os.environ.get('BRIDGE_API_CONFIG') else ROOT / 'config' / 'bridge_api.env' + os.environ.setdefault('BRIDGE_API_CONFIG', str(config_path)) + values = configured_values() + tokens = load_agent_tokens(config_path) + + def deliver(event): + url = values.get(env_key_for_agent('BRIDGE_NOTIFY_URL_', event.target_agent), '').strip() + token = tokens.get(event.target_agent, '') + if not url: + return False, f'no notify endpoint configured for {event.target_agent}' + if not token: + return False, f'no token configured for {event.target_agent}' + request = Request( + url, + method='POST', + data=json.dumps(event.to_dict(), sort_keys=True).encode('utf-8'), + headers={ + 'Accept': 'application/json', + 'Authorization': f'Bearer {token}', + 'Content-Type': 'application/json', + 'X-Bridge-Event-ID': event.event_id, + }, + ) + try: + with urlopen(request, timeout=args.timeout): + return True, 'delivered' + except Exception as exc: + return False, str(exc) + + processed = bridge_service.process_outbox( + deliver, + limit=args.limit, + max_attempts=args.max_attempts, + base_delay_seconds=args.base_delay_seconds, + max_delay_seconds=args.max_delay_seconds, + ) + print(json.dumps([event.to_dict() for event in processed], indent=2, sort_keys=True)) + + def main(): p = argparse.ArgumentParser(description='Agent bridge helper') sub = p.add_subparsers(dest='cmd', required=True) @@ -182,6 +285,9 @@ def main(): c.add_argument('--related-path', action='append') c.add_argument('--approval-needed', action='store_true') c.add_argument('--approval-context', default='') + c.add_argument('--idempotency-key') + c.add_argument('--correlation-id') + c.add_argument('--causation-id') c.set_defaults(func=create) s = sub.add_parser('status') @@ -198,13 +304,53 @@ def main(): u.add_argument('handoff_id') u.add_argument('status', choices=sorted(ALL_STATUSES)) u.add_argument('--outcome', default='') + u.add_argument('--expected-revision', type=int) u.set_defaults(func=set_status) a = sub.add_parser('archive') a.add_argument('--actor', required=True) a.add_argument('handoff_id') + a.add_argument('--expected-revision', type=int) a.set_defaults(func=archive) + ol = sub.add_parser('outbox-list') + ol.add_argument('--state', action='append', choices=['pending', 'retrying', 'delivered', 'dead_letter']) + ol.set_defaults(func=outbox_list) + + replay_parser = sub.add_parser("outbox-replay", help="Replay an event without changing its event ID") + replay_parser.add_argument("event_id") + replay_parser.set_defaults(func=outbox_replay) + + resolve_parser = sub.add_parser( + "outbox-resolve-transaction", + help="Explicitly enqueue or discard a prepared transaction with audit evidence", + ) + resolve_parser.add_argument("event_id") + resolve_parser.add_argument("--action", choices=("enqueue", "discard"), required=True) + resolve_parser.add_argument("--reason", required=True) + resolve_parser.add_argument( + "--force-recoverable-discard", + action="store_true", + help="Permit discard even when canonical state still matches and automatic recovery is safe", + ) + resolve_parser.set_defaults(func=outbox_resolve_transaction) + + rec = sub.add_parser('reconcile') + rec.add_argument('handoff_id', nargs='?') + rec.add_argument('--check-only', action='store_true') + rec.set_defaults(func=reconcile) + + recover = sub.add_parser('outbox-recover') + recover.set_defaults(func=recover_outbox) + + deliver = sub.add_parser('outbox-deliver') + deliver.add_argument('--limit', type=int, default=100) + deliver.add_argument('--timeout', type=float, default=2.0) + deliver.add_argument('--max-attempts', type=int, default=int(os.environ.get('BRIDGE_OUTBOX_MAX_ATTEMPTS', '8'))) + deliver.add_argument('--base-delay-seconds', type=float, default=float(os.environ.get('BRIDGE_OUTBOX_BASE_DELAY_SECONDS', '1'))) + deliver.add_argument('--max-delay-seconds', type=float, default=float(os.environ.get('BRIDGE_OUTBOX_MAX_DELAY_SECONDS', '300'))) + deliver.set_defaults(func=deliver_outbox) + args = p.parse_args() args.func(args) diff --git a/scripts/bridge_intake_watch.py b/scripts/bridge_intake_watch.py index 5b212fb..bc43ab4 100755 --- a/scripts/bridge_intake_watch.py +++ b/scripts/bridge_intake_watch.py @@ -130,12 +130,18 @@ def _read_json_body(self) -> dict[str, Any]: def _send_json(self, status: HTTPStatus, payload: dict[str, Any]) -> None: data = json.dumps(payload, sort_keys=True).encode('utf-8') - self.send_response(int(status)) - self.send_header('Content-Type', 'application/json; charset=utf-8') - self.send_header('Content-Length', str(len(data))) - self.send_header('Cache-Control', 'no-store') - self.end_headers() - self.wfile.write(data) + try: + self.send_response(int(status)) + self.send_header('Content-Type', 'application/json; charset=utf-8') + self.send_header('Content-Length', str(len(data))) + self.send_header('Cache-Control', 'no-store') + self.end_headers() + self.wfile.write(data) + except (BrokenPipeError, ConnectionResetError): + # The Bridge API posts notify events with a short timeout and does not + # need the response body after the event command has completed. Avoid + # noisy tracebacks when the client closes first. + return def build_notify_server( diff --git a/scripts/bridge_patrol.py b/scripts/bridge_patrol.py index 02874f9..03e2364 100755 --- a/scripts/bridge_patrol.py +++ b/scripts/bridge_patrol.py @@ -16,6 +16,7 @@ sys.path.insert(0, str(SCRIPT_ROOT)) from bridge_core.models import ACTIVE_STATUSES +from bridge_core.file_repository import FileBridgeRepository from bridge_core.policy import allowed_routes from bridge_core.runtime import discover_agents, env_key_for_agent from bridge_core.tooling import summarize_handoffs @@ -293,6 +294,7 @@ def main(): ap.add_argument('--reminder-repeat-hours', type=float, default=_float_setting('BRIDGE_PATROL_REMINDER_REPEAT_HOURS', DEFAULT_REMINDER_REPEAT_HOURS)) ap.add_argument('--escalate-after-hours', type=float, default=_float_setting('BRIDGE_PATROL_ESCALATE_AFTER_HOURS', DEFAULT_ESCALATE_AFTER_HOURS)) ap.add_argument('--escalate-repeat-hours', type=float, default=_float_setting('BRIDGE_PATROL_ESCALATE_REPEAT_HOURS', DEFAULT_ESCALATE_REPEAT_HOURS)) + ap.add_argument('--outbox-pending-seconds', type=float, default=_float_setting('BRIDGE_PATROL_OUTBOX_PENDING_SECONDS', 300.0)) args = ap.parse_args() now = datetime.now(timezone.utc) @@ -402,6 +404,12 @@ def main(): warnings.append(escalation_warning) state_changed = True + repository_diagnostics = FileBridgeRepository(BRIDGE, initialize=False).diagnose( + pending_age_seconds=max(args.outbox_pending_seconds, 0.0) + ) + warnings.extend(repository_diagnostics) + evidence.append(f'bridge v2 diagnostics issues={len(repository_diagnostics)}') + active_ids = {item.record.handoff_id for item in summarized if item.record.status in ACTIVE} stale_state_keys = [handoff_id for handoff_id in reminder_state if handoff_id not in active_ids] for handoff_id in stale_state_keys: @@ -437,7 +445,7 @@ def main(): print(f'- ... ({len(checks) - 12} more checks)') print('actions_taken:') print('- scanned bridge directories, markdown files, and incoming handoffs') - print('- checked route policy, Bridge API health, service state, file permission posture, active handoff aging, and unresolved-handoff reminders') + print('- checked route policy, Bridge API health, service state, file permission posture, mirrors/revisions, outbox, idempotency indexes, active handoff aging, and unresolved-handoff reminders') print('evidence:') for item in evidence: print(f'- {item}') diff --git a/scripts/bridge_rotate_tokens.py b/scripts/bridge_rotate_tokens.py index eaa7bea..a7d4339 100755 --- a/scripts/bridge_rotate_tokens.py +++ b/scripts/bridge_rotate_tokens.py @@ -6,9 +6,14 @@ import secrets import shutil import subprocess +import sys from datetime import datetime, timezone from pathlib import Path +SCRIPT_ROOT = Path(__file__).resolve().parents[1] +if str(SCRIPT_ROOT) not in sys.path: + sys.path.insert(0, str(SCRIPT_ROOT)) + from bridge_core.runtime import TOKEN_ENV_PREFIX, normalize_agent_env_suffix DEFAULT_ROOT = Path(__file__).resolve().parents[1] diff --git a/scripts/bridge_wrapper_common.py b/scripts/bridge_wrapper_common.py index 766d197..8f7f6e5 100755 --- a/scripts/bridge_wrapper_common.py +++ b/scripts/bridge_wrapper_common.py @@ -109,6 +109,7 @@ def create_result(payload: dict[str, Any]) -> dict[str, Any]: 'handoff_id': handoff_id, 'outbox': str(root / 'outgoing' / sender / f'{handoff_id}.md'), 'inbox': str(root / 'incoming' / recipient / f'{handoff_id}.md'), + 'revision': int(payload.get('revision', 0)), } @@ -144,6 +145,7 @@ def status_result(agent: str, handoff_id: str, payload: dict[str, Any]) -> dict[ 'actor': agent, 'paths': paths, 'statuses': sorted({str(payload.get('status', ''))}), + 'revisions': [int(payload.get('revision', 0))], } @@ -152,6 +154,7 @@ def set_status_result(agent: str, payload: dict[str, Any]) -> dict[str, Any]: 'handoff_id': str(payload['handoff_id']), 'actor': agent, 'status': str(payload['status']), + 'revision': int(payload.get('revision', 0)), } diff --git a/tests/test_bridge_api.py b/tests/test_bridge_api.py index 2d1ba92..8b18708 100644 --- a/tests/test_bridge_api.py +++ b/tests/test_bridge_api.py @@ -9,7 +9,10 @@ import pytest +from bridge_core.file_repository import FileBridgeRepository from bridge_core.frontmatter import parse_frontmatter +from bridge_core.models import CreateHandoffInput +from bridge_core.service import BridgeService import bridge_api_server import bridge_intake_watch @@ -292,6 +295,41 @@ def test_create_keeps_open_handoff_when_immediate_notify_fails(api_server, monke assert handoff["status"] == "open" +def test_immediate_delivery_updates_loaded_event_without_rescanning(tmp_path: Path, monkeypatch) -> None: + repository = FileBridgeRepository(tmp_path / "bridge") + service = BridgeService(repository) + created = service.create_handoff( + CreateHandoffInput( + sender="agent-a", + recipient="agent-c", + issue_type="task", + subject="Single outbox scan", + requested_action="Deliver once.", + minimal_context="Regression coverage for API-triggered delivery.", + ) + ) + context = bridge_api_server.ApiContext( + bridge_root=repository.bridge_root, + config_path=None, + service=service, + ) + original_list_events = repository.list_events + scans = 0 + + def counted_list_events(*, states=None): + nonlocal scans + scans += 1 + return original_list_events(states=states) + + monkeypatch.setattr(repository, "list_events", counted_list_events) + monkeypatch.setattr(bridge_api_server, "_notify_agent", lambda *args, **kwargs: (True, "delivered")) + + bridge_api_server._deliver_pending_for_handoff(context, created.handoff_id, created.revision) + + assert scans == 1 + assert original_list_events()[0].delivery_state == "delivered" + + def test_close_notifies_sender_with_lifecycle_payload(api_server, monkeypatch) -> None: captured_requests: list[dict[str, object]] = [] @@ -467,3 +505,115 @@ def _failing_urlopen(request, timeout=0): assert status == 200 assert fetched["status"] == "closed" assert fetched["resolution_summary"] == "Completed despite notify failure." + + +def test_api_idempotency_conflict_returns_409(api_server) -> None: + body: dict[str, object] = { + "recipient": "agent-c", + "issue_type": "task", + "subject": "Idempotent API create", + "requested_action": "Create once.", + "minimal_context": "Concurrent-safe create.", + } + first_status, first = _request( + api_server["base_url"], + "POST", + "/v1/handoffs", + token="token-agent-a", + body=body, + extra_headers={"Idempotency-Key": "api-create-1"}, + ) + duplicate_status, duplicate = _request( + api_server["base_url"], + "POST", + "/v1/handoffs", + token="token-agent-a", + body=body, + extra_headers={"Idempotency-Key": "api-create-1"}, + ) + conflict_status, conflict = _request( + api_server["base_url"], + "POST", + "/v1/handoffs", + token="token-agent-a", + body={**body, "subject": "Different payload"}, + extra_headers={"Idempotency-Key": "api-create-1"}, + ) + + assert first_status == duplicate_status == 201 + assert duplicate["handoff_id"] == first["handoff_id"] + assert conflict_status == 409 + assert conflict["error"] == "idempotency_conflict" + + +def test_api_if_match_stale_revision_returns_409(api_server) -> None: + _status, created = _request( + api_server["base_url"], + "POST", + "/v1/handoffs", + token="token-agent-a", + body={ + "recipient": "agent-c", + "issue_type": "task", + "subject": "Revision API", + "requested_action": "Exercise optimistic concurrency.", + "minimal_context": "If-Match test.", + }, + ) + handoff_id = str(created["handoff_id"]) + ack_status, acked = _request( + api_server["base_url"], + "POST", + f"/v1/handoffs/{handoff_id}/ack", + token="token-agent-c", + extra_headers={"If-Match": '"0"'}, + ) + stale_status, stale = _request( + api_server["base_url"], + "POST", + f"/v1/handoffs/{handoff_id}/block", + token="token-agent-c", + body={"outcome": "Stale update."}, + extra_headers={"If-Match": '"0"'}, + ) + + assert ack_status == 200 + assert acked["revision"] == 1 + assert stale_status == 409 + assert stale["error"] == "revision_conflict" + + +def test_api_role_authorization_rejects_sender_close_and_allows_cancel(api_server) -> None: + _status, created = _request( + api_server["base_url"], + "POST", + "/v1/handoffs", + token="token-agent-a", + body={ + "recipient": "agent-c", + "issue_type": "task", + "subject": "Role matrix API", + "requested_action": "Prove role-specific lifecycle rules.", + "minimal_context": "Sender may cancel, not close.", + }, + ) + handoff_id = str(created["handoff_id"]) + close_status, close_error = _request( + api_server["base_url"], + "POST", + f"/v1/handoffs/{handoff_id}/close", + token="token-agent-a", + body={"outcome": "Unauthorized close."}, + ) + cancel_status, cancelled = _request( + api_server["base_url"], + "POST", + f"/v1/handoffs/{handoff_id}/cancel", + token="token-agent-a", + body={"outcome": "Withdrawn safely."}, + ) + + assert close_status == 403 + assert close_error["error"] == "forbidden" + assert cancel_status == 200 + assert cancelled["status"] == "cancelled" diff --git a/tests/test_bridge_cli.py b/tests/test_bridge_cli.py index 6f39ed1..2200301 100644 --- a/tests/test_bridge_cli.py +++ b/tests/test_bridge_cli.py @@ -114,10 +114,28 @@ def test_archive_requires_closed_status(bridge_sandbox, capsys): assert outgoing.exists() assert incoming.exists() - with pytest.raises(SystemExit, match='only closed handoffs can be archived'): + with pytest.raises(SystemExit, match='only closed or cancelled handoffs can be archived'): bridge_cli.archive(Namespace(actor='agent-c', handoff_id=handoff_id)) +def test_recipient_can_archive_sender_cancelled_handoff(bridge_sandbox, capsys): + handoff_id = _create_handoff(capsys) + bridge_cli.set_status( + Namespace( + actor='agent-a', + handoff_id=handoff_id, + status='cancelled', + outcome='Request withdrawn by sender.', + ) + ) + capsys.readouterr() + + bridge_cli.archive(Namespace(actor='agent-c', handoff_id=handoff_id)) + archive_result = json.loads(capsys.readouterr().out) + + assert Path(archive_result['archive_dir']).exists() + + def test_relocated_bridge_cli_uses_repo_relative_root(tmp_path): source_root = Path(__file__).resolve().parents[1] relocated_root = tmp_path / 'bridge-copy' diff --git a/tests/test_bridge_intake_watch.py b/tests/test_bridge_intake_watch.py index 16754a6..ee97bbd 100644 --- a/tests/test_bridge_intake_watch.py +++ b/tests/test_bridge_intake_watch.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from http import HTTPStatus from pathlib import Path import sys import threading @@ -323,6 +324,25 @@ def test_intake_once_dry_run_reports_without_mutating(api_server, monkeypatch) - assert payload['status'] == 'open' +def test_notify_send_json_ignores_client_disconnect() -> None: + calls: list[tuple[object, ...]] = [] + + class BrokenPipeWriter: + def write(self, _data: bytes) -> None: + raise BrokenPipeError('client closed') + + handler = object.__new__(bridge_intake_watch.IntakeNotifyHandler) + handler.wfile = BrokenPipeWriter() + handler.send_response = lambda status: calls.append(('response', status)) + handler.send_header = lambda name, value: calls.append(('header', name, value)) + handler.end_headers = lambda: calls.append(('end_headers', None)) + + handler._send_json(HTTPStatus.OK, {'ok': True}) + + assert calls[0] == ('response', 200) + assert calls[-1] == ('end_headers', None) + + def test_notify_server_acknowledges_open_handoff(api_server, monkeypatch) -> None: _configure_watch_env(monkeypatch, api_server['bridge_root'], api_server['base_url']) notify_server = bridge_intake_watch.build_notify_server(agent='agent-c', host='127.0.0.1', port=0) diff --git a/tests/test_bridge_patrol.py b/tests/test_bridge_patrol.py index 1c80985..4b3cdc1 100644 --- a/tests/test_bridge_patrol.py +++ b/tests/test_bridge_patrol.py @@ -9,7 +9,9 @@ import bridge_patrol from bridge_core.frontmatter import dump_document -from bridge_core.models import HandoffRecord +from bridge_core.models import CreateHandoffInput, HandoffRecord +from bridge_core.file_repository import FileBridgeRepository +from bridge_core.service import BridgeService class _NotifyCaptureServer(ThreadingHTTPServer): @@ -129,8 +131,12 @@ def _write_handoff( body='## Requested Action\nAck it\n\n## Minimal Context\nTest\n\n## Constraints\n- none\n\n## Outcome\n- pending\n', ) path = bridge / 'incoming' / recipient / f'{handoff_id}.md' - path.write_text(dump_document(record.to_frontmatter(), record.body), encoding='utf-8') + document = dump_document(record.to_frontmatter(), record.body) + path.write_text(document, encoding='utf-8') path.chmod(0o600) + outgoing = bridge / 'outgoing' / record.sender / f'{handoff_id}.md' + outgoing.write_text(document, encoding='utf-8') + outgoing.chmod(0o600) return path @@ -321,3 +327,39 @@ def test_patrol_skips_active_alert_when_summary_is_actionable(tmp_path: Path, mo assert 'active unresolved handoff for agent-c: HND-TEST-0006' not in out assert 'status: ok' in out + + +def test_patrol_detects_v2_mirror_outbox_and_idempotency_drift(tmp_path: Path, monkeypatch, capsys) -> None: + _root, bridge, _config = _setup_patrol_root(tmp_path, monkeypatch) + repository = FileBridgeRepository(bridge) + service = BridgeService(repository) + record = service.create_handoff( + CreateHandoffInput( + sender='agent-a', + recipient='agent-c', + issue_type='task', + subject='V2 patrol drift', + requested_action='Detect all drift.', + minimal_context='Patrol regression.', + idempotency_key='patrol-v2-1', + ) + ) + (bridge / 'incoming' / 'agent-c' / f'{record.handoff_id}.md').unlink() + event = service.list_outbox()[0] + service.mark_event_failed(event.event_id, 'consumer down', max_attempts=1) + index_path = next(repository.idempotency_root.glob('*/*.json')) + index = json.loads(index_path.read_text(encoding='utf-8')) + index['request_hash'] = 'inconsistent' + index_path.write_text(json.dumps(index), encoding='utf-8') + index_path.chmod(0o600) + + monkeypatch.setattr(bridge_patrol, 'check_systemd_service', lambda unit: ('ok', f'{unit} active')) + monkeypatch.setattr(bridge_patrol, 'check_api_health', lambda url: ('ok', f'bridge API health ok at {url}')) + monkeypatch.setattr('sys.argv', ['bridge_patrol.py', '--reminder-after-hours', '99', '--escalate-after-hours', '99']) + bridge_patrol.main() + out = capsys.readouterr().out + + assert 'status: warning' in out + assert 'missing mirror incoming/agent-c/' in out + assert 'dead-letter event' in out + assert 'idempotency index inconsistency' in out diff --git a/tests/test_bridge_service.py b/tests/test_bridge_service.py index ef1843a..2729947 100644 --- a/tests/test_bridge_service.py +++ b/tests/test_bridge_service.py @@ -248,5 +248,5 @@ def test_archive_requires_closed_status(tmp_path: Path) -> None: ) ) - with pytest.raises(StatusPolicyError, match="only closed handoffs can be archived"): + with pytest.raises(StatusPolicyError, match="only closed or cancelled handoffs can be archived"): service.archive_handoff(handoff.handoff_id, actor="agent-b") diff --git a/tests/test_bridge_v2_reliability.py b/tests/test_bridge_v2_reliability.py new file mode 100644 index 0000000..11dc8a4 --- /dev/null +++ b/tests/test_bridge_v2_reliability.py @@ -0,0 +1,491 @@ +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import json + +import pytest + +from bridge_core.file_repository import FileBridgeRepository +from bridge_core.frontmatter import dump_document, parse_frontmatter +from bridge_core.models import CreateHandoffInput, HandoffRecord +from bridge_core.policy import AccessPolicyError +from bridge_core.repository import IdempotencyConflictError, RepositoryConflictError, RevisionConflictError +from bridge_core.service import BridgeService + + +def _request(*, key: str = "create-1", subject: str = "Bridge v2") -> CreateHandoffInput: + return CreateHandoffInput( + sender="agent-a", + recipient="agent-b", + issue_type="task", + subject=subject, + requested_action="Perform the bounded work.", + minimal_context="Reliability regression test.", + idempotency_key=key, + correlation_id="CORR-1", + causation_id="CAUSE-1", + ) + + +def _service(tmp_path: Path, **repo_kwargs) -> tuple[BridgeService, FileBridgeRepository]: + repository = FileBridgeRepository(tmp_path / "bridge", **repo_kwargs) + return BridgeService(repository), repository + + +def _write_v1_handoff(root: Path, *, handoff_id: str = "HND-V1-0001") -> HandoffRecord: + record = HandoffRecord( + handoff_id=handoff_id, + status="open", + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + acknowledged_at="none", + acknowledgment_source="none", + sender="agent-a", + recipient="agent-b", + issue_type="task", + subject="Legacy handoff", + body="## Outcome\n- pending\n", + ) + frontmatter = record.to_frontmatter() + for key in ( + "schema_version", + "revision", + "correlation_id", + "causation_id", + "canonical_request_hash", + "idempotency_key_hash", + ): + frontmatter.pop(key) + document = dump_document(frontmatter, record.body) + for path in ( + root / "outgoing" / "agent-a" / f"{handoff_id}.md", + root / "incoming" / "agent-b" / f"{handoff_id}.md", + ): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(document, encoding="utf-8") + return record + + +def test_concurrent_identical_create_requests_produce_one_handoff(tmp_path: Path) -> None: + service, repository = _service(tmp_path) + + with ThreadPoolExecutor(max_workers=8) as executor: + records = list(executor.map(lambda _index: service.create_handoff(_request()), range(16))) + + assert len({record.handoff_id for record in records}) == 1 + assert len(list(repository.records_root.glob("*.md"))) == 1 + assert len(repository.list_events()) == 1 + + +def test_same_idempotency_key_with_different_payload_conflicts(tmp_path: Path) -> None: + service, repository = _service(tmp_path) + service.create_handoff(_request()) + + with pytest.raises(IdempotencyConflictError, match="different request payload"): + service.create_handoff(_request(subject="Materially different")) + assert list(repository.transactions_root.glob("*.json")) == [] + + +def test_create_conflict_removes_unrecoverable_prepared_transaction(tmp_path: Path, monkeypatch) -> None: + service, repository = _service(tmp_path) + request = _request() + request.idempotency_key = None + existing = HandoffRecord.from_create(request) + repository.create(existing) + monkeypatch.setattr(HandoffRecord, "from_create", lambda _request: existing) + + with pytest.raises(RepositoryConflictError, match="handoff already exists"): + service.create_handoff(request) + + assert list(repository.transactions_root.glob("*.json")) == [] + + +def test_crash_after_first_mirror_write_is_recoverable(tmp_path: Path) -> None: + def fail_after_outgoing(stage: str) -> None: + if stage == "after_outgoing_write": + raise RuntimeError("simulated crash") + + root = tmp_path / "bridge" + crashing_service = BridgeService(FileBridgeRepository(root, fault_injector=fail_after_outgoing)) + with pytest.raises(RuntimeError, match="simulated crash"): + crashing_service.create_handoff(_request()) + + repository = FileBridgeRepository(root) + recovered = BridgeService(repository).create_handoff(_request()) + outgoing = root / "outgoing" / "agent-a" / f"{recovered.handoff_id}.md" + incoming = root / "incoming" / "agent-b" / f"{recovered.handoff_id}.md" + assert outgoing.exists() and incoming.exists() + recovered_event_ids = repository.recover_transactions() + assert len(recovered_event_ids) == 1 + assert repository.list_events()[0].event_id == recovered_event_ids[0] + + +def test_retry_rebuilds_missing_idempotency_index_after_crash(tmp_path: Path, monkeypatch) -> None: + service, repository = _service(tmp_path) + original_atomic_write_json = repository._atomic_write_json + failed = False + + def fail_first_index_write(path, payload): + nonlocal failed + if repository.idempotency_root in path.parents and not failed: + failed = True + raise RuntimeError("simulated index-write crash") + return original_atomic_write_json(path, payload) + + monkeypatch.setattr(repository, "_atomic_write_json", fail_first_index_write) + with pytest.raises(RuntimeError, match="simulated index-write crash"): + service.create_handoff(_request()) + + orphan_id = next(repository.records_root.glob("*.md")).stem + assert list(repository.idempotency_root.glob("*/*.json")) == [] + monkeypatch.setattr(repository, "_atomic_write_json", original_atomic_write_json) + + recovered = service.create_handoff(_request()) + + assert recovered.handoff_id == orphan_id + assert len(list(repository.records_root.glob("*.md"))) == 1 + assert len(list(repository.idempotency_root.glob("*/*.json"))) == 1 + assert len(list(repository.transactions_root.glob("*.json"))) == 1 + recovered_event_ids = repository.recover_transactions() + assert len(recovered_event_ids) == 1 + assert repository.list_events()[0].handoff_id == orphan_id + + +def test_recovery_does_not_enqueue_stale_prepared_event(tmp_path: Path) -> None: + service, repository = _service(tmp_path) + created = service.create_handoff(_request()) + created_event = service.list_outbox()[0] + repository.complete_event(created_event.event_id) + event_path = repository._event_path(created_event.event_id) + event_path.unlink() + repository.prepare_event(created_event) + + service.set_status(created.handoff_id, actor="agent-b", status="acknowledged", expected_revision=0) + + assert repository.recover_transactions() == [] + assert not event_path.exists() + assert (repository.transactions_root / f"{created_event.event_id}.json").exists() + + +def test_stale_transaction_can_be_explicitly_enqueued_with_resolution_evidence(tmp_path: Path) -> None: + service, repository = _service(tmp_path) + created = service.create_handoff(_request()) + created_event = service.list_outbox()[0] + repository.complete_event(created_event.event_id) + event_path = repository._event_path(created_event.event_id) + event_path.unlink() + repository.prepare_event(created_event) + service.set_status(created.handoff_id, actor="agent-b", status="acknowledged", expected_revision=0) + + resolved = service.resolve_transaction( + created_event.event_id, + action="enqueue", + reason="Verified missing historical create delivery after canonical state advanced.", + ) + + assert resolved.event_id == created_event.event_id + assert event_path.exists() + assert not (repository.transactions_root / f"{created_event.event_id}.json").exists() + evidence = repository.resolved_transactions_root / f"{created_event.event_id}.json" + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert payload["action"] == "enqueue" + assert payload["reason"].startswith("Verified missing historical") + + +def test_transaction_resolution_rejects_path_traversal_event_id(tmp_path: Path) -> None: + _service_instance, repository = _service(tmp_path) + + with pytest.raises(ValueError, match="invalid event_id"): + repository.resolve_transaction("../outside", action="discard", reason="invalid input") + with pytest.raises(ValueError, match="invalid event_id"): + repository.resolve_transaction("EVT-valid\nforged", action="discard", reason="invalid input") + + +def test_transaction_resolution_rejects_payload_event_id_mismatch(tmp_path: Path) -> None: + service, repository = _service(tmp_path) + service.create_handoff(_request()) + event = service.list_outbox()[0] + repository._event_path(event.event_id).unlink() + repository.prepare_event(event) + transaction_path = repository._transaction_path(event.event_id) + payload = json.loads(transaction_path.read_text(encoding="utf-8")) + payload["event"]["event_id"] = "EVT-substituted" + transaction_path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(RepositoryConflictError, match="event_id mismatch"): + service.resolve_transaction(event.event_id, action="enqueue", reason="must fail closed") + with pytest.raises(RepositoryConflictError, match="event_id mismatch"): + repository.recover_transactions() + + assert transaction_path.exists() + assert not repository._event_path("EVT-substituted").exists() + assert any("event_id mismatch" in issue for issue in repository.diagnose()) + + +def test_prepared_transaction_can_be_discarded_with_resolution_evidence(tmp_path: Path) -> None: + service, repository = _service(tmp_path) + service.create_handoff(_request()) + event = service.list_outbox()[0] + repository._event_path(event.event_id).unlink() + repository.prepare_event(event) + + with pytest.raises(ValueError, match="still recoverable"): + service.resolve_transaction( + event.event_id, + action="discard", + reason="Must not discard a recoverable event by default.", + ) + assert repository._transaction_path(event.event_id).exists() + + resolved = service.resolve_transaction( + event.event_id, + action="discard", + reason="Verified duplicate consumer delivery; no replay required.\n- not a second audit entry", + force_recoverable_discard=True, + ) + + assert resolved.event_id == event.event_id + assert not repository._event_path(event.event_id).exists() + evidence = json.loads(repository._resolved_transaction_path(event.event_id).read_text(encoding="utf-8")) + assert evidence["action"] == "discard" + assert evidence["canonical_match"] is True + assert evidence["force_recoverable_discard"] is True + assert len(repository.audit_file.read_text(encoding="utf-8").splitlines()) == 2 + + +def test_stale_delivery_failure_cannot_regress_delivered_event(tmp_path: Path) -> None: + service, repository = _service(tmp_path) + service.create_handoff(_request()) + event = service.list_outbox()[0] + stale_copy = type(event).from_mapping(event.to_dict()) + + delivered = service.mark_event_delivered(event.event_id) + stale_result = service._mark_event_failed( + stale_copy, + "late failure from concurrent deliverer", + max_attempts=3, + base_delay_seconds=0, + ) + + assert delivered.delivery_state == "delivered" + assert stale_result.delivery_state == "delivered" + assert repository.list_events()[0].delivery_state == "delivered" + + +def test_process_outbox_does_not_rescan_events_for_each_delivery(tmp_path: Path, monkeypatch) -> None: + service, repository = _service(tmp_path) + for index in range(4): + service.create_handoff(_request(key=f"batch-{index}", subject=f"Batch {index}")) + + original_list_events = repository.list_events + calls = 0 + + def counted_list_events(*, states=None): + nonlocal calls + calls += 1 + return original_list_events(states=states) + + monkeypatch.setattr(repository, "list_events", counted_list_events) + processed = service.process_outbox(lambda _event: (True, "ok")) + + assert len(processed) == 4 + assert calls == 1 + + +def test_status_update_leaves_queue_views_at_same_revision(tmp_path: Path) -> None: + service, repository = _service(tmp_path) + created = service.create_handoff(_request()) + updated = service.set_status( + created.handoff_id, + actor="agent-b", + status="acknowledged", + expected_revision=0, + ) + + records = repository.load_records(created.handoff_id) + assert updated.revision == 1 + assert {stored.record.revision for stored in records} == {1} + assert {stored.record.status for stored in records} == {"acknowledged"} + + +def test_concurrent_lifecycle_updates_have_one_revision_conflict(tmp_path: Path) -> None: + service, _repository = _service(tmp_path) + created = service.create_handoff(_request()) + + def update(status: str): + return service.set_status( + created.handoff_id, + actor="agent-b", + status=status, + outcome="Waiting." if status == "blocked" else "", + expected_revision=0, + ) + + outcomes: list[object] = [] + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(update, status) for status in ("acknowledged", "blocked")] + for future in futures: + try: + outcomes.append(future.result()) + except Exception as exc: # asserted below + outcomes.append(exc) + + assert sum(isinstance(item, HandoffRecord) for item in outcomes) == 1 + assert sum(isinstance(item, RevisionConflictError) for item in outcomes) == 1 + + +def test_consumer_outage_keeps_event_and_recovery_reuses_event_id(tmp_path: Path) -> None: + service, _repository = _service(tmp_path) + created = service.create_handoff(_request()) + event = service.list_outbox(states={"pending"})[0] + + failed = service.process_outbox( + lambda _event: (False, "consumer unavailable"), + max_attempts=3, + base_delay_seconds=0, + max_delay_seconds=0, + )[0] + assert failed.event_id == event.event_id + assert failed.delivery_state == "retrying" + + delivered_ids: list[str] = [] + processed = service.process_outbox( + lambda retry: (delivered_ids.append(retry.event_id) or True, "ok"), + ) + assert processed[0].delivery_state == "delivered" + assert delivered_ids == [event.event_id] + assert created.handoff_id == processed[0].handoff_id + + +def test_replay_preserves_event_id_and_supports_consumer_deduplication(tmp_path: Path) -> None: + service, _repository = _service(tmp_path) + service.create_handoff(_request()) + event = service.list_outbox()[0] + service.mark_event_delivered(event.event_id) + + replayed = service.replay_event(event.event_id) + assert replayed.event_id == event.event_id + seen: set[str] = {event.event_id} + duplicate_effects: list[str] = [] + + def deduplicating_consumer(item): + if item.event_id not in seen: + duplicate_effects.append(item.event_id) + seen.add(item.event_id) + return True, "deduplicated" + + service.process_outbox(deduplicating_consumer) + assert duplicate_effects == [] + + +def test_exhausted_retries_dead_letter_and_manual_replay(tmp_path: Path) -> None: + service, _repository = _service(tmp_path) + service.create_handoff(_request()) + event = service.list_outbox()[0] + + first = service.mark_event_failed(event.event_id, "down", max_attempts=2, base_delay_seconds=0, jitter_ratio=0) + assert first.delivery_state == "retrying" + second = service.mark_event_failed(event.event_id, "still down", max_attempts=2, base_delay_seconds=0, jitter_ratio=0) + assert second.delivery_state == "dead_letter" + replayed = service.replay_event(event.event_id) + assert replayed.delivery_state == "pending" + assert replayed.attempts == 0 + assert replayed.event_id == event.event_id + + +def test_role_specific_lifecycle_actions_fail_closed(tmp_path: Path) -> None: + service, _repository = _service(tmp_path) + created = service.create_handoff(_request()) + + with pytest.raises(AccessPolicyError, match="not authorized"): + service.set_status(created.handoff_id, actor="agent-a", status="closed", outcome="Not sender-owned.") + with pytest.raises(AccessPolicyError, match="not authorized"): + service.set_status(created.handoff_id, actor="agent-b", status="cancelled", outcome="Not recipient-owned.") + + cancelled = service.cancel_handoff(created.handoff_id, actor="agent-a", reason="Request withdrawn.") + assert cancelled.status == "cancelled" + + +def test_existing_v1_handoff_loads_and_transitions_to_v2(tmp_path: Path) -> None: + root = tmp_path / "bridge" + repository = FileBridgeRepository(root) + _write_v1_handoff(root) + + loaded = BridgeService(repository).get_handoff("HND-V1-0001") + assert loaded.schema_version == 1 + assert loaded.revision == 0 + transitioned = BridgeService(repository).set_status( + "HND-V1-0001", + actor="agent-b", + status="acknowledged", + expected_revision=0, + ) + assert transitioned.schema_version == 2 + assert transitioned.revision == 1 + assert repository._canonical_path("HND-V1-0001").exists() + + +def test_identical_v1_mirrors_are_not_divergence_and_migrate_without_conflicts(tmp_path: Path) -> None: + root = tmp_path / "bridge" + repository = FileBridgeRepository(root) + _write_v1_handoff(root) + + assert repository.reconcile_handoff("HND-V1-0001", repair=False) == [] + + issues = repository.reconcile_handoff("HND-V1-0001", repair=True) + canonical = repository._canonical_path("HND-V1-0001") + outgoing = root / "outgoing" / "agent-a" / "HND-V1-0001.md" + incoming = root / "incoming" / "agent-b" / "HND-V1-0001.md" + + assert issues == ["HND-V1-0001: missing canonical record"] + assert canonical.exists() + assert outgoing.read_bytes() == canonical.read_bytes() + assert incoming.read_bytes() == canonical.read_bytes() + assert list(repository.conflicts_root.glob("HND-V1-0001.*")) == [] + + +def test_repository_diagnostics_detect_new_failure_states(tmp_path: Path) -> None: + service, repository = _service(tmp_path) + created = service.create_handoff(_request()) + incoming = repository.bridge_root / "incoming" / "agent-b" / f"{created.handoff_id}.md" + incoming.unlink() + + event = service.list_outbox()[0] + service.mark_event_failed(event.event_id, "down", max_attempts=1) + + index_path = next(repository.idempotency_root.glob("*/*.json")) + index = json.loads(index_path.read_text(encoding="utf-8")) + index["request_hash"] = "bad-hash" + index_path.write_text(json.dumps(index), encoding="utf-8") + + stale = service.create_handoff(_request(key="create-stale", subject="Stale mirror")) + stale_outgoing = repository.bridge_root / "outgoing" / "agent-a" / f"{stale.handoff_id}.md" + stale_data, stale_body = parse_frontmatter(stale_outgoing.read_text(encoding="utf-8")) + stale_data["revision"] = "99" + stale_outgoing.write_text(dump_document(stale_data, stale_body), encoding="utf-8") + + divergent = service.create_handoff(_request(key="create-divergent", subject="Divergent mirror")) + divergent_outgoing = repository.bridge_root / "outgoing" / "agent-a" / f"{divergent.handoff_id}.md" + divergent_outgoing.write_text(divergent_outgoing.read_text(encoding="utf-8") + "\nlocal divergence\n", encoding="utf-8") + + issues = repository.diagnose(pending_age_seconds=0) + assert any("missing mirror" in issue for issue in issues) + assert any("stale revision" in issue for issue in issues) + assert any("divergent mirror" in issue for issue in issues) + assert any("pending outbox event beyond threshold" in issue for issue in issues) + assert any("dead-letter event" in issue for issue in issues) + assert any("idempotency index inconsistency" in issue for issue in issues) + + +def test_reconcile_preserves_divergent_evidence_before_repair(tmp_path: Path) -> None: + service, repository = _service(tmp_path) + created = service.create_handoff(_request()) + incoming = repository.bridge_root / "incoming" / "agent-b" / f"{created.handoff_id}.md" + incoming.write_text(incoming.read_text(encoding="utf-8") + "\nlocal note\n", encoding="utf-8") + + issues = repository.reconcile_handoff(created.handoff_id, repair=True) + assert any("divergent mirror" in issue for issue in issues) + assert list(repository.conflicts_root.glob(f"{created.handoff_id}.*.incoming.md")) + canonical = repository._canonical_path(created.handoff_id) + assert incoming.read_bytes() == canonical.read_bytes() diff --git a/tests/test_bridge_wrappers.py b/tests/test_bridge_wrappers.py index f10b1d3..117fee0 100644 --- a/tests/test_bridge_wrappers.py +++ b/tests/test_bridge_wrappers.py @@ -43,6 +43,7 @@ def fake_invoke(agent, args, *, api_call, cli_args): '--subject', 'API wrapper test', '--requested-action', 'Do the thing', '--minimal-context', 'Minimal context', + '--idempotency-key', 'wrapper-create-1', ], ) @@ -55,8 +56,10 @@ def fake_invoke(agent, args, *, api_call, cli_args): assert captured['path'] == '/v1/handoffs' assert 'sender' not in captured['payload'] assert captured['payload']['recipient'] == 'agent-b' + assert captured['payload']['idempotency_key'] == 'wrapper-create-1' assert '--sender' in captured['cli_args'] assert captured['cli_args'][captured['cli_args'].index('--sender') + 1] == 'agent-a' + assert captured['cli_args'][captured['cli_args'].index('--idempotency-key') + 1] == 'wrapper-create-1' assert payload['handoff_id'] == 'HND-TEST-001' assert payload['outbox'].endswith('/outgoing/agent-a/HND-TEST-001.md') assert payload['inbox'].endswith('/incoming/agent-b/HND-TEST-001.md')