diff --git a/CHANGELOG.md b/CHANGELOG.md index b47e30d..025068f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/). +## [Unreleased] + +### Added + +- **`kaizen history`** — parses the `cd-aor: step N` git checkpoint commits + written by the orchestrator (ADR-0006) and prints the denoising trajectory + (confidence, delta, tests, files) with regression detection. Supports + `--path`, `--json`, `--limit`, and `--task` (scope to one task id when a + shared workspace interleaves multiple tasks' checkpoints). Tolerates + additive schema minor bumps (e.g. 1.1.0/1.2.0) and surfaces checkpoints + written by an unsupported schema major instead of mis-rendering them. + ## [1.0.1] - 2026-05-08 ### Added diff --git a/cli/commands/history.py b/cli/commands/history.py new file mode 100644 index 0000000..f71902c --- /dev/null +++ b/cli/commands/history.py @@ -0,0 +1,236 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`kaizen history` — render the CD-AOR denoising trajectory from git. + +ADR-0006 Phase 3 analysis tooling. The Rust orchestrator writes one git +commit per denoising step whose subject is ``cd-aor: step {N} — {desc}`` and +whose body is a JSON metadata object (schema versioned). This command parses +those commits in a workspace and prints the trajectory as a table — with +regression detection (a step whose composite confidence dropped vs. the +previous step) — or as raw JSON. + +Stdlib only; reads git via subprocess. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from .. import output +from ..output import Style + +# git log field/record separators: ASCII Unit/Record separators never appear +# in JSON or normal commit text, so the split is reliable across the +# multi-line JSON bodies. +_FIELD_SEP = "\x1f" +_RECORD_SEP = "\x1e" +_SUBJECT_PREFIX = "cd-aor: step " +# Highest checkpoint metadata schema MAJOR this CLI understands. Minor bumps +# are additive (e.g. 1.1.0 added the optional rl_signals block) and remain +# parseable; a different MAJOR may have moved/removed fields, so such commits +# are surfaced rather than silently mis-rendered. +_SUPPORTED_SCHEMA_MAJOR = 1 + + +def add_history_parser(subparsers: argparse._SubParsersAction) -> argparse.ArgumentParser: + p = subparsers.add_parser( + "history", + help="Show the CD-AOR denoising trajectory from git checkpoints", + description=( + "Parse `cd-aor: step N` checkpoint commits in a workspace and " + "print the denoising trajectory (confidence, delta, tests, files) " + "with regression detection." + ), + ) + p.add_argument("--path", default=".", metavar="DIR", + help="Workspace git repository to inspect (default: cwd)") + p.add_argument("--json", action="store_true", dest="json", + help="Emit the parsed step records as a JSON array") + p.add_argument("--limit", type=int, default=None, metavar="N", + help="Show only the most recent N steps") + p.add_argument("--task", default=None, metavar="ID", + help="Only show steps for this task id (a shared workspace " + "may interleave multiple tasks' checkpoints)") + return p + + +def _git(path: str, args: List[str]) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", "-C", path, *args], + capture_output=True, + text=True, + encoding="utf-8", + ) + + +def _schema_major(version: Any) -> Any: + """Leading integer of a dotted ``schema_version``; None if unparseable.""" + try: + return int(str(version).split(".", 1)[0]) + except (ValueError, AttributeError): + return None + + +def _parse_commits( + raw: str, +) -> Tuple[List[Dict[str, Any]], int, List[str]]: + """Return (step_records, skipped_count, unsupported_versions). + + A commit is a step iff its subject starts with the cd-aor step prefix and + its body is valid JSON carrying a ``schema_version``. Commits that look + like cd-aor steps but whose body is unparseable/legacy are counted as + *skipped* (an actionable signal). A step whose schema MAJOR differs from + what this CLI understands is not rendered (fields may have moved) and its + version is collected in ``unsupported_versions``. Non-step commits + (workspace baseline, human commits) are silently ignored — counting them + would be noise in a repo with ordinary history. + """ + records: List[Dict[str, Any]] = [] + skipped = 0 + unsupported: List[str] = [] + for chunk in raw.split(_RECORD_SEP): + chunk = chunk.strip("\n") + if not chunk or _FIELD_SEP not in chunk: + continue + commit_hash, _, body = chunk.partition(_FIELD_SEP) + subject = body.split("\n", 1)[0].strip() + if not subject.startswith(_SUBJECT_PREFIX): + continue + # Body after the blank line that follows the subject. + payload = body.split("\n\n", 1)[1] if "\n\n" in body else "" + try: + rec = json.loads(payload) + except json.JSONDecodeError: + skipped += 1 + continue + if not isinstance(rec, dict) or "schema_version" not in rec: + skipped += 1 + continue + major = _schema_major(rec.get("schema_version")) + if major != _SUPPORTED_SCHEMA_MAJOR: + ver = str(rec.get("schema_version")) + if ver not in unsupported: + unsupported.append(ver) + continue + rec["_commit"] = commit_hash.strip()[:7] + records.append(rec) + return records, skipped, unsupported + + +def _flag_regressions(records: List[Dict[str, Any]]) -> List[int]: + """Indices of records whose confidence dropped vs. the previous step.""" + regressed: List[int] = [] + prev: Optional[float] = None + for i, rec in enumerate(records): + c = float(rec.get("confidence_score", 0.0) or 0.0) + if prev is not None and c < prev: + regressed.append(i) + prev = c + return regressed + + +def _render_table(style: Style, records: List[Dict[str, Any]], + regressed: List[int], skipped: int) -> None: + regressed_set = set(regressed) + print(style.bold( + f"{'':1}{'step':>4} {'C':>6} {'delta':>7} " + f"{'tests':>9} {'files':>5} {'commit':<7} timestamp" + )) + + confidences: List[float] = [] + for i, rec in enumerate(records): + step = rec.get("step_number", "?") + c = float(rec.get("confidence_score", 0.0) or 0.0) + confidences.append(c) + delta = float(rec.get("convergence_delta", 0.0) or 0.0) + tr = rec.get("test_results", {}) or {} + passed = tr.get("passed", 0) + total = tr.get("total", 0) + files = len(rec.get("files_modified", []) or []) + commit = rec.get("_commit", "") + ts = rec.get("timestamp_utc", "") + is_reg = i in regressed_set + mark = "!" if is_reg else " " + row = ( + f"{mark}{step:>4} {c:>6.3f} {delta:>+7.3f} " + f"{f'{passed}/{total}':>9} {files:>5} {commit:<7} {ts}" + ) + print(style.red(row) if is_reg else row) + + if confidences: + traj = " -> ".join(f"{c:.3f}" for c in confidences) + print() + print(f"trajectory: {traj}") + + if regressed: + steps = ", ".join(str(records[i].get("step_number", "?")) for i in regressed) + print(f"regressions: steps {steps}") + else: + print("regressions: none") + + if skipped: + print(style.dim(f" (skipped {skipped} unparseable cd-aor checkpoint(s))")) + + +def history_command(args: argparse.Namespace) -> int: + style = Style(use_color=(not args.no_color) if hasattr(args, "no_color") else None) + root = Path(args.path).resolve() + + if not root.exists() or not root.is_dir(): + output.error(style, f"path does not exist: {root}") + return 2 + + try: + inside = _git(str(root), ["rev-parse", "--is-inside-work-tree"]) + if inside.returncode != 0 or inside.stdout.strip() != "true": + output.error(style, f"not a git repository: {root}") + return 2 + + log = _git(str(root), ["log", "--reverse", + f"--format=%H{_FIELD_SEP}%B{_RECORD_SEP}"]) + # A repo with no commits exits non-zero here — that is simply an + # empty trajectory, not an error. + raw = log.stdout if log.returncode == 0 else "" + + records, skipped, unsupported = _parse_commits(raw) + except FileNotFoundError: + output.error(style, "git executable not found on PATH") + return 1 + except Exception as exc: # pragma: no cover - defensive + output.error(style, f"{exc.__class__.__name__}: {exc}") + return 1 + + task = getattr(args, "task", None) + if task: + records = [r for r in records if r.get("task_id") == task] + + if args.limit is not None and args.limit >= 0: + records = records[-args.limit:] if args.limit else [] + + if args.json: + print(json.dumps(records, indent=2)) + return 0 + + def _note_unsupported() -> None: + if unsupported: + vers = ", ".join(unsupported) + output.warn( + style, + f"{len(unsupported)} unsupported schema version(s) skipped " + f"({vers}); upgrade kaizen to read these checkpoints", + ) + + if not records: + print(f"No cd-aor checkpoints found in {root}") + if skipped: + print(style.dim(f" (skipped {skipped} unparseable cd-aor checkpoint(s))")) + _note_unsupported() + return 0 + + regressed = _flag_regressions(records) + _render_table(style, records, regressed, skipped) + _note_unsupported() + return 0 diff --git a/cli/main.py b/cli/main.py index ad5ec26..14e954a 100644 --- a/cli/main.py +++ b/cli/main.py @@ -26,6 +26,7 @@ from cli.commands.priors import add_priors_parser, priors_command from cli.commands.resume import add_resume_parser, resume_command from cli.commands.status import add_status_parser, status_command +from cli.commands.history import add_history_parser, history_command from cli.commands.web import add_web_parser, web_command from cli.commands.bench import add_bench_parser, bench_command from cli.commands.demo import add_demo_parser, demo_command @@ -64,6 +65,7 @@ def _build_parser() -> argparse.ArgumentParser: add_migrate_plan_parser(subparsers) # Inspection / utility. add_status_parser(subparsers) + add_history_parser(subparsers) add_priors_parser(subparsers) add_resume_parser(subparsers) # First-run wizard + config inspection. @@ -94,6 +96,8 @@ def main(argv: Optional[List[str]] = None) -> int: try: if args.command == "status": return status_command(args) + if args.command == "history": + return history_command(args) if args.command == "priors": return priors_command(args) if args.command == "memsafe-roadmap": diff --git a/cli/tests/test_history.py b/cli/tests/test_history.py new file mode 100644 index 0000000..e74d3d2 --- /dev/null +++ b/cli/tests/test_history.py @@ -0,0 +1,241 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tests for `cli.commands.history` (ADR-0006 Phase 3).""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path + +import pytest + +from cli.commands.history import add_history_parser, history_command + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_args(**kwargs) -> argparse.Namespace: + defaults = dict(path=".", json=False, limit=None, task=None, + no_color=True, verbose=False) + defaults.update(kwargs) + return argparse.Namespace(**defaults) + + +def _git(repo: Path, *args: str) -> None: + subprocess.run(["git", "-C", str(repo), *args], + check=True, capture_output=True, text=True) + + +def _init_repo(repo: Path) -> None: + _git(repo, "init", "-q") + _git(repo, "config", "user.email", "test@example.com") + _git(repo, "config", "user.name", "Test User") + _git(repo, "config", "commit.gpgsign", "false") + + +def _metadata(step: int, confidence: float, delta: float = 0.05) -> dict: + return { + "schema_version": "1.0.0", + "step_number": step, + "confidence_score": confidence, + "convergence_delta": delta, + "test_results": {"total": 10, "passed": 9, "failed": 1, "coverage_pct": 88.0}, + "agent_messages": [ + {"agent": "researcher", "summary": "r", "hash": "sha256:aaa"}, + {"agent": "red_team", "summary": "rt", "hash": "sha256:bbb"}, + {"agent": "draft", "summary": "d", "hash": "sha256:ccc"}, + {"agent": "write", "summary": "w", "hash": "sha256:ddd", + "taor_turns_used": None, "taor_turns_max": 50}, + {"agent": "evaluator", "composite_score": confidence, + "signals": {"test_pass_rate": 0.9}}, + ], + "files_modified": [f"src/step{step}.rs"], + "timestamp_utc": f"2026-05-15T00:0{step}:00Z", + } + + +def _commit(repo: Path, n: int, subject: str, body: str) -> None: + """Create a commit with `subject\\n\\n{body}` and a real file change.""" + (repo / f"file{n}.txt").write_text(f"content {n}\n", encoding="utf-8") + _git(repo, "add", "-A") + msg = f"{subject}\n\n{body}" if body else subject + _git(repo, "commit", "-q", "-m", msg) + + +@pytest.fixture +def workspace(tmp_path: Path) -> Path: + """Repo with: baseline, step1 (0.60), step2 (0.70), step3 (0.65 regression), + a human commit, and a legacy non-JSON `cd-aor: step` commit.""" + repo = tmp_path / "ws" + repo.mkdir() + _init_repo(repo) + _commit(repo, 0, "cd-aor: workspace baseline", "") + _commit(repo, 1, "cd-aor: step 1 — denoising iteration", + json.dumps(_metadata(1, 0.60), indent=2)) + _commit(repo, 2, "cd-aor: step 2 — denoising iteration", + json.dumps(_metadata(2, 0.70), indent=2)) + _commit(repo, 3, "cd-aor: step 3 — denoising iteration", + json.dumps(_metadata(3, 0.65), indent=2)) + _commit(repo, 4, "chore: a human commit", "") + _commit(repo, 5, "cd-aor: step 9 — old format", "not-json-legacy-body") + return repo + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +def test_step_row_count(workspace: Path, capsys: pytest.CaptureFixture) -> None: + rc = history_command(_make_args(path=str(workspace))) + assert rc == 0 + out = capsys.readouterr().out + # 3 valid step rows. + assert out.count("denoising") == 0 # subjects are not echoed + for s in ("1", "2", "3"): + assert s in out + assert "trajectory:" in out + + +def test_regression_flagged(workspace: Path, capsys: pytest.CaptureFixture) -> None: + rc = history_command(_make_args(path=str(workspace))) + assert rc == 0 + out = capsys.readouterr().out + # Step 3 (0.65) regressed vs step 2 (0.70); marked with '!'. + reg_lines = [ln for ln in out.splitlines() if ln.strip().startswith("!")] + assert len(reg_lines) == 1 + assert " 3 " in reg_lines[0] or reg_lines[0].strip().startswith("! 3") + + +def test_regression_summary(workspace: Path, capsys: pytest.CaptureFixture) -> None: + history_command(_make_args(path=str(workspace))) + out = capsys.readouterr().out + assert "regressions: steps 3" in out + + +def test_skipped_note(workspace: Path, capsys: pytest.CaptureFixture) -> None: + history_command(_make_args(path=str(workspace))) + out = capsys.readouterr().out + # Only the legacy `cd-aor: step 9` commit with a non-JSON body is an + # unparseable checkpoint; the baseline and human commits are not cd-aor + # steps and are silently ignored (not counted as noise). + assert "skipped 1 unparseable cd-aor checkpoint(s)" in out + + +def test_json_mode(workspace: Path, capsys: pytest.CaptureFixture) -> None: + rc = history_command(_make_args(path=str(workspace), json=True)) + assert rc == 0 + data = json.loads(capsys.readouterr().out) + assert isinstance(data, list) + assert len(data) == 3 + assert [r["step_number"] for r in data] == [1, 2, 3] + assert all("_commit" in r for r in data) + + +def test_limit_respected(workspace: Path, capsys: pytest.CaptureFixture) -> None: + rc = history_command(_make_args(path=str(workspace), json=True, limit=2)) + assert rc == 0 + data = json.loads(capsys.readouterr().out) + assert [r["step_number"] for r in data] == [2, 3] + + +def test_trajectory_format(workspace: Path, capsys: pytest.CaptureFixture) -> None: + history_command(_make_args(path=str(workspace))) + out = capsys.readouterr().out + assert "trajectory: 0.600 -> 0.700 -> 0.650" in out + + +def test_non_git_path_exit_2(tmp_path: Path) -> None: + plain = tmp_path / "plain" + plain.mkdir() + rc = history_command(_make_args(path=str(plain))) + assert rc == 2 + + +def test_nonexistent_path_exit_2() -> None: + rc = history_command(_make_args(path="/no/such/dir/xyz123")) + assert rc == 2 + + +def test_only_human_commits_exit_0(tmp_path: Path, + capsys: pytest.CaptureFixture) -> None: + repo = tmp_path / "human" + repo.mkdir() + _init_repo(repo) + _commit(repo, 1, "chore: only human commits here", "") + rc = history_command(_make_args(path=str(repo))) + assert rc == 0 + assert "No cd-aor checkpoints found" in capsys.readouterr().out + + +def test_minor_schema_bump_still_parsed(tmp_path: Path, + capsys: pytest.CaptureFixture) -> None: + """A 1.x minor bump (additive, e.g. 1.1.0 rl_signals) stays parseable.""" + repo = tmp_path / "minor" + repo.mkdir() + _init_repo(repo) + m = _metadata(1, 0.60) + m["schema_version"] = "1.1.0" + _commit(repo, 1, "cd-aor: step 1 — minor", json.dumps(m)) + + rc = history_command(_make_args(path=str(repo))) + assert rc == 0 + assert "trajectory:" in capsys.readouterr().out + + +def test_unsupported_schema_major_surfaced(tmp_path: Path, + capsys: pytest.CaptureFixture) -> None: + """A different schema MAJOR is not rendered and is surfaced to stderr.""" + repo = tmp_path / "future" + repo.mkdir() + _init_repo(repo) + m = _metadata(1, 0.60) + m["schema_version"] = "2.0.0" + _commit(repo, 1, "cd-aor: step 1 — future", json.dumps(m)) + + rc = history_command(_make_args(path=str(repo))) + assert rc == 0 + cap = capsys.readouterr() + assert "No cd-aor checkpoints found" in cap.out + assert "unsupported schema version" in cap.err + assert "2.0.0" in cap.err + + +def test_task_filter_scopes_to_one_task(tmp_path: Path, + capsys: pytest.CaptureFixture) -> None: + """A shared workspace interleaving two tasks: --task isolates one.""" + repo = tmp_path / "multi" + repo.mkdir() + _init_repo(repo) + a1 = _metadata(1, 0.60) + a1["task_id"] = "task-A" + b1 = _metadata(1, 0.55) + b1["task_id"] = "task-B" + a2 = _metadata(2, 0.72) + a2["task_id"] = "task-A" + _commit(repo, 1, "cd-aor: step 1 — A", json.dumps(a1)) + _commit(repo, 2, "cd-aor: step 1 — B", json.dumps(b1)) + _commit(repo, 3, "cd-aor: step 2 — A", json.dumps(a2)) + + rc = history_command(_make_args(path=str(repo), task="task-A", json=True)) + assert rc == 0 + data = json.loads(capsys.readouterr().out) + assert len(data) == 2 + assert {r["task_id"] for r in data} == {"task-A"} + assert [r["step_number"] for r in data] == [1, 2] + + +def test_add_history_parser() -> None: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command") + add_history_parser(subparsers) + + ns = parser.parse_args(["history", "--path", "X", "--json", + "--limit", "5", "--task", "t1"]) + assert ns.command == "history" + assert ns.path == "X" + assert ns.json is True + assert ns.limit == 5 + assert ns.task == "t1" diff --git a/docs/CLI_GUIDE.md b/docs/CLI_GUIDE.md index 7a6e009..8228b1e 100644 --- a/docs/CLI_GUIDE.md +++ b/docs/CLI_GUIDE.md @@ -112,6 +112,7 @@ kaizen [options] | `memsafe-roadmap` | CISA-format memory-safety roadmap + ADRs (C/C++ → Rust wedge) | | `migrate-plan` | framework migration plan + ADRs (9 pairs) | | `status` | summary of recent Kaizen runs under a path | +| `history` | CD-AOR denoising trajectory from git checkpoints | | `priors` | inspect or reset Thompson-sampling priors | | `resume` | re-run recompose from the most recent (or specified) ADR | | `init` | first-run configuration wizard — writes `~/.kaizen/config.toml` | @@ -196,6 +197,24 @@ kaizen status --path ./out Summarizes the most recent `taor_observations.jsonl` and `priors.json` files it finds, with the last confidence trajectory. +### `kaizen history` — the denoising audit trail + +```bash +kaizen history # inspect ./ +kaizen history --path ./workspace # a specific workspace repo +kaizen history --json # raw step records (JSON array) +kaizen history --limit 10 # most recent 10 steps +kaizen history --task # scope to one task in a shared workspace +``` + +Parses the `cd-aor: step N` checkpoint commits the orchestrator writes +(ADR-0006) and prints the denoising trajectory: per-step composite +confidence, convergence delta, test pass counts, files changed, commit, and +timestamp. Steps whose confidence dropped vs. the previous step are flagged +as regressions. Checkpoint commits whose metadata body is legacy/unparseable +are skipped and counted; ordinary commits (the workspace baseline, human +commits) are ignored. + ### `kaizen priors show / reset` Inspect or wipe the Thompson-sampling priors file that the adaptive