From 3593776ac54577e8b14e01f69a3d4db362fdb0e2 Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Fri, 30 Oct 2026 10:23:51 +0800 Subject: [PATCH] perf(benchmark): return focused experiment row receipts by default Signed-off-by: huangruiteng --- .../capabilities/benchmark_toolkit/README.md | 13 ++++ .../benchmark_experiment_board.py | 40 ++++++++++++- skills/loopx-benchmark/SKILL.md | 6 ++ .../test_benchmark_experiment_board.py | 59 ++++++++++++++++++- 4 files changed, 116 insertions(+), 2 deletions(-) diff --git a/loopx/capabilities/benchmark_toolkit/README.md b/loopx/capabilities/benchmark_toolkit/README.md index a7d1cc2eea..052d49e0f6 100644 --- a/loopx/capabilities/benchmark_toolkit/README.md +++ b/loopx/capabilities/benchmark_toolkit/README.md @@ -1105,6 +1105,19 @@ loopx benchmark experiment-board-upsert \ --format json ``` +Upsert now returns the complete normalized affected `row` and `write` receipt +by default, for both preview and execution. Check `write.dry_run` and +`write.write_performed` before treating a preview as a saved update. The receipt +retains metrics, countability, protocol identity and total ledger `row_count`; +it does not imply a matched comparison or replace whole-study inspection. +Consumers of the previous board-shaped response can add `--include-board`. +`--benchmark-id` and `--study-id` filter that optional board projection; they do +not constrain the write or hide the affected row in the default receipt. +For a later comparison, call `experiment-board-show` with the same Goal/project +route and relevant benchmark/study filters. Reuse a current board already read +when it answers the decision; avoid rebuilding and returning it after every +single-row update. Validation and invalid-transition rejection still apply. + The default ledger is locked, atomically updated, and keyed by benchmark, study, case, and run identity. A compact row carries arm role, exact and comparison protocol ids, model, score metrics, countability, treatment fidelity, bounded diff --git a/loopx/cli_commands/benchmark_experiment_board.py b/loopx/cli_commands/benchmark_experiment_board.py index f4cfdf12ba..942f997b36 100644 --- a/loopx/cli_commands/benchmark_experiment_board.py +++ b/loopx/cli_commands/benchmark_experiment_board.py @@ -8,6 +8,7 @@ from typing import Any from ..capabilities.benchmark_toolkit import ( + BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION, benchmark_experiment_board_row_key, build_benchmark_experiment_board, default_benchmark_experiment_board_path, @@ -92,6 +93,14 @@ def register_benchmark_experiment_board_commands( required=True, help="Compact benchmark_experiment_board_row_v0 JSON path, or - for stdin.", ) + upsert_parser.add_argument( + "--include-board", + action="store_true", + help=( + "Return the full board projection (legacy response) instead of the " + "default affected row and write receipt. Board filters apply to this projection." + ), + ) upsert_parser.add_argument( "--execute", action="store_true", @@ -135,6 +144,16 @@ def _board_payload( ) +def _render_upsert_receipt(payload: dict[str, object]) -> str: + return "\n".join([ + "# Benchmark experiment row", + "", + "```json", + json.dumps(payload, ensure_ascii=False, indent=2), + "```", + ]) + + def _reconcile_experiment_board( args: argparse.Namespace, *, @@ -200,7 +219,9 @@ def handle_benchmark_experiment_board_command( row = _read_json_object(args.row_json) if args.execute: receipt = upsert_benchmark_experiment_board_row(ledger_path, row) - rows = read_benchmark_experiment_board_rows(ledger_path) + affected_row = receipt["row"] + if args.include_board: + rows = read_benchmark_experiment_board_rows(ledger_path) write = { **receipt["write"], "dry_run": False, @@ -209,6 +230,7 @@ def handle_benchmark_experiment_board_command( rows, preview_status = preview_benchmark_experiment_board_upsert( rows, row ) + affected_row = next(item for item in rows if _row_key(item) == _row_key(row)) write = { "status": f"preview_{preview_status}", "write_performed": False, @@ -216,6 +238,22 @@ def handle_benchmark_experiment_board_command( "path_recorded": False, "dry_run": True, } + if not args.include_board: + payload = { + "ok": True, + "schema_version": BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION, + "goal_id": args.goal_id, + "path_recorded": False, + "row": affected_row, + "write": write, + "read_hint": ( + "Use this row and receipt for the update decision. For comparisons, " + "use experiment-board-show with the same route and relevant " + "--benchmark-id/--study-id; --include-board restores the full upsert response." + ), + } + print_payload(payload, output_format(args), _render_upsert_receipt) + return 0 elif args.benchmark_command == "experiment-board-reconcile": rows, write = _reconcile_experiment_board( args, diff --git a/skills/loopx-benchmark/SKILL.md b/skills/loopx-benchmark/SKILL.md index c9dbc17bda..07db8a72c3 100644 --- a/skills/loopx-benchmark/SKILL.md +++ b/skills/loopx-benchmark/SKILL.md @@ -155,6 +155,12 @@ separate: fewer bytes, calls or tokens are not proof of a better task outcome. The running row uses `status=running`, empty `metrics`, and `countability={integrity_qualified:false, official_result_present:false, score_countable:false}`. Keep the same stable `run_id` for every transition. + Upsert returns the normalized affected `row` and `write` receipt by default. + Check its identity and preview/write outcome directly. Read the board again + when the next comparison needs it, using the same route and relevant + `--benchmark-id`/`--study-id`; do not re-fetch a whole study merely to confirm + one update. `--include-board` restores the previous full-board response for + consumers that need it. A row receipt does not establish comparison eligibility. 4. **Preview and upsert terminal score, countability, effort, and insight.** First run integrity qualification. An automated restricted-access match is a diff --git a/tests/capabilities/test_benchmark_experiment_board.py b/tests/capabilities/test_benchmark_experiment_board.py index a4e2d56238..ef6331039d 100644 --- a/tests/capabilities/test_benchmark_experiment_board.py +++ b/tests/capabilities/test_benchmark_experiment_board.py @@ -882,11 +882,14 @@ def test_cli_previews_then_writes_and_reads_board(tmp_path: Path) -> None: ) assert preview_payload["write"]["status"] == "preview_inserted" assert preview_payload["write"]["write_performed"] is False + assert preview_payload["row"] == normalize_benchmark_experiment_board_row(_baseline()) + assert "runs" not in preview_payload assert not ledger.exists() executed = run_source_cli([*command, "--execute"], check=True) executed_payload = json.loads(executed.stdout) assert executed_payload["write"]["status"] == "inserted" + assert executed_payload["row"] == preview_payload["row"] assert executed_payload["path_recorded"] is False assert str(tmp_path) not in executed.stdout assert ledger.exists() @@ -911,6 +914,60 @@ def test_cli_previews_then_writes_and_reads_board(tmp_path: Path) -> None: ) +@pytest.mark.parametrize("execute", [False, True]) +def test_cli_upsert_returns_exact_row_without_board_growth(tmp_path: Path, execute: bool) -> None: + ledger = default_benchmark_experiment_board_path(project=tmp_path, goal_id="fixture-goal") + ledger.parent.mkdir(parents=True, exist_ok=True) + # Run ids may repeat across cases. Preserve an uncountable zero exactly. + candidate = _baseline() + candidate["countability"] = { + "integrity_qualified": False, "official_result_present": True, "score_countable": False, + } + rows = [_baseline(case_id=f"case-{index}") for index in range(1000)] + ledger.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + original = ledger.read_bytes() + row_path = tmp_path / "candidate.json" + row_path.write_text(json.dumps(candidate), encoding="utf-8") + command = ["benchmark", "experiment-board-upsert", "--goal-id", "fixture-goal", + "--project", str(tmp_path), "--row-json", str(row_path), "--format", "json"] + if execute: + command.append("--execute") + result = run_source_cli(command) + payload = json.loads(result.stdout) + assert payload["row"] == normalize_benchmark_experiment_board_row(candidate) + assert payload["write"]["row_count"] == 1000 + assert payload["write"]["write_performed"] is execute + assert "experiment-board-show" in payload["read_hint"] + assert "runs" not in payload and "summary" not in payload + assert len(result.stdout.encode()) < 6000 # One complete row, not 1000 rows. + if execute: + observed = read_benchmark_experiment_board_rows(ledger) + assert len(observed) == 1000 + assert all(row["countability"]["score_countable"] is (row["case_id"] != "case-12") + for row in observed) + else: + assert ledger.read_bytes() == original + full = json.loads(run_source_cli([*command, "--include-board"]).stdout) + assert full["summary"]["run_count"] == 1000 + assert len(full["runs"]) == 1000 + assert full["write"]["write_performed"] is execute + + +def test_cli_compact_upsert_rejects_invalid_transition_without_writing(tmp_path: Path) -> None: + ledger = default_benchmark_experiment_board_path(project=tmp_path, goal_id="fixture-goal") + upsert_benchmark_experiment_board_row(ledger, _baseline()) + before = ledger.read_bytes() + row_path = tmp_path / "row.json" + row_path.write_text(json.dumps(_running_baseline()), encoding="utf-8") + result = run_source_cli([ + "benchmark", "experiment-board-upsert", "--goal-id", "fixture-goal", + "--project", str(tmp_path), "--row-json", str(row_path), "--execute", "--format", "json", + ], check=False) + assert result.returncode != 0 + assert "invalid benchmark experiment board input" in result.stderr + assert ledger.read_bytes() == before + + def test_cli_show_projects_factorial_contrast_from_explicit_contract( tmp_path: Path, ) -> None: @@ -1031,7 +1088,7 @@ def test_cli_omitted_project_routes_board_to_registered_goal_repo( goal_id="fixture-goal", ) - assert payload["summary"]["run_count"] == 1 + assert payload["write"]["row_count"] == 1 assert ledger.is_file() assert not (unrelated / ".loopx").exists()