From 0323daba8e0a2828d616f25d7872e3d14575eb95 Mon Sep 17 00:00:00 2001 From: Vitor Castro Date: Thu, 22 Oct 2026 16:07:25 +0100 Subject: [PATCH 1/3] Ship v0.2.0: adaptive harness, honest agent loop, and release announce. Make weak local models finish read-edit-test with per-model profiles, finish gates, and safer permissions; package calibrate benches and LinkedIn announce for the commercial release story. --- .github/workflows/announce-release.yml | 85 + README.md | 222 ++- benchmarks/README.md | 10 +- benchmarks/bench.mjs | 602 ++++++- benchmarks/headlessAgent.mjs | 117 +- benchmarks/layout.mjs | 53 + benchmarks/tasks/T4-refactor-rename/task.json | 2 +- docs/releases/v0.2.0.md | 94 ++ extensions/codeforge/HEURISTICS.md | 9 +- extensions/codeforge/package-lock.json | 39 +- extensions/codeforge/package.json | 47 +- .../resources/calibrate-bench/.packaged | 2 + .../resources/calibrate-bench/README.md | 4 + .../resources/calibrate-bench/bench.mjs | 1441 +++++++++++++++++ .../fixtures/billing/.gitignore | 6 + .../fixtures/billing/Acme.Billing.slnx | 8 + .../fixtures/billing/README.md | 15 + .../fixtures/billing/config/coupons.json | 5 + .../fixtures/billing/docs/pricing.md | 36 + .../src/Acme.Billing/Acme.Billing.csproj | 9 + .../fixtures/billing/src/Acme.Billing/Cart.cs | 40 + .../billing/src/Acme.Billing/Coupons.cs | 32 + .../billing/src/Acme.Billing/Money.cs | 7 + .../billing/src/Acme.Billing/PricingRules.cs | 176 ++ .../fixtures/billing/src/Acme.Billing/Tax.cs | 18 + .../billing/src/Acme.Billing/Totals.cs | 29 + .../Acme.Billing.Tests.csproj | 25 + .../tests/Acme.Billing.Tests/CartTests.cs | 58 + .../calibrate-bench/fixtures/shop/README.md | 11 + .../fixtures/shop/config/coupons.json | 5 + .../fixtures/shop/docs/pricing.md | 36 + .../fixtures/shop/package.json | 10 + .../fixtures/shop/src/cart/cart.js | 30 + .../fixtures/shop/src/cart/totals.js | 28 + .../fixtures/shop/src/checkout/invoice.js | 42 + .../fixtures/shop/src/index.js | 26 + .../fixtures/shop/src/pricing/coupons.js | 12 + .../fixtures/shop/src/pricing/rules.js | 195 +++ .../fixtures/shop/src/pricing/tax.js | 15 + .../fixtures/shop/src/util/money.js | 11 + .../fixtures/shop/test/cart.test.js | 51 + .../fixtures/shop/test/invoice.test.js | 29 + .../calibrate-bench/headlessAgent.mjs | 408 +++++ .../resources/calibrate-bench/layout.mjs | 53 + .../calibrate-bench/support/vscodeFsMock.js | 252 +++ .../hidden/CouponHiddenTests.cs | 57 + .../tasks/CS2-feature-coupon/solution.json | 6 + .../tasks/CS2-feature-coupon/task.json | 13 + .../CS3-nav-vip-rate/hidden/VipHiddenTests.cs | 39 + .../tasks/CS3-nav-vip-rate/solution.json | 4 + .../tasks/CS3-nav-vip-rate/task.json | 15 + .../T2-feature-coupon/hidden/coupon.test.js | 47 + .../tasks/T2-feature-coupon/solution.json | 8 + .../tasks/T2-feature-coupon/task.json | 9 + .../T4-refactor-rename/hidden/rename.test.js | 23 + .../tasks/T4-refactor-rename/solution.json | 6 + .../tasks/T4-refactor-rename/task.json | 11 + .../codeforge/resources/webview/purify.min.js | 7 + extensions/codeforge/src/agent/agentLoop.ts | 382 ++++- extensions/codeforge/src/agent/editTool.ts | 22 +- .../codeforge/src/agent/experimentLoop.ts | 40 +- .../src/agent/finishGate.selftest.ts | 81 +- extensions/codeforge/src/agent/finishGate.ts | 126 +- .../src/agent/harnessRevision.selftest.ts | 249 +++ .../codeforge/src/agent/harnessRevision.ts | 657 ++++++++ .../codeforge/src/agent/internalPlans.ts | 10 +- extensions/codeforge/src/agent/llmThinking.ts | 42 +- .../codeforge/src/agent/loopDetector.ts | 56 +- .../src/agent/modelProfile.selftest.ts | 105 ++ .../codeforge/src/agent/modelProfile.ts | 267 +++ extensions/codeforge/src/agent/nudges.ts | 47 +- .../codeforge/src/agent/orchestrator.ts | 9 +- .../codeforge/src/agent/orchestratorRun.ts | 26 +- .../src/agent/profileHysteresis.selftest.ts | 103 ++ .../codeforge/src/agent/profileHysteresis.ts | 78 + .../codeforge/src/agent/projectStatus.ts | 19 +- .../src/agent/protocolTelemetry.selftest.ts | 65 + .../codeforge/src/agent/protocolTelemetry.ts | 227 +++ .../codeforge/src/agent/toolCallNormalizer.ts | 12 +- .../src/agent/toolContracts.selftest.ts | 15 +- .../codeforge/src/agent/toolContracts.ts | 36 +- extensions/codeforge/src/agent/toolHistory.ts | 45 + .../src/agent/xmlToolCalls.selftest.ts | 44 + .../codeforge/src/agent/xmlToolCalls.ts | 55 +- .../src/bridge/shellPrep.selftest.ts | 38 + extensions/codeforge/src/bridge/shellPrep.ts | 71 + .../codeforge/src/bridge/vscodeBridge.ts | 116 +- .../src/bridge/workspaceWriteGate.ts | 83 + extensions/codeforge/src/extension.ts | 29 +- .../src/hooks/hooksRunner.selftest.ts | 11 + extensions/codeforge/src/hooks/hooksRunner.ts | 36 +- .../codeforge/src/hooks/hooksShellQuote.ts | 8 + .../codeforge/src/mcp/mcpClient.selftest.ts | 17 + extensions/codeforge/src/mcp/mcpClient.ts | 63 +- .../codeforge/src/mcp/mcpResultFormat.ts | 25 + .../src/memory/extendedMemory.selftest.ts | 17 + .../codeforge/src/memory/extendedMemory.ts | 18 + .../src/policy/approvalPolicy.selftest.ts | 18 + .../codeforge/src/policy/approvalPolicy.ts | 102 +- .../codeforge/src/sandbox/processKill.ts | 37 + .../codeforge/src/sandbox/runtime.selftest.ts | 39 + extensions/codeforge/src/sandbox/runtime.ts | 59 +- .../codeforge/src/sessions/sessionStore.ts | 129 +- .../codeforge/src/settings/aiSecrets.ts | 153 ++ .../codeforge/src/settings/aiSettingsPanel.ts | 467 +++++- .../codeforge/src/settings/aiSettingsStore.ts | 96 +- .../modelProfileCalibrate.selftest.ts | 23 + .../src/settings/modelProfileCalibrate.ts | 478 ++++++ extensions/codeforge/src/storage/paths.ts | 26 + extensions/codeforge/src/ui/approvalDialog.ts | 16 + extensions/codeforge/src/views/chatView.ts | 132 +- .../codeforge/src/views/chatViewHtml.ts | 50 +- package.json | 5 +- scripts/sync-calibrate-bench.mjs | 87 + tests/unit/extension-platform.test.ts | 9 +- tests/unit/harness-modules.test.ts | 35 + tests/unit/loop-detector.test.ts | 36 +- .../LinkedInAnnouncer.csproj | 9 + tools/linkedin-announcer/LinkedInClient.cs | 52 + tools/linkedin-announcer/Models.cs | 44 + tools/linkedin-announcer/OAuthHelper.cs | 87 + tools/linkedin-announcer/PostFormatter.cs | 154 ++ tools/linkedin-announcer/Program.cs | 116 ++ tools/linkedin-announcer/README.md | 24 + .../Debug/net9.0/LinkedInAnnouncer.deps.json | 23 + .../bin/Debug/net9.0/LinkedInAnnouncer.dll | Bin 0 -> 41472 bytes .../bin/Debug/net9.0/LinkedInAnnouncer.exe | Bin 0 -> 156160 bytes .../bin/Debug/net9.0/LinkedInAnnouncer.pdb | Bin 0 -> 23496 bytes .../LinkedInAnnouncer.runtimeconfig.json | 12 + ...CoreApp,Version=v9.0.AssemblyAttributes.cs | 4 + .../net9.0/LinkedInAnnouncer.AssemblyInfo.cs | 22 + ...LinkedInAnnouncer.AssemblyInfoInputs.cache | 1 + ....GeneratedMSBuildEditorConfig.editorconfig | 18 + .../LinkedInAnnouncer.GlobalUsings.g.cs | 8 + .../net9.0/LinkedInAnnouncer.assets.cache | Bin 0 -> 228 bytes ...InAnnouncer.csproj.CoreCompileInputs.cache | 1 + ...kedInAnnouncer.csproj.FileListAbsolute.txt | 15 + .../obj/Debug/net9.0/LinkedInAnnouncer.dll | Bin 0 -> 41472 bytes .../LinkedInAnnouncer.genruntimeconfig.cache | 1 + .../obj/Debug/net9.0/LinkedInAnnouncer.pdb | Bin 0 -> 23496 bytes .../net9.0/LinkedInAnnouncer.sourcelink.json | 1 + .../obj/Debug/net9.0/apphost.exe | Bin 0 -> 156160 bytes .../Debug/net9.0/ref/LinkedInAnnouncer.dll | Bin 0 -> 12800 bytes .../Debug/net9.0/refint/LinkedInAnnouncer.dll | Bin 0 -> 12800 bytes ...LinkedInAnnouncer.csproj.nuget.dgspec.json | 77 + .../LinkedInAnnouncer.csproj.nuget.g.props | 16 + .../LinkedInAnnouncer.csproj.nuget.g.targets | 2 + .../obj/project.assets.json | 83 + .../obj/project.nuget.cache | 8 + 149 files changed, 10193 insertions(+), 582 deletions(-) create mode 100644 .github/workflows/announce-release.yml create mode 100644 benchmarks/layout.mjs create mode 100644 docs/releases/v0.2.0.md create mode 100644 extensions/codeforge/resources/calibrate-bench/.packaged create mode 100644 extensions/codeforge/resources/calibrate-bench/README.md create mode 100644 extensions/codeforge/resources/calibrate-bench/bench.mjs create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/.gitignore create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/Acme.Billing.slnx create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/README.md create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/config/coupons.json create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/docs/pricing.md create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Acme.Billing.csproj create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Cart.cs create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Coupons.cs create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Money.cs create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/PricingRules.cs create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Tax.cs create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Totals.cs create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/tests/Acme.Billing.Tests/Acme.Billing.Tests.csproj create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/billing/tests/Acme.Billing.Tests/CartTests.cs create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/README.md create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/config/coupons.json create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/docs/pricing.md create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/package.json create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/cart/cart.js create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/cart/totals.js create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/checkout/invoice.js create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/index.js create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/pricing/coupons.js create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/pricing/rules.js create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/pricing/tax.js create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/util/money.js create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/test/cart.test.js create mode 100644 extensions/codeforge/resources/calibrate-bench/fixtures/shop/test/invoice.test.js create mode 100644 extensions/codeforge/resources/calibrate-bench/headlessAgent.mjs create mode 100644 extensions/codeforge/resources/calibrate-bench/layout.mjs create mode 100644 extensions/codeforge/resources/calibrate-bench/support/vscodeFsMock.js create mode 100644 extensions/codeforge/resources/calibrate-bench/tasks/CS2-feature-coupon/hidden/CouponHiddenTests.cs create mode 100644 extensions/codeforge/resources/calibrate-bench/tasks/CS2-feature-coupon/solution.json create mode 100644 extensions/codeforge/resources/calibrate-bench/tasks/CS2-feature-coupon/task.json create mode 100644 extensions/codeforge/resources/calibrate-bench/tasks/CS3-nav-vip-rate/hidden/VipHiddenTests.cs create mode 100644 extensions/codeforge/resources/calibrate-bench/tasks/CS3-nav-vip-rate/solution.json create mode 100644 extensions/codeforge/resources/calibrate-bench/tasks/CS3-nav-vip-rate/task.json create mode 100644 extensions/codeforge/resources/calibrate-bench/tasks/T2-feature-coupon/hidden/coupon.test.js create mode 100644 extensions/codeforge/resources/calibrate-bench/tasks/T2-feature-coupon/solution.json create mode 100644 extensions/codeforge/resources/calibrate-bench/tasks/T2-feature-coupon/task.json create mode 100644 extensions/codeforge/resources/calibrate-bench/tasks/T4-refactor-rename/hidden/rename.test.js create mode 100644 extensions/codeforge/resources/calibrate-bench/tasks/T4-refactor-rename/solution.json create mode 100644 extensions/codeforge/resources/calibrate-bench/tasks/T4-refactor-rename/task.json create mode 100644 extensions/codeforge/resources/webview/purify.min.js create mode 100644 extensions/codeforge/src/agent/harnessRevision.selftest.ts create mode 100644 extensions/codeforge/src/agent/harnessRevision.ts create mode 100644 extensions/codeforge/src/agent/modelProfile.selftest.ts create mode 100644 extensions/codeforge/src/agent/modelProfile.ts create mode 100644 extensions/codeforge/src/agent/profileHysteresis.selftest.ts create mode 100644 extensions/codeforge/src/agent/profileHysteresis.ts create mode 100644 extensions/codeforge/src/agent/protocolTelemetry.selftest.ts create mode 100644 extensions/codeforge/src/agent/protocolTelemetry.ts create mode 100644 extensions/codeforge/src/bridge/shellPrep.selftest.ts create mode 100644 extensions/codeforge/src/bridge/shellPrep.ts create mode 100644 extensions/codeforge/src/bridge/workspaceWriteGate.ts create mode 100644 extensions/codeforge/src/hooks/hooksRunner.selftest.ts create mode 100644 extensions/codeforge/src/hooks/hooksShellQuote.ts create mode 100644 extensions/codeforge/src/mcp/mcpClient.selftest.ts create mode 100644 extensions/codeforge/src/mcp/mcpResultFormat.ts create mode 100644 extensions/codeforge/src/memory/extendedMemory.selftest.ts create mode 100644 extensions/codeforge/src/policy/approvalPolicy.selftest.ts create mode 100644 extensions/codeforge/src/sandbox/processKill.ts create mode 100644 extensions/codeforge/src/sandbox/runtime.selftest.ts create mode 100644 extensions/codeforge/src/settings/aiSecrets.ts create mode 100644 extensions/codeforge/src/settings/modelProfileCalibrate.selftest.ts create mode 100644 extensions/codeforge/src/settings/modelProfileCalibrate.ts create mode 100644 scripts/sync-calibrate-bench.mjs create mode 100644 tools/linkedin-announcer/LinkedInAnnouncer.csproj create mode 100644 tools/linkedin-announcer/LinkedInClient.cs create mode 100644 tools/linkedin-announcer/Models.cs create mode 100644 tools/linkedin-announcer/OAuthHelper.cs create mode 100644 tools/linkedin-announcer/PostFormatter.cs create mode 100644 tools/linkedin-announcer/Program.cs create mode 100644 tools/linkedin-announcer/README.md create mode 100644 tools/linkedin-announcer/bin/Debug/net9.0/LinkedInAnnouncer.deps.json create mode 100644 tools/linkedin-announcer/bin/Debug/net9.0/LinkedInAnnouncer.dll create mode 100644 tools/linkedin-announcer/bin/Debug/net9.0/LinkedInAnnouncer.exe create mode 100644 tools/linkedin-announcer/bin/Debug/net9.0/LinkedInAnnouncer.pdb create mode 100644 tools/linkedin-announcer/bin/Debug/net9.0/LinkedInAnnouncer.runtimeconfig.json create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/.NETCoreApp,Version=v9.0.AssemblyAttributes.cs create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/LinkedInAnnouncer.AssemblyInfo.cs create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/LinkedInAnnouncer.AssemblyInfoInputs.cache create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/LinkedInAnnouncer.GeneratedMSBuildEditorConfig.editorconfig create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/LinkedInAnnouncer.GlobalUsings.g.cs create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/LinkedInAnnouncer.assets.cache create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/LinkedInAnnouncer.csproj.CoreCompileInputs.cache create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/LinkedInAnnouncer.csproj.FileListAbsolute.txt create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/LinkedInAnnouncer.dll create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/LinkedInAnnouncer.genruntimeconfig.cache create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/LinkedInAnnouncer.pdb create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/LinkedInAnnouncer.sourcelink.json create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/apphost.exe create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/ref/LinkedInAnnouncer.dll create mode 100644 tools/linkedin-announcer/obj/Debug/net9.0/refint/LinkedInAnnouncer.dll create mode 100644 tools/linkedin-announcer/obj/LinkedInAnnouncer.csproj.nuget.dgspec.json create mode 100644 tools/linkedin-announcer/obj/LinkedInAnnouncer.csproj.nuget.g.props create mode 100644 tools/linkedin-announcer/obj/LinkedInAnnouncer.csproj.nuget.g.targets create mode 100644 tools/linkedin-announcer/obj/project.assets.json create mode 100644 tools/linkedin-announcer/obj/project.nuget.cache diff --git a/.github/workflows/announce-release.yml b/.github/workflows/announce-release.yml new file mode 100644 index 0000000..c2fb0f0 --- /dev/null +++ b/.github/workflows/announce-release.yml @@ -0,0 +1,85 @@ +name: announce-release + +on: + release: + types: [published] + workflow_dispatch: + inputs: + dry_run: + description: "Print posts without publishing" + required: false + default: "true" + release_tag: + description: "Tag for manual dispatch (optional)" + required: false + default: "" + skip_announce: + description: "Skip LinkedIn (true/false)" + required: false + default: "false" + +permissions: + contents: read + +jobs: + linkedin: + if: github.repository == 'Kortexio/CodeForge' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: 9.0.x + - name: Resolve release notes + id: notes + env: + GH_TOKEN: ${{ github.token }} + EVENT_TAG: ${{ github.event.release.tag_name }} + INPUT_TAG: ${{ github.event.inputs.release_tag }} + run: | + TAG="${EVENT_TAG:-$INPUT_TAG}" + if [ -z "$TAG" ]; then + echo "tag=" >> "$GITHUB_OUTPUT" + echo "skip=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + # Always fetch from the Releases API so multiline notes survive workflow_dispatch. + gh api "repos/${{ github.repository }}/releases/tags/${TAG}" --jq .body > /tmp/release-body.md + { + echo "body<> "$GITHUB_OUTPUT" + if grep -qiE '\[skip[[:space:]-]?announce\]|skip-announce:\s*true' /tmp/release-body.md; then + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + - name: Publish LinkedIn + env: + LINKEDIN_CLIENT_ID: ${{ secrets.LINKEDIN_CLIENT_ID }} + LINKEDIN_CLIENT_SECRET: ${{ secrets.LINKEDIN_CLIENT_SECRET }} + LINKEDIN_REFRESH_TOKEN: ${{ secrets.LINKEDIN_REFRESH_TOKEN }} + LINKEDIN_ACCESS_TOKEN: ${{ secrets.LINKEDIN_ACCESS_TOKEN }} + LINKEDIN_PERSON_URN: ${{ secrets.LINKEDIN_PERSON_URN }} + LINKEDIN_ORG_URN: ${{ secrets.LINKEDIN_ORG_URN }} + RELEASE_TAG: ${{ steps.notes.outputs.tag }} + RELEASE_BODY: ${{ steps.notes.outputs.body }} + REPO_URL: ${{ github.event.repository.html_url }} + DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }} + SKIP_ANNOUNCE: ${{ github.event.inputs.skip_announce || steps.notes.outputs.skip || 'false' }} + run: | + if [ "$SKIP_ANNOUNCE" = "true" ]; then + echo "skip-announce requested — skipping LinkedIn." + exit 0 + fi + if [ -z "$LINKEDIN_PERSON_URN" ] || [ -z "$RELEASE_TAG" ]; then + echo "LINKEDIN_PERSON_URN or RELEASE_TAG missing — skipping." + exit 0 + fi + if [ -z "$LINKEDIN_REFRESH_TOKEN" ] && [ -z "$LINKEDIN_ACCESS_TOKEN" ]; then + echo "LINKEDIN_REFRESH_TOKEN or LINKEDIN_ACCESS_TOKEN missing — skipping." + exit 0 + fi + dotnet run --project tools/linkedin-announcer -- post diff --git a/README.md b/README.md index 572ba4b..746f9a2 100644 --- a/README.md +++ b/README.md @@ -4,26 +4,85 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) [![CI](https://img.shields.io/github/actions/workflow/status/Kortexio/CodeForge/ci.yml?branch=master&label=CI)](https://github.com/Kortexio/CodeForge/actions) -**An open-source, model-independent, memory-native, agentic coding IDE.** +**The open-source AI coding IDE that makes a consumer GPU feel like a coding agent — not a chatbot with tools bolted on.** -CodeForge is an AI-native code editor built on [Code-OSS](https://github.com/microsoft/vscode). One Windows install, your own models (cloud or local), no Docker required. +Built on [Code-OSS](https://github.com/microsoft/vscode). Your models. Your machine. No Docker. No vendor lock-in. -Also discoverable via: *Cursor AI*, *Cursor-style AI IDE*, *open source AI coding IDE*, *Ollama coding agent*. +> **Download:** [Latest Windows installer](https://github.com/Kortexio/CodeForge/releases/latest) · **Source:** [`extensions/codeforge/`](extensions/codeforge/) -> **Download:** [Latest Windows installer](https://github.com/Kortexio/CodeForge/releases/latest) · **Source of truth:** [`extensions/codeforge/`](extensions/codeforge/) +--- + +## The AHA moment + +Most “local AI IDEs” stall when the model is small: they over-read, under-edit, or declare victory without verifying. **CodeForge fixes the agent loop itself** — read → edit → test → finish — and then adapts the harness to each model. + +On an **RTX 3060 12GB**, with **Qwen3.6-35B (IQ2_S)** via llama.cpp, the headless harness suite scored: + +| Task | Mean (3 reps) | Stuck | +|------|---------------:|------:| +| T1 — bugfix | **100** | 0 | +| T2 — feature (coupon) | **97** | 0 | +| T3 — multi-file | **100** | 0 | +| T4 — rename refactor | **100** | 0 | + +**Zero stuck runs. Zero bad tool-args.** A ~Q2 35B completing real agentic coding tasks end-to-end on a gaming GPU. + +That is the product bet: **the model is not the product — the contract around the model is.** + +Frontier APIs still work. Local weak models stop being a demo and start being a workflow. + +--- + +## What we actually built + +Not another chat sidebar. A **tool + system contract** that refuses to paper over failures with intent matchers. + +| Capability | Why it matters | +|---|---| +| **Agentic loop that finishes** | Unlimited exploration within the turn budget (Cursor/Claude Code style). Continue button when the step budget ends. No soft “explore cap” that refuses new reads. | +| **Adaptive harness** | Per-model profiles (`~/.CodeForge/ai/model-profiles/`) — sampling, write gates, finish limits, optional `toolTransport` for XML-leaky locals. Calibrate once; hysteresis adjusts online. | +| **Honest outcomes** | Stuck / truncated / oracle-red are **failed**, not “done”. Spec self-check and oracle before finish with writes. | +| **You own the freedoms** | Sandbox level, Allow all, outside-workspace writes, MCP, Full agent freedom — all user settings, revocable. The model cannot grant itself permissions. | +| **Measurable learning** | Harness revise proposes changes; keep only when paired bootstrap shows a real gain. No quiet contamination of `harness.json`. | +| **Same IDE for weak and strong** | Compensations are per model. A clean 32B does not pay the IQ2_S tax. | + +--- + +## Where we’re going + +**Near term** +- Ship releases that treat the adaptive harness + agent loop as the core product story +- Keep the calibrate suite (T2/T4/CS2/CS3) as the release gate for local models +- Harden permissions, sandbox, and secrets for daily use without babysitting + +**Next** +- Broader task suites (navigation, long multi-card, review) with the same failure taxonomy: harness / protocol / semantic +- Profile knobs proven by A/B (thinking off on tool turns, one mutating call per step) — only when the bench says so +- Packaging and onboarding so “install → pick local model → agent works” is a five-minute path + +**Not the roadmap** +- Hardcoded intent crutches (“the user said fix, so nudge edit”) +- One global “weak mode” that punishes strong models +- Claiming 100% on every task — residual failures stay visible as model limits, not hidden by prose + +--- ## Why CodeForge? | | CodeForge | |---|---| -| Models | OpenAI, Anthropic, Gemini, xAI, OpenRouter, Ollama, vLLM, LM Studio, any OpenAI-compatible API | -| Agent | Multi-file agent with approvals; composer modes Ask / Plan / Agent + permission chip (Default → Assisted → Allow all) | -| Chat UI | Cursor-like sidebar: footer composer, context ring, clean tool rows, Undo/Review, Restore checkpoints | -| Weak local models | Opt-in small-model harness (`codeforge.ai.weakModelMode`: on/off) | -| Sandbox | Configurable shell sandbox + optional Full agent freedom | -| Memory | Session wiki, project wiki, stable facts on disk (`~/.CodeForge/`) | -| Governance | Editable skills, rules, policies, hard guardrails | -| MCP | Inbound tools + outbound wiki server + Plugins catalog | +| Models | OpenAI, Anthropic, Gemini, xAI, OpenRouter, Ollama, vLLM, LM Studio, llama.cpp / any OpenAI-compatible API | +| Adaptive harness | Per-model profiles + calibrate + hysteresis | +| Agent | Multi-file loop with approvals; Ask / Plan / Agent + permission chip | +| Chat | Cursor-like sidebar: composer, context ring, tool rows, Undo/Review, Restore, Continue | +| Sandbox | User-configured shell sandbox + optional Full agent freedom | +| Memory | Session wiki, project wiki, stable facts | +| Governance | Skills, rules, policies, hard guardrails | +| MCP | Inbound tools + outbound wiki + Plugins catalog | + +Also discoverable as: *open source Cursor alternative*, *Ollama coding agent*, *llama.cpp coding IDE*, *local LLM agent IDE*. + +--- ## Install (Windows) @@ -33,126 +92,109 @@ Also discoverable via: *Cursor AI*, *Cursor-style AI IDE*, *open source AI codin Or build from source (below). -## Quick Start (development) +## Quick start (development) ```bash git clone https://github.com/Kortexio/CodeForge.git cd CodeForge npm install npm run vscode:setup # clone Code-OSS, brand, install deps -npm run build:extensions # compile AI extension +npm run build:extensions # sync calibrate suite + compile AI extension npm run vscode:compile # compile editor (first time is slow) npm run vscode:dev # launch CodeForge with AI sidebar ``` -See [docs/code-oss-integration.md](docs/code-oss-integration.md) for details. - -## Features - -- **Embedded AI Platform** — agent, chat, settings, and tools inside the IDE -- **Cursor-like composer** — Ask / Plan / Agent in the chat footer; Shift+Tab cycles mode; permissions chip separate from mode (Allow all ≈ former Auto) -- **Context ring** — live prompt tokens vs window limit on the composer -- **Tool timeline** — short Cursor-style labels (`Shell npm test`, `Read path`), exploration grouping, failures stay expanded -- **Edit journal** — Undo All / Review Changes (`vscode.diff` against pre-edit snapshots) -- **Restore checkpoint** — revert a user turn (messages + related tool traces + journaled files) -- **Shell sandbox** — `codeforge.ai.shellSandbox` (restricted → unrestricted) and **Full agent freedom** in Agent settings -- **Model Independence** — bring your own API keys or local servers -- **Plan mode** — explore + write a wiki plan without mutating source -- **Small-model harness** — opt-in phase tools + tighter guardrails for small/local LLMs -- **Persistent Memory** — session/project wiki and temporal facts -- **Code Intelligence** — hybrid search (lexical + semantic embeddings) -- **MCP + Plugins** — inbound tools, outbound wiki server, marketplace-style plugins catalog -- **Tab Completion** — inline completions with your model -- **Browser Agent** — visual checks (Playwright optional) -- **Git Intelligence** — status, diff, blame, conflicts, commit messages +Calibrate from the app: **CodeForge Settings → Benchmark**, or: + +```bash +node benchmarks/bench.mjs calibrate --model qwen3.6-35b --reps 3 +node benchmarks/bench.mjs run T2 --label smoke --model qwen3.6-35b +``` + +See [docs/code-oss-integration.md](docs/code-oss-integration.md) and [benchmarks/README.md](benchmarks/README.md). + +--- + +## Benchmarks (llama.cpp, RTX 3060 12GB) + +| | | +|---|---| +| **GPU** | NVIDIA GeForce RTX 3060 **12GB** | +| **Server** | llama.cpp OpenAI-compatible API | +| **Primary local model** | `qwen3.6-35b` (Qwen3.6-35B-A3B **IQ2_S**) | +| **Stronger local** | `Qwen3-32B-exl3-4.0bpw` (calibrate A/B) | +| **Runner** | `benchmarks/bench.mjs` headless / packaged `resources/calibrate-bench/` | + +### Agent suite — post harness fix (2026-10-22) + +T1–T4 × 3 reps, learned harness off for isolation. Score 0–100. + +| Model | T1 | T2 | T3 | T4 | Stuck | +|---|---:|---:|---:|---:|---:| +| `qwen3.6-35b` (IQ2_S) | **100** | **97** | **100** | **100** | **0** | + +### Calibration suite — 2026-10-20 + +T2 · T4 · CS2 · CS3 × 3 reps × 2 models. Protocol quality decides weak vs strong profile knobs — not the model name. Details: [`benchmarks/results/CALIBRATE-2026-10-20.md`](benchmarks/results/CALIBRATE-2026-10-20.md). + +Harness design notes: [`extensions/codeforge/HEURISTICS.md`](extensions/codeforge/HEURISTICS.md). + +--- ## Architecture ``` CodeForge/ -├── extensions/codeforge/ # Shipped AI platform (source of truth) -│ ├── src/agent/ # Agent loop, edit journal, tool registry, harness -│ ├── src/views/ # Chat host + chatViewHtml (Cursor-like UI) -│ ├── src/memory/ # Session + project wiki + temporal facts -│ ├── src/storage/ # ~/.CodeForge paths, sessions, artifacts, traces -│ ├── src/context/ # Context budget / ranking -│ ├── src/sandbox/ # Shell isolation + unrestricted / freedom -│ ├── src/intelligence/ # Index, LSP, git, embeddings -│ ├── src/browser/ # Browser agent -│ ├── src/mcp/ # Inbound + outbound MCP -│ └── src/governance/ # Skills, rules, guardrails, HITL -├── docs/ # Documentation -├── tests/ # Test suites -└── scripts/ # Code-OSS setup / package +├── extensions/codeforge/ # Shipped AI platform (source of truth) +│ ├── src/agent/ # Loop, profiles, telemetry, revise, tools +│ ├── resources/calibrate-bench/ # Packaged calibrate suite +│ ├── src/views/ # Chat host + Cursor-like UI +│ ├── src/memory/ # Session + project wiki +│ ├── src/sandbox/ # Shell isolation +│ ├── src/intelligence/ # Index, LSP, git, embeddings +│ ├── src/mcp/ # Inbound + outbound MCP +│ └── src/governance/ # Skills, rules, guardrails +├── benchmarks/ # Full harness suite + CLI +├── docs/ +└── scripts/ # Code-OSS setup / package / sync ``` -See [docs/architecture.md](docs/architecture.md) for the full component model. - ## Configuration -Prefer **CodeForge Settings** (Models / MCP / Guardrails / Agent) in the app. - -Legacy mirrors (active server only): +Prefer **CodeForge Settings** in the app (Models / Permissions / MCP / Guardrails / Agent). ```json { "codeforge.ai.mode": "agent", - "codeforge.ai.weakModelMode": "off", "codeforge.ai.shellSandbox": "restricted", + "codeforge.ai.allowAll": false, + "codeforge.ai.outsideWorkspaceWrites": "ask", "codeforge.ai.fullAgentFreedom": false } ``` -Supported providers: OpenAI, Anthropic, Google (Gemini), xAI, OpenRouter, Ollama, vLLM, LM Studio, custom OpenAI-compatible endpoints. +Profiles live per machine under `~/.CodeForge/ai/model-profiles/` — calibrate once per model. ## Development -### Prerequisites - -- Node.js matching `vscode/.nvmrc` (for full IDE builds) -- npm, Git -- On Windows: Visual Studio C++ workload + Spectre-mitigated libs - -### Building & tests +**Prerequisites:** Node matching `vscode/.nvmrc`, npm, Git. On Windows: VS C++ workload + Spectre-mitigated libs. ```bash -npm install npm run build:extensions npm test +npm run vscode:package:win # needs Inno Setup ``` -### Packaging (Windows) - -```bash -# Installer + portable + zip (recommended for distribution) -npm run vscode:package:win - -# Installer only (from existing portable build) -npm run installer:win -``` - -Requires Inno Setup: `winget install JRSoftware.InnoSetup` - -Artifact: `dist/codeforge-win32-x64/CodeForge-Setup--win32-x64.exe` - -## Discoverability - -Search on GitHub for **CodeForge**, **AI coding IDE**, **Cursor AI**, or topics below. - -- Repo: [github.com/Kortexio/CodeForge](https://github.com/Kortexio/CodeForge) -- Releases: [github.com/Kortexio/CodeForge/releases](https://github.com/Kortexio/CodeForge/releases) -- Topics: `ide` · `ai` · `vscode` · `code-oss` · `openai` · `ollama` · `agent` · `mcp` · `electron` · `typescript` · `cursor` · `cursor-ai` · `ai-ide` · `ai-coding` · `coding-agent` · `llm` - ## License -MIT License — see [LICENSE](LICENSE) for details. +MIT — see [LICENSE](LICENSE). ## Contributing -Contributions are welcome! Please read our [Contributing Guide](CONTRIBUTING.md) before submitting a Pull Request. +See [Contributing Guide](CONTRIBUTING.md). ## Acknowledgments -- Built on [Code-OSS](https://github.com/microsoft/vscode), the open-source foundation of Visual Studio Code -- Inspired by modern AI coding assistants -- Powered by open AI research and models +- Built on [Code-OSS](https://github.com/microsoft/vscode) +- Validated on consumer GPUs (RTX 3060 12GB + llama.cpp) and cloud APIs +- The agent loop is measured, not marketed — failure classes stay visible diff --git a/benchmarks/README.md b/benchmarks/README.md index 7ccb950..30112f3 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -22,9 +22,17 @@ npm run bench -- selftest CS # validates fixtures + soluti npm run build:extensions node benchmarks/bench.mjs run T1 --label baseline-qwen35b # Optional: --server --model --weak off + +# Calibrate one model (harness classifies weak vs strong from protocol) +node benchmarks/bench.mjs calibrate --model qwen3.6-35b --reps 3 +# Optional lab A/B: +node benchmarks/bench.mjs calibrate --weak-model qwen3.6-35b --strong-model Qwen3-32B-exl3-4.0bpw --reps 3 ``` -Headless results include `loopExit`, `steps`, `refusals`, `protocol` (parse/tool-name failures), and `toolCallMode` (`native` | `content` | `none`). +In the app: **CodeForge Settings → Benchmark** (pick Model → Run). Suite packaged at `resources/calibrate-bench/` via `npm run sync:calibrate-bench`. + + +Headless results include `loopExit`, `steps`, `refusals`, `protocol` (parse/tool-name failures), and `toolCallMode` (`native` | `content` | `none`). Protocol counters are also appended to `~/.CodeForge/ai/model-profiles///runs.jsonl` (rollup in `rollup.json`). Score (0–100): hidden 60 · visible 15 · protected untouched 10 · checks 10 · no junk files 5. diff --git a/benchmarks/bench.mjs b/benchmarks/bench.mjs index 3dc77f3..16c254e 100644 --- a/benchmarks/bench.mjs +++ b/benchmarks/bench.mjs @@ -17,9 +17,11 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { spawn, spawnSync } from 'node:child_process'; -import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; +import { resolveBenchLayout } from './layout.mjs'; -const HERE = path.dirname(fileURLToPath(import.meta.url)); +const LAYOUT = resolveBenchLayout(); +const HERE = LAYOUT.here; const FIXTURES = { shop: path.join(HERE, 'fixtures', 'shop'), billing: path.join(HERE, 'fixtures', 'billing'), @@ -28,7 +30,8 @@ const FIXTURES = { 'router-cards': path.join(HERE, 'fixtures', 'router-cards'), }; const TASKS_DIR = path.join(HERE, 'tasks'); -const RESULTS_DIR = path.join(HERE, 'results'); +const RESULTS_DIR = + process.env.CODEFORGE_BENCH_RESULTS?.trim() || LAYOUT.resultsDir; let RUNS_ROOT = process.env.CODEFORGE_BENCH_DIR || path.join(os.homedir(), 'codeforge-bench'); const CF_HOME = path.join(os.homedir(), '.CodeForge'); const HIDDEN_DIR = '.bench-hidden'; @@ -511,15 +514,13 @@ function runVisibleAndHidden(dir, task) { }; return { visible, hidden, runtime }; } - if (hasHidden) { - fs.rmSync(hiddenTarget, { recursive: true, force: true }); - copyFixture(hiddenSrc, hiddenTarget); - } const visible = runDotnetTests(dir, { filter: hasHidden ? 'FullyQualifiedName!~BenchHidden' : undefined, }); let hidden = { pass: 0, fail: 0, output: '(no hidden)', skipped: !hasHidden }; if (hasHidden) { + fs.rmSync(hiddenTarget, { recursive: true, force: true }); + copyFixture(hiddenSrc, hiddenTarget); hidden = runDotnetTests(dir, { filter: 'FullyQualifiedName~BenchHidden' }); fs.rmSync(hiddenTarget, { recursive: true, force: true }); } else if (task.category === 'scaffold') { @@ -666,14 +667,16 @@ function cmdRun(taskKey, flags) { const outJson = path.join(RESULTS_DIR, `${meta.name}.headless.json`); fs.mkdirSync(RESULTS_DIR, { recursive: true }); - const compile = spawnSync('npm', ['run', 'build:extensions'], { - cwd: path.join(HERE, '..'), - encoding: 'utf8', - shell: true, - timeout: 300_000, - }); - if (compile.status !== 0) { - die(`build:extensions failed:\n${compile.stdout}\n${compile.stderr}`); + if (!flags.skipBuild && !LAYOUT.packaged) { + const compile = spawnSync('npm', ['run', 'build:extensions'], { + cwd: LAYOUT.repoRoot, + encoding: 'utf8', + shell: true, + timeout: 300_000, + }); + if (compile.status !== 0) { + die(`build:extensions failed:\n${compile.stdout}\n${compile.stderr}`); + } } const args = [ @@ -694,22 +697,61 @@ function cmdRun(taskKey, flags) { if (flags.maxSteps) args.push('--maxSteps', String(flags.maxSteps)); if (flags.hardCap) args.push('--hardCap', String(flags.hardCap)); if (flags.sampling) args.push('--sampling', String(flags.sampling)); + args.push('--profileHome', path.join(os.homedir(), '.CodeForge')); console.log(`Headless run ${meta.name}…`); + const agentEnv = { + ...process.env, + CODEFORGE_EXT_OUT: LAYOUT.extOut, + NODE_PATH: [ + path.join(LAYOUT.extRoot, 'node_modules'), + path.join(LAYOUT.repoRoot, 'node_modules'), + process.env.NODE_PATH || '', + ] + .filter(Boolean) + .join(path.delimiter), + }; + if (flags.disableHysteresis || flags.calibrate) { + agentEnv.CODEFORGE_DISABLE_HYSTERESIS = '1'; + } + if (flags.harnessFile) { + agentEnv.CODEFORGE_HARNESS_FILE = path.resolve(String(flags.harnessFile)); + } const agent = spawnSync(process.execPath, args, { - cwd: path.join(HERE, '..'), + cwd: LAYOUT.repoRoot, encoding: 'utf8', timeout: 3_600_000, + env: agentEnv, }); if (agent.status === 2) { process.stdout.write(agent.stdout || ''); process.stderr.write(agent.stderr || ''); + if (flags.softFail) { + return { + pass: false, + score: 0, + infrastructureError: true, + notRun: true, + loopExit: 'error', + task: taskKey, + }; + } process.exit(2); } if (agent.status !== 0) { + if (flags.softFail) { + return { + pass: false, + score: 0, + infrastructureError: true, + notRun: true, + loopExit: 'error', + task: taskKey, + }; + } console.error(agent.stdout); console.error(agent.stderr); - die(`headless agent exited ${agent.status}`); + throw new Error(`headless agent exited ${agent.status}`); } process.stdout.write(agent.stdout || ''); @@ -727,6 +769,7 @@ function cmdRun(taskKey, flags) { toolCallMode: headless.toolCallMode, elapsedMs: headless.elapsedMs, headlessReply: headless.replyPreview, + replyTail: headless.replyTail, }; const resultPath = path.join(RESULTS_DIR, `${meta.name}.json`); fs.writeFileSync(resultPath, JSON.stringify(merged, null, 2)); @@ -768,10 +811,39 @@ function cmdSelftest(filter) { if (!ok) process.exit(1); } +/** + * Classify a failed (or suspicious) run from structured fields — not reply text. + * A = harness (no writes / stuck / refusals), B = protocol (badArgs/normalize/truncated), + * C = semantic (wrote + oracle green but hidden red). + */ +function classifyFailure(r) { + if (r.pass && !r.falseDone) return null; + const writes = Number(r.session?.writes ?? r.changedFiles?.length ?? 0); + const exit = String(r.loopExit || ''); + const refusals = r.refusals || {}; + const refusalN = Object.values(refusals).reduce((a, b) => a + Number(b || 0), 0); + const proto = r.protocol || {}; + const badArgs = Number(proto.badArgs || 0); + const normalizeErr = Number(proto.normalize_error || proto.normalizeError || 0); + const truncated = Number(proto.lengthExits || proto.truncated_tool_json || 0); + const classes = []; + if (writes === 0 || exit === 'stuck' || refusalN > 0) classes.push('A-harness'); + if (badArgs > 0 || normalizeErr > 0 || truncated > 0 || exit === 'length') classes.push('B-protocol'); + if ( + writes > 0 && + (r.falseDone || (Number(r.hidden?.fail || 0) > 0 && !r.falseDone)) + ) { + classes.push('C-semantic'); + } + if (!classes.length) classes.push(r.pass ? 'ok' : 'unknown'); + return classes.join('+'); +} + function printResult(r) { const s = r.session; + const failClass = classifyFailure(r); console.log( - `${r.pass ? 'PASS' : 'FAIL'} ${r.run} score ${r.score}/100 (${r.model || 'no session found'}${r.runtime ? ` · ${r.runtime}` : ''})` + `${r.pass ? 'PASS' : 'FAIL'} ${r.run} score ${r.score}/100 (${r.model || 'no session found'}${r.runtime ? ` · ${r.runtime}` : ''}${failClass ? ` · ${failClass}` : ''})` ); if (r.review) { const v = r.review; @@ -831,6 +903,8 @@ function cmdReport() { model: (r.model || '-').slice(0, 18), pass: r.pass ? 'PASS' : 'fail', score: r.score, + class: classifyFailure(r) || (r.pass ? '' : '?'), + exit: r.loopExit || '', hidden: r.review ? `F1 ${r.review.f1}` : `${r.hidden.pass}/${r.hidden.pass + r.hidden.fail}`, cards: r.hidden.cards ? r.hidden.cards.filter(c => c.ratio === 1).length + `/${r.hidden.cards.length}` : '', falseDone: r.falseDone ? 'YES' : '', @@ -841,6 +915,487 @@ function cmdReport() { min: Math.round(r.session.wallSeconds / 60), })); console.table(rows); + const tallies = { 'A-harness': 0, 'B-protocol': 0, 'C-semantic': 0 }; + for (const r of results) { + const c = classifyFailure(r); + if (!c) continue; + if (c.includes('A-harness')) tallies['A-harness'] += 1; + if (c.includes('B-protocol')) tallies['B-protocol'] += 1; + if (c.includes('C-semantic')) tallies['C-semantic'] += 1; + } + console.log( + `Failure classes: A-harness=${tallies['A-harness']} B-protocol=${tallies['B-protocol']} C-semantic=${tallies['C-semantic']}` + ); +} + +function modelSafe(raw) { + return String(raw ?? '') + .trim() + .replace(/[^\w.-]+/g, '_') + .replace(/^_+|_+$/g, '') + .slice(0, 120) || 'unknown'; +} + +/** + * Calibrate one model (UI / --model) or A/B weak+strong (lab CLI). + * Classification is from protocol stats, not from the model name. + */ +function cmdCalibrate(flags) { + const settings = readJson(path.join(os.homedir(), '.CodeForge', 'ai', 'settings.json')) || {}; + const serverId = String(flags.server || settings.activeServerId || 'default'); + const singleModel = String(flags.model || '').trim(); + const weakFlag = String(flags['weak-model'] || flags.weakModel || '').trim(); + const strongFlag = String(flags['strong-model'] || flags.strongModel || '').trim(); + const models = singleModel + ? [singleModel] + : weakFlag && strongFlag + ? [weakFlag, strongFlag] + : [ + settings.activeModel || weakFlag || 'qwen3.6-35b', + ].filter(Boolean); + if (!models.length) die('calibrate needs --model (or --weak-model + --strong-model)'); + const reps = Math.max(1, Number(flags.reps || 3) || 3); + const suite = ['T2', 'T4', 'CS2', 'CS3']; + + if (!flags.skipBuild && !LAYOUT.packaged) { + const compile = spawnSync( + 'npx', + ['tsc', '-p', 'extensions/codeforge/tsconfig.json'], + { + cwd: LAYOUT.repoRoot, + encoding: 'utf8', + shell: true, + timeout: 300_000, + } + ); + if (compile.status !== 0) { + die(`tsc failed:\n${compile.stdout}\n${compile.stderr}`); + } + } + + const require = createRequire(import.meta.url); + const { + loadModelProfile, + saveModelProfile, + applyCalibrationFromStats, + } = require(path.join(LAYOUT.extOut, 'agent', 'modelProfile.js')); + + const byModel = {}; + for (const model of models) { + const results = []; + const labelBase = `calibrate-${modelSafe(model)}`; + for (let rep = 1; rep <= reps; rep++) { + for (const task of suite) { + console.log(`\n=== calibrate ${model} ${task} rep ${rep}/${reps} ===`); + try { + const r = cmdRun(task, { + ...flags, + label: `${labelBase}-r${rep}`, + model, + server: serverId, + skipBuild: true, + disableHysteresis: true, + // Session weak harness on during calibrate so the suite matches product weak path; + // profile knobs after classify decide long-term cost. + weak: 'on', + }); + results.push(r); + } catch (err) { + console.error(err); + results.push({ pass: false, score: 0, protocol: {}, task }); + } + } + } + byModel[model] = results; + } + + const date = new Date().toISOString().slice(0, 10); + const lines = [ + `# Calibration — ${date}`, + '', + `Models: ${models.map(m => `\`${m}\``).join(', ')} · reps=${reps} · suite=${suite.join(',')}`, + '', + '| Model | Class | Pass | Mean score | badArgs/native | lengthExits | Knobs |', + '|---|---|---:|---:|---:|---:|---|', + ]; + + for (const model of models) { + const results = byModel[model] || []; + const n = results.length || 1; + const passes = results.filter(r => r.pass).length; + const meanScore = results.reduce((s, r) => s + (Number(r.score) || 0), 0) / n; + let badArgs = 0; + let native = 0; + let lengthExits = 0; + for (const r of results) { + const p = r.protocol || {}; + badArgs += Number(p.badArgs) || 0; + native += Number(p.nativeToolCalls) || 0; + lengthExits += Number(p.lengthExits) || 0; + } + const badArgsRate = badArgs / Math.max(1, native); + const passRate = passes / n; + let profile = loadModelProfile(serverId, model, { applyHysteresis: false }); + profile.calibration = { + at: new Date().toISOString(), + suite: suite.join(','), + passRate, + meanScore, + protocolMeans: { + badArgs: badArgs / n, + nativeToolCalls: native / n, + lengthExits: lengthExits / n, + }, + }; + const { profile: next, classification } = applyCalibrationFromStats(profile, { + passRate, + badArgsRate, + lengthExits, + totalRuns: n, + nativeToolCalls: native, + }); + profile = next; + profile.calibration.classification = classification; + // Prefer text transport when native parse leaks arguments (Qwen XML closers). + const refusals = results.reduce((s, r) => { + const ref = r.refusals || {}; + return s + Object.values(ref).reduce((a, b) => a + Number(b || 0), 0); + }, 0); + if (badArgsRate >= 0.1 || (native > 0 && refusals / Math.max(1, native) >= 0.1)) { + profile.toolTransport = 'text'; + } else if (classification === 'strong' && badArgsRate < 0.05) { + profile.toolTransport = 'native'; + } + saveModelProfile(profile); + const knobs = + classification === 'strong' + ? `strong: writeMax=${profile.mutations.writeMaxContentLines}, massRewrite=${profile.mutations.enforceMassRewriteGate}, weakSkill=${profile.prompt.forcedWeakSkill}, transport=${profile.toolTransport || 'native'}` + : `weak: defaults, transport=${profile.toolTransport || 'native'}`; + lines.push( + `| ${model} | ${classification} | ${passes}/${n} | ${meanScore.toFixed(1)} | ${badArgsRate.toFixed(3)} | ${lengthExits} | ${knobs.replace(/\|/g, '/')} |` + ); + } + + lines.push(''); + lines.push( + 'Class **strong** when `passRate ≥ 0.5`, `badArgs/native < 0.05`, `lengthExits === 0`, and native tool calls > 0; otherwise **weak**. High badArgs → `toolTransport=text`.' + ); + lines.push('Profiles: `~/.CodeForge/ai/model-profiles///profile.json`.'); + const outMd = path.join(RESULTS_DIR, `CALIBRATE-${date}.md`); + fs.mkdirSync(RESULTS_DIR, { recursive: true }); + fs.writeFileSync(outMd, lines.join('\n') + '\n', 'utf8'); + console.log(lines.join('\n')); + console.log(`\nWrote ${outMd}`); +} + +function isInfrastructureError(r) { + return !!(r?.infrastructureError || r?.notRun || r?.loopExit === 'error'); +} + +function emptyPairedByTask(tasks) { + return Object.fromEntries(tasks.map(t => [t, { parent: [], candidate: [] }])); +} + +async function probeReviseLlm(server, model) { + try { + const res = await fetch(`${String(server.baseUrl).replace(/\/$/, '')}/chat/completions`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(server.apiKey ? { Authorization: `Bearer ${server.apiKey}` } : {}), + }, + body: JSON.stringify({ + model, + max_tokens: 1, + messages: [{ role: 'user', content: 'ping' }], + }), + signal: AbortSignal.timeout(15_000), + }); + return res.ok; + } catch { + return false; + } +} + +function writeReviseDeferredSnapshot(rev, serverId, model, reason) { + const dir = path.dirname(rev.harnessFilePath(serverId, model)); + fs.mkdirSync(dir, { recursive: true }); + const snap = { + at: new Date().toISOString(), + note: 'Harness revise deferred — local LLM was not reachable (same pattern as offline baselines).', + reason, + }; + const fp = path.join(dir, 'revise-deferred.json'); + fs.writeFileSync(fp, JSON.stringify(snap, null, 2), 'utf8'); + console.log(`Deferred harness revision (no live LLM). Wrote ${fp}`); + return fp; +} + +/** + * One test-time harness revision. The same model proposes; T2/T4 score the + * change (CS2/CS3 stay held out). harness.json updates only when a keep survives + * holdout checks. + */ +async function cmdRevise(flags) { + const settings = readJson(path.join(os.homedir(), '.CodeForge', 'ai', 'settings.json')) || {}; + const serverId = String(flags.server || settings.activeServerId || 'default'); + const model = String( + flags.model || + (settings.activeServerId === serverId ? settings.activeModel : '') || + '' + ).trim(); + if (!model) die('revise needs --model (or an active model in settings)'); + const repsEach = Math.max(1, Number(flags.reps || 3) || 3); + const suite = String(flags.tasks || 'T2,T4') + .split(',') + .map(s => s.trim()) + .filter(Boolean); + if (!suite.length) die('revise needs at least one task'); + + const server = (settings.servers || []).find(s => s.id === serverId && s.enabled !== false); + if (!server?.baseUrl?.trim()) die(`no usable server (id=${serverId})`); + + const require = createRequire(import.meta.url); + // Probe LLM before compile / score-count gates so offline runs defer immediately. + const revEarly = (() => { + try { + return require(path.join(LAYOUT.extOut, 'agent', 'harnessRevision.js')); + } catch { + return null; + } + })(); + const llmOkEarly = await probeReviseLlm(server, model); + if (!llmOkEarly) { + if (revEarly) { + writeReviseDeferredSnapshot(revEarly, serverId, model, 'LLM probe failed'); + return; + } + die('LLM probe failed and harnessRevision.js is not built (run build:extensions)'); + } + + if (repsEach * suite.length < 3) { + die( + `revise needs at least 3 scores (tasks × reps). Got ${suite.length}×${repsEach}. One run is noise.` + ); + } + + if (!flags.skipBuild && !LAYOUT.packaged) { + const compile = spawnSync('npx', ['tsc', '-p', 'extensions/codeforge/tsconfig.json'], { + cwd: LAYOUT.repoRoot, + encoding: 'utf8', + shell: true, + timeout: 300_000, + }); + if (compile.status !== 0) die(`tsc failed:\n${compile.stdout}\n${compile.stderr}`); + } + + const rev = revEarly || require(path.join(LAYOUT.extOut, 'agent', 'harnessRevision.js')); + const telemetry = require(path.join(LAYOUT.extOut, 'agent', 'protocolTelemetry.js')); + const file = rev.harnessFilePath(serverId, model); + const parent = rev.loadHarnessArtifact(serverId, model, file); + const report = rev.formatExecutionReport(telemetry.loadModelRollup(serverId, model)); + console.log(`Harness revision for ${model} on ${server.name || serverId}`); + console.log(`Selection suite: ${suite.join(', ')} × ${repsEach} (held out of this decision: tasks not listed).`); + console.log(report); + + const parentOutcomes = []; + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cf-revise-harness-')); + const taskKeys = suite.map(k => findTask(k).id.split('-')[0]); + + const benchRun = (taskKey, labelSuffix, harnessFile) => { + const r = cmdRun(taskKey, { + ...flags, + model, + server: serverId, + label: labelSuffix, + skipBuild: true, + disableHysteresis: true, + softFail: true, + harnessFile, + }); + if (isInfrastructureError(r)) { + console.warn(`Skipping infrastructure error for ${taskKey} (${labelSuffix})`); + return undefined; + } + const score = Number(r?.score); + if (!Number.isFinite(score)) { + throw new Error(`no score for ${taskKey} (${labelSuffix})`); + } + return { score, r, taskKey }; + }; + + const harnessFileFor = artifact => { + if (artifact.revision === parent.revision) return file; + const fp = path.join(tempDir, `harness-r${artifact.revision}.json`); + rev.saveHarnessArtifact(artifact, fp); + return fp; + }; + + const pushOutcome = (taskId, r, score) => { + parentOutcomes.push({ + task: taskId, + score, + loopExit: r.loopExit || undefined, + steps: typeof r.steps === 'number' ? r.steps : undefined, + hiddenFailed: r.hidden?.failed ?? [], + checksFailed: (r.checks ?? []) + .filter(c => !c.ok) + .map(c => [c.why || c.type, c.detail].filter(Boolean).join(' ')), + replyTail: + typeof r.replyTail === 'string' + ? r.replyTail + : typeof r.headlessReply === 'string' + ? r.headlessReply.slice(-500) + : undefined, + }); + }; + + const scorePhase = async input => { + const byTask = emptyPairedByTask(taskKeys); + if (input.phase === 'parent') { + for (const task of suite) { + const taskId = findTask(task).id; + const short = taskId.split('-')[0]; + for (let rep = 1; rep <= repsEach; rep++) { + console.log(`\n=== revise parent ${short} rep ${rep}/${repsEach} ===`); + const run = benchRun( + task, + `revise-parent-${short}-rep${rep}`, + harnessFileFor(input.artifact) + ); + if (!run) continue; + byTask[short].parent.push(run.score); + pushOutcome(taskId, run.r, run.score); + } + } + const text = rev.formatTaskOutcomes(parentOutcomes); + if (text) console.log(`\n${text}`); + return { byTask }; + } + + for (const task of suite) { + const taskId = findTask(task).id; + const short = taskId.split('-')[0]; + for (let rep = 1; rep <= repsEach; rep++) { + console.log(`\n=== revise paired ${short} rep ${rep}/${repsEach} (parent) ===`); + const parentRun = benchRun( + task, + `revise-paired-parent-${short}-rep${rep}`, + harnessFileFor(input.parent) + ); + if (!parentRun) continue; + byTask[short].parent.push(parentRun.score); + + console.log(`\n=== revise paired ${short} rep ${rep}/${repsEach} (candidate r${input.candidate.revision}) ===`); + const candRun = benchRun( + task, + `revise-paired-cand-r${input.candidate.revision}-${short}-rep${rep}`, + harnessFileFor(input.candidate) + ); + if (!candRun) continue; + byTask[short].candidate.push(candRun.score); + } + } + return { byTask }; + }; + + const scoreHoldout = async (artifact, holdoutTasks, labelPrefix, side) => { + const byTask = emptyPairedByTask(holdoutTasks.map(t => findTask(t).id.split('-')[0])); + for (const task of holdoutTasks) { + const short = findTask(task).id.split('-')[0]; + for (let rep = 1; rep <= repsEach; rep++) { + console.log(`\n=== ${labelPrefix} holdout ${short} rep ${rep}/${repsEach} ===`); + const run = benchRun( + task, + `${labelPrefix}-holdout-${short}-rep${rep}`, + harnessFileFor(artifact) + ); + if (!run) continue; + byTask[short][side].push(run.score); + } + } + return byTask; + }; + + try { + let outcome = await rev.executeHarnessRevision({ + artifact: parent, + report, + minRepsPerTask: 3, + parentOutcomes: () => parentOutcomes, + propose: async messages => { + const res = await fetch(`${String(server.baseUrl).replace(/\/$/, '')}/chat/completions`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(server.apiKey ? { Authorization: `Bearer ${server.apiKey}` } : {}), + }, + body: JSON.stringify({ + model, + temperature: 0.2, + max_tokens: 4096, + enable_thinking: false, + chat_template_kwargs: { enable_thinking: false }, + messages, + }), + }); + const body = await res.text(); + if (!res.ok) throw new Error(`proposer HTTP ${res.status}: ${body.slice(0, 500)}`); + const data = JSON.parse(body); + const message = data?.choices?.[0]?.message ?? {}; + const content = [message.content, message.reasoning_content, message.reasoning] + .filter(part => typeof part === 'string' && part.trim()) + .join('\n'); + if (!content.trim()) throw new Error('proposer returned an empty message'); + console.log(`Proposer reply:\n${content.slice(0, 1500)}`); + return content; + }, + score: scorePhase, + }); + + if (outcome.kept) { + const holdout = ['CS2', 'CS3']; + console.log(`\nHoldout validation: ${holdout.join(', ')} × ${repsEach}`); + const parentHold = await scoreHoldout(parent, holdout, 'parent', 'parent'); + const candHold = await scoreHoldout(outcome.artifact, holdout, 'candidate', 'candidate'); + const holdReason = rev.holdoutRegressed(parentHold, candHold); + if (holdReason) { + console.log(`Holdout failed: ${holdReason}`); + const entry = { + at: new Date().toISOString(), + decision: 'revert', + reason: `holdout: ${holdReason}`, + diagnosis: outcome.diagnosis, + parentMean: outcome.parentMean, + candidateMean: outcome.candidateMean, + parentReps: outcome.artifact.history.at(-1)?.parentReps ?? 0, + candidateReps: outcome.artifact.history.at(-1)?.candidateReps ?? 0, + }; + outcome = { + ...outcome, + kept: false, + reason: entry.reason, + artifact: { + ...parent, + history: [...parent.history, entry].slice(-20), + }, + }; + } + } + + if (outcome.kept) { + rev.saveHarnessArtifact(outcome.artifact, file); + } else if (outcome.scored || outcome.artifact.history.length > parent.history.length) { + rev.saveHarnessArtifact(outcome.artifact, file); + } + console.log( + `\n${outcome.kept ? 'KEPT' : 'REVERTED'} revision ${outcome.kept ? outcome.artifact.revision : parent.revision}: ${outcome.reason}` + ); + if (outcome.diagnosis) console.log(`Diagnosis: ${outcome.diagnosis}`); + console.log(`Harness file: ${file}`); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } } // --------------------------------------------------------------------------- @@ -870,8 +1425,17 @@ switch (pos[0]) { case 'run': cmdRun(pos[1], flags); break; + case 'calibrate': + cmdCalibrate(flags); + break; + case 'revise': + cmdRevise(flags).catch(err => { + console.error(err instanceof Error ? err.message : err); + process.exit(1); + }); + break; default: console.log( - 'Usage: node benchmarks/bench.mjs --label [--open] | check [run|latest] | report | selftest [CS] | run --label >' + 'Usage: node benchmarks/bench.mjs --label [--open] | check [run|latest] | report | selftest [CS] | run --label | calibrate [--model m | --weak-model m --strong-model m] [--reps n] | revise [--model m] [--reps n] [--tasks T2,T4]>' ); } diff --git a/benchmarks/headlessAgent.mjs b/benchmarks/headlessAgent.mjs index 15ea143..33c85f6 100644 --- a/benchmarks/headlessAgent.mjs +++ b/benchmarks/headlessAgent.mjs @@ -9,13 +9,14 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import Module from 'node:module'; -import { fileURLToPath } from 'node:url'; import { createRequire } from 'node:module'; +import { resolveBenchLayout } from './layout.mjs'; -const HERE = path.dirname(fileURLToPath(import.meta.url)); -const REPO = path.resolve(HERE, '..'); -const EXT_OUT = path.join(REPO, 'extensions', 'codeforge', 'out'); -const VSCODE_MOCK = path.join(REPO, 'tests', 'support', 'vscodeFsMock.js'); +const LAYOUT = resolveBenchLayout(); +const HERE = LAYOUT.here; +const REPO = LAYOUT.repoRoot; +const EXT_OUT = process.env.CODEFORGE_EXT_OUT?.trim() || LAYOUT.extOut; +const VSCODE_MOCK = LAYOUT.vscodeMock; const SETTINGS_FILE = path.join(os.homedir(), '.CodeForge', 'ai', 'settings.json'); function parseArgs(argv) { @@ -255,27 +256,20 @@ async function main() { } const activities = []; - const protocol = { - nativeToolCalls: 0, - contentToolCalls: 0, - badArgs: 0, - unknownTools: 0, - lengthExits: 0, - emptyReplies: 0, - systemChars: 0, - toolsChars: 0, - }; + const { createProtocolAggregator, flushRunTelemetry } = require( + path.join(EXT_OUT, 'agent', 'protocolTelemetry.js') + ); + const protocolAgg = createProtocolAggregator(); const samplesDir = flags.samplesDir ? path.resolve(String(flags.samplesDir)) : path.join(runHome, 'protocol-samples'); fs.mkdirSync(samplesDir, { recursive: true }); let sampleN = 0; - - const bump = (key, n = 1) => { - if (Object.prototype.hasOwnProperty.call(protocol, key)) { - protocol[key] += n; - } - }; + if (flags.profileHome) { + process.env.CODEFORGE_MODEL_PROFILE_HOME = path.resolve(String(flags.profileHome)); + } else if (!process.env.CODEFORGE_MODEL_PROFILE_HOME) { + process.env.CODEFORGE_MODEL_PROFILE_HOME = path.join(os.homedir(), '.CodeForge'); + } const started = Date.now(); let reply = ''; @@ -300,50 +294,43 @@ async function main() { weakProfile: weak, isolated: true, sampling, + serverId: llm.serverId, onActivity: ev => { activities.push(ev); - if (ev.kind === 'protocol') { - const label = String(ev.label || ''); - if (label === 'sample') { - sampleN += 1; - const file = path.join(samplesDir, `${String(sampleN).padStart(3, '0')}.json`); - try { - const parsed = JSON.parse(String(ev.detail || '{}')); - fs.writeFileSync( - file, - JSON.stringify( - { - at: new Date().toISOString(), - model: llm.model, - server: llm.serverName, - ...parsed, - }, - null, - 2 - ) - ); - } catch { - fs.writeFileSync(file, String(ev.detail || ''), 'utf8'); - } - return; - } - if (label === 'systemChars' || label === 'toolsChars') { - protocol[label] = Number(ev.detail || 0) || 0; - return; + protocolAgg.observe(ev); + if (ev.kind === 'protocol' && ev.label === 'sample') { + sampleN += 1; + const file = path.join(samplesDir, `${String(sampleN).padStart(3, '0')}.json`); + try { + const parsed = JSON.parse(String(ev.detail || '{}')); + fs.writeFileSync( + file, + JSON.stringify( + { + at: new Date().toISOString(), + model: llm.model, + server: llm.serverName, + ...parsed, + }, + null, + 2 + ) + ); + } catch { + fs.writeFileSync(file, String(ev.detail || ''), 'utf8'); } - bump(label, Number(ev.detail || 1) || 1); - return; } - const blob = `${ev.label || ''} ${ev.detail || ''}`; - if (/bad-args|invalid tool arguments/i.test(blob)) bump('badArgs'); - if (/no tool named|Unknown bridge tool|not available/i.test(blob)) bump('unknownTools'); - if (/Parsed tool_calls from text/i.test(ev.label || '')) bump('contentToolCalls'); }, onStatus: text => { + // Fallback only — prefer onLoopExit when the agent reports it. + if (loopExit !== 'unknown') return; if (/stuck/i.test(text)) loopExit = 'stuck'; else if (/budget|maximum steps|step budget/i.test(text)) loopExit = 'budget'; else if (/length|truncat/i.test(text)) loopExit = 'length'; }, + onLoopExit: exit => { + loopExit = exit; + }, }; // Same path as chat: multi-card tasks run one isolated sub-run per card. const orchestrated = await runOrchestrated(agentOpts, { forced: false }); @@ -373,6 +360,21 @@ async function main() { } } + const elapsedMs = Date.now() - started; + const protocolSnap = protocolAgg.snapshot(); + try { + flushRunTelemetry({ + serverId: llm.serverId, + model: llm.model, + source: 'bench', + loopExit, + elapsedMs, + aggregator: protocolAgg, + }); + } catch { + /* best-effort */ + } + const result = { runtime: 'headless', model: llm.model, @@ -382,16 +384,17 @@ async function main() { numCtx, toolCallMode: probe.toolCallMode, toolCallModeDetail: probe.detail, - elapsedMs: Date.now() - started, + elapsedMs, loopExit, steps: activities.filter(a => a.kind === 'tool').length, refusals, - protocol, + protocol: protocolSnap.protocol, sampleCount: sampleN, toolLog: activities .filter(a => a.kind === 'tool' || a.kind === 'checkpoint') .map(a => `${a.kind === 'tool' ? a.toolStatus || 'tool' : 'checkpoint'} ${a.label || ''} ${String(a.detail || '').slice(0, 160)}`.trim()), replyPreview: String(reply).slice(0, 500), + replyTail: String(reply).trim().slice(-500), samplesDir, }; const outPath = flags.out ? path.resolve(String(flags.out)) : path.join(runHome, 'headless-result.json'); diff --git a/benchmarks/layout.mjs b/benchmarks/layout.mjs new file mode 100644 index 0000000..b426fb4 --- /dev/null +++ b/benchmarks/layout.mjs @@ -0,0 +1,53 @@ +/** + * Resolve bench paths for repo checkout vs extension-packaged calibrate suite. + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); + +/** + * @returns {{ + * packaged: boolean; + * here: string; + * extRoot: string; + * extOut: string; + * vscodeMock: string; + * resultsDir: string; + * repoRoot: string; + * }} + */ +export function resolveBenchLayout() { + const packagedOut = path.resolve(HERE, '..', '..', 'out'); + const packagedMock = path.join(HERE, 'support', 'vscodeFsMock.js'); + const packagedMarker = path.join(HERE, '.packaged'); + if ( + fs.existsSync(packagedMarker) || + (fs.existsSync(packagedOut) && fs.existsSync(packagedMock)) + ) { + const extRoot = path.resolve(HERE, '..', '..'); + return { + packaged: true, + here: HERE, + extRoot, + extOut: packagedOut, + vscodeMock: packagedMock, + resultsDir: path.join(os.homedir(), '.CodeForge', 'ai', 'calibrate-results'), + repoRoot: extRoot, + }; + } + const repo = path.resolve(HERE, '..'); + return { + packaged: false, + here: HERE, + extRoot: path.join(repo, 'extensions', 'codeforge'), + extOut: path.join(repo, 'extensions', 'codeforge', 'out'), + vscodeMock: path.join(repo, 'tests', 'support', 'vscodeFsMock.js'), + resultsDir: path.join(HERE, 'results'), + repoRoot: repo, + }; +} + +export { HERE }; diff --git a/benchmarks/tasks/T4-refactor-rename/task.json b/benchmarks/tasks/T4-refactor-rename/task.json index 003022c..fc44e9e 100644 --- a/benchmarks/tasks/T4-refactor-rename/task.json +++ b/benchmarks/tasks/T4-refactor-rename/task.json @@ -2,7 +2,7 @@ "id": "T4-refactor-rename", "category": "refactor", "title": "Rename calcTax to computeTax across src/", - "prompt": "Rename the function `calcTax` to `computeTax` throughout `src/` (definition, imports, usages and the export in `src/index.js`) without changing behavior. Confirm with `npm test`.", + "prompt": "Rename the function `calcTax` to `computeTax` throughout `src/` (definition, imports, usages and the export in `src/index.js`) without changing behavior. Do not leave `calcTax` as an alias or re-export under the old name — every reference must use `computeTax`. Confirm with `npm test`.", "protected": ["test/"], "checks": [ { "type": "absent", "dir": "src", "pattern": "calcTax", "why": "no leftover references" } diff --git a/docs/releases/v0.2.0.md b/docs/releases/v0.2.0.md new file mode 100644 index 0000000..fc814c5 --- /dev/null +++ b/docs/releases/v0.2.0.md @@ -0,0 +1,94 @@ +# CodeForge v0.2.0 — The contract around the model + +**Release date:** 2026-10-22 +**Download:** [Windows installer](https://github.com/Kortexio/CodeForge/releases/tag/v0.2.0) · [Repo](https://github.com/Kortexio/CodeForge) + +--- + +## The AHA moment + +Most “local AI IDEs” stall when the model is small: they over-read, under-edit, or declare victory without verifying. + +**CodeForge fixes the agent loop itself** — read → edit → test → finish — and adapts the harness to each model. + +On an **RTX 3060 12GB**, with **Qwen3.6-35B (IQ2_S)** via llama.cpp: + +| Task | Mean (3 reps) | Stuck | +|------|---------------:|------:| +| T1 — bugfix | **100** | 0 | +| T2 — feature (coupon) | **97** | 0 | +| T3 — multi-file | **100** | 0 | +| T4 — rename refactor | **100** | 0 | + +**Zero stuck runs. Zero bad tool-args.** A ~Q2 35B completing real agentic coding tasks end-to-end on a gaming GPU. + +**The model is not the product — the contract around the model is.** + +--- + +## What's new + +### Agentic loop that finishes +- Unlimited exploration within the turn budget (Cursor / Claude Code style) +- Continue when the step budget ends — no soft “explore cap” that refuses new reads +- Honest outcomes: stuck / truncated / oracle-red are **failed**, not “done” +- Spec self-check and oracle before finish with writes +- Diagnose-only finishes refused until the agent mutates or says `blocked:` + +### Adaptive harness +- Per-model profiles (`~/.codeforge/ai/model-profiles/`) +- Calibrate suite packaged in the extension (T2 / T4 / CS2 / CS3) +- Harness revise with paired bootstrap — keep only when the gain is real +- Optional `toolTransport` for XML-leaky locals +- Online hysteresis without quietly contaminating `harness.json` + +### You own the freedoms +- Sandbox level, Allow all, outside-workspace writes, MCP, Full agent freedom +- All user settings, revocable — the model cannot grant itself permissions +- API keys in SecretStorage (not plain settings JSON) +- DOMPurify + CSP nonce on chat / settings webviews + +### Measurable & robust +- Protocol telemetry: native vs content tool calls, bad args, length exits +- MCP / hooks / sessions hardened for daily use +- Windows process kill and shell prep fixes +- LinkedIn release announce workflow (optional) + +--- + +## Install + +1. Download `CodeForge-Setup-0.2.0-win32-x64.exe` +2. Run the installer (per-user under `%LOCALAPPDATA%\Programs\CodeForge`) +3. Open Settings → AI → point at your OpenAI-compatible endpoint (llama.cpp, Ollama, vLLM, cloud APIs…) +4. Pick a model → Agent mode → ship + +No Docker. No vendor lock-in. Your models. Your machine. + +--- + +## LinkedIn draft (ready to paste) + +The model is not the product — the contract around the model is. + +CodeForge v0.2.0 is out. + +What's new: +• Agentic loop that finishes: read → edit → test → done (no explore soft-caps) +• Adaptive harness per model — IQ2_S on a gaming GPU through frontier APIs +• Bench on RTX 3060 + Qwen3.6-35B: T1–T4 means 100 / 97 / 100 / 100, zero stuck +• You own permissions, sandbox, and secrets — the model cannot self-escalate + +Repo: https://github.com/Kortexio/CodeForge +Download: https://github.com/Kortexio/CodeForge/releases/download/v0.2.0/CodeForge-Setup-0.2.0-win32-x64.exe +Release notes: https://github.com/Kortexio/CodeForge/releases/tag/v0.2.0 + +#AI #OpenSource #LocalLLM #CodingAgent #DeveloperTools #llama.cpp + +--- + +## Not in this release + +- Intent crutches that rewrite user messages based on phrase matchers +- One global “weak mode” that punishes strong models +- Claiming 100% forever — residual failures stay visible as model limits diff --git a/extensions/codeforge/HEURISTICS.md b/extensions/codeforge/HEURISTICS.md index 9da1fa9..c6b29d1 100644 --- a/extensions/codeforge/HEURISTICS.md +++ b/extensions/codeforge/HEURISTICS.md @@ -60,11 +60,13 @@ Classes: | Step budget | keep | silent extension when progress changes | | Checkpoints every N steps / `progressReview.ts` / REPLAN | replace | done — orchestrator (`orchestrator.ts`): one item per card, ~25 steps, 2 attempts, then `blocked`; auto when task mentions cards and ≥2 exist (slash `/cards` forces); headless uses `runOrchestrated` | | Spec self-check "confirm each rule" — model confirms boundaries by reading and is wrong | replace | done — `specSelfCheckNote` requires executing the code at each boundary value (inclusive bounds, date-only = midnight, caps, rounding); one-off commands; no files created/edited under test directories (scratch goes to the OS temp dir); no scaffolded project when the code is compiled — the existing suite is the executable check | +| Done on build/test green without launching the app (opens but feature/runtime still broken) | replace | done — one-shot `shouldRequestRuntimeVerify` / `runtimeVerifyNote`: after source writes + green (or no oracle), require browser observe or non-oracle shell/dotnet launch before prose finish; `npm test` / `dotnet test` do not count | | Qwen XML parameter closed with `` instead of ``, next parameter leaks into the value | replace | done — `toolCallNormalizer` splits known-parameter markers (missing, echo and self-leak cases), refuses real conflicts with a fix hint; counted as `badArgs` with a raw sample | | `presence_penalty: 1.0` on tool rounds | remove | done — A/B 10/14 vs 13/14 with 0; tool calls must repeat tokens | | Unix/PowerShell habits on `cmd.exe` (`rm`, `/tmp`, heredoc, `Set-Content`) | replace | done — `shell` description maps them to `del`, `type`, `findstr`, `%TEMP%`, `write` | | Spec-check scratch files inside the workspace, and edits that change an existing test assertion | replace | done — `mutationGuard`: during the spec check, no test-dir changes, no new workspace files, no deletes; a pre-existing test file keeps every assertion line | | A background `dotnet test` kept the headless process alive after the agent had already stopped | replace | done — `bgShell` calls `child.unref()`, so the bench records the result instead of waiting out the timeout | +| Same weak-model compensations for every model | replace | done — `ModelProfile` per server/model (`~/.CodeForge/ai/model-profiles/…`); protocol telemetry rollup; `bench calibrate` (weak keeps defaults; strong drops write-cap / mass-rewrite / forced-weak-skill when `badArgs/native < 0.05`); online hysteresis at run start (write cap on at trunc≥5%, off only after 50-run clean window; tool temp −0.1 if leak≥8%, floor 0.3). Suite packaged at `resources/calibrate-bench/` (`sync:calibrate-bench`); Settings → **Benchmark** | | Rename finished on green tests with the old name still exported (`oldName: x.newName`) | replace | done — system contract: a rename/removal is done only when a search for the old name returns no hits, export keys included; `rename` tool is file-only | | Reply that only echoes the `/no_think` soft switch accepted as the final answer | replace | done — `splitThinkContent` drops lines that are only `/think` or `/no_think`, so the empty-turn retry applies | | Windows shell mangles inner `"` (`node -e "…"` exits 0 without running) | replace | done — `cmdExeSpawn`: outer quotes + `windowsVerbatimArguments` in bridge, sandbox runtime and `bgShell` | @@ -89,7 +91,9 @@ Classes: | Model retries `edit` whose `new_string` is already on disk (fake "missing brace") | replace | done — `applyEdit` returns `alreadyApplied` success so the 35B moves on | | Intent/continue enrich, thin-final/stuck soft nudges, write-count continue | remove | done — model works from system+tool contract; harness keeps protocol (length/JSON/budget) + tool gates only | | Stuck wrap used MAX_STEPS_PROMPT (wrong situation) / PT status labels | replace | done — `stuckWrapPrompt(cause)` vs `MAX_STEPS_PROMPT` vs `LENGTH_EXHAUSTED_PROMPT`; `loopExit` is `budget`/`stuck`/`length`; status/activity strings English | -| Soft cache / noop-step stuck heuristics | replace | done — `loopDetector.ts` (identical call+result for 3 steps, explore-only ≥20, UI unchanged ≥3) | +| Soft cache / noop-step stuck heuristics | replace | done — `loopDetector.ts` (identical call+result for 3 steps, UI unchanged ≥3); no explore-step soft/hard caps (Cursor/Claude Code style: unlimited reads within turn step budget) | +| Soft explore-cap refused new reads and stalled T2 | remove | done — removed `EXPLORE_SOFT_CAP` / `EXPLORE_HARD_CAP`; budget exhaustion shows Continue button (`continueTurn`) instead of refusing reads | +| Concrete tip after stuck still only re-reads | replace | done — stuck wrap says next turn must use edit/write/shell/browser; identical-repeat detector still stops loops | | Stale internal plan injection | replace | done — `internalPlans.ts` scoped to session/taskHash; archive on done/unrelated task | | Mechanical mid-compact digest | replace | done — structured LLM checkpoint (Objective / Work State / Next Move) with mechanical fallback | | Final round sends `tools: []` (breaks prefix cache) | replace | done — budget/stuck/length wraps use `postAgentRound` with tools + `tool_choice: none` | @@ -118,6 +122,9 @@ Classes: | `chunker.ts` / `workspaceIndex.ts` / `rerank.ts` / `indexWatcher.ts` | hybrid retrieve index | | `toolHistory.ts` | soft/mid compact + read ledger | | `exploreMode.ts` | optional read-only explore subagent | +| `modelProfile.ts` | per-model knobs (sampling, write cap, prompt extras, finish/orchestrator); disk under `~/.CodeForge/ai/model-profiles/` | +| `protocolTelemetry.ts` | protocol aggregator + `runs.jsonl` / `rollup.json` per model | +| `profileHysteresis.ts` | online flips at run start from rollup (write cap, tool temperature); quality knobs stay calibrate-only | ## Pending diff --git a/extensions/codeforge/package-lock.json b/extensions/codeforge/package-lock.json index 9832f3e..19b70ac 100644 --- a/extensions/codeforge/package-lock.json +++ b/extensions/codeforge/package-lock.json @@ -12,7 +12,8 @@ "highlight.js": "^11.12.0", "katex": "^0.19.0", "marked": "^18.0.14", - "mermaid": "^12.0.0" + "mermaid": "^12.0.0", + "minimatch": "^10.2.6" }, "devDependencies": { "@types/vscode": "^1.136.0", @@ -400,6 +401,27 @@ "d3-transition": "^3.0.1" } }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, "node_modules/chevrotain": { "version": "13.2.0", "resolved": "https://registry.npmjs.org/chevrotain/-/chevrotain-13.2.0.tgz", @@ -1137,6 +1159,21 @@ "node": ">= 20" } }, + "node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/package-manager-detector": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.8.0.tgz", diff --git a/extensions/codeforge/package.json b/extensions/codeforge/package.json index 55124f1..669c06f 100644 --- a/extensions/codeforge/package.json +++ b/extensions/codeforge/package.json @@ -2,7 +2,7 @@ "name": "codeforge", "displayName": "CodeForge AI", "description": "AI Agent sidebar and approval UI for CodeForge", - "version": "0.1.3", + "version": "0.2.0", "publisher": "kortexio", "engines": { "vscode": "^1.136.0" @@ -93,6 +93,16 @@ "title": "Settings (Servers & Models)", "category": "CodeForge AI" }, + { + "command": "codeforge.calibrateModelProfiles", + "title": "Benchmark & Calibrate Model Profiles", + "category": "CodeForge AI" + }, + { + "command": "codeforge.reviseHarness", + "title": "Revise Harness From Benchmark", + "category": "CodeForge AI" + }, { "command": "codeforge.resumeSession", "title": "Open Session", @@ -103,6 +113,11 @@ "title": "Delete Session", "category": "CodeForge AI" }, + { + "command": "codeforge.revokePermissions", + "title": "Revoke All Tool Permissions", + "category": "CodeForge AI" + }, { "command": "codeforge.clearSessions", "title": "Clear All Sessions", @@ -249,7 +264,7 @@ "on" ], "default": "off", - "markdownDescription": "Small-model harness: **on** = phase tool filtering + tighter guardrails; **off** = full tool surface (default). Legacy values: `force`→on, `auto`→off." + "markdownDescription": "Small-model harness: **on** = phase tool filtering + tighter guardrails; **off** = full tool surface (default). Legacy values: `force`→on, `auto`→off. Per-model knobs: [CodeForge Settings → Benchmark](command:codeforge.openSettings?%5B%22benchmark%22%5D)." }, "codeforge.ai.collapseToolCards": { "type": "boolean", @@ -269,6 +284,26 @@ "default": false, "description": "Automatically apply all file edits (write/rename/delete) without asking" }, + "codeforge.ai.allowAll": { + "type": "boolean", + "default": false, + "description": "Auto-approve every agent tool without prompts (global). Same as the chat Allow all chip." + }, + "codeforge.ai.confirmRunPython": { + "type": "boolean", + "default": false, + "description": "Show a confirmation dialog before running Python snippets from the chat Run button" + }, + "codeforge.ai.outsideWorkspaceWrites": { + "type": "string", + "enum": [ + "ask", + "allow", + "deny" + ], + "default": "ask", + "description": "Policy for write/edit/delete/rename/apply_patch targeting paths outside the workspace folder" + }, "codeforge.ai.previewEdits": { "type": "boolean", "default": false, @@ -284,12 +319,12 @@ "unrestricted" ], "default": "restricted", - "markdownDescription": "Default shell sandbox. **restricted** (default): scrub secret env vars and block dangerous patterns. **unrestricted**: full env + no blocked patterns (still runs in the workspace folder). Overridden by **Full agent freedom**." + "markdownDescription": "Shell sandbox for agent commands (the model cannot override this). **restricted** (default): scrub secret env vars, block dangerous patterns, refuse chaining in safe-local. **unrestricted**: full env + no blocked patterns (still runs in the workspace folder). Overridden by **Full agent freedom**." }, "codeforge.ai.fullAgentFreedom": { "type": "boolean", "default": false, - "markdownDescription": "Maximum autonomy: unrestricted shell sandbox, auto-approve file edits, and auto-approve shell for the session. Use only on trusted machines and workspaces." + "markdownDescription": "Maximum autonomy: unrestricted shell sandbox, **Allow all** (`codeforge.ai.allowAll`), auto-approve file edits, and session shell auto-approve. Use only on trusted machines and workspaces." }, "codeforge.ai.traceLevel": { "type": "string", @@ -542,7 +577,9 @@ "@highlightjs/cdn-assets": "^11.12.0", "highlight.js": "^11.12.0", "katex": "^0.19.0", + "dompurify": "^3.4.12", "marked": "^18.0.14", - "mermaid": "^12.0.0" + "mermaid": "^12.0.0", + "minimatch": "^10.2.6" } } diff --git a/extensions/codeforge/resources/calibrate-bench/.packaged b/extensions/codeforge/resources/calibrate-bench/.packaged new file mode 100644 index 0000000..76ef117 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/.packaged @@ -0,0 +1,2 @@ +synced 2026-10-22T09:38:13.629Z +source=C:/Users/vitor/source/repos/Kortexio/CodeForge/benchmarks diff --git a/extensions/codeforge/resources/calibrate-bench/README.md b/extensions/codeforge/resources/calibrate-bench/README.md new file mode 100644 index 0000000..b9da5b5 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/README.md @@ -0,0 +1,4 @@ +# Packaged calibrate suite + +Generated by `node scripts/sync-calibrate-bench.mjs` (also via `build:extensions`). +Do not edit by hand — change files under `/benchmarks` and re-sync. diff --git a/extensions/codeforge/resources/calibrate-bench/bench.mjs b/extensions/codeforge/resources/calibrate-bench/bench.mjs new file mode 100644 index 0000000..16c254e --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/bench.mjs @@ -0,0 +1,1441 @@ +#!/usr/bin/env node +/** + * CodeForge harness benchmark — semi-automatic (the agent runs inside the IDE). + * + * node benchmarks/bench.mjs list + * node benchmarks/bench.mjs setup CS1 --label bonsai [--open] + * node benchmarks/bench.mjs check latest + * node benchmarks/bench.mjs report + * node benchmarks/bench.mjs selftest + * node benchmarks/bench.mjs run T1 --label baseline-qwen35b [--server id] [--model id] [--weak on|off] + * + * Runs live in %USERPROFILE%/codeforge-bench (override with CODEFORGE_BENCH_DIR). + * Headless `run` uses the active server/model from ~/.CodeForge/ai/settings.json. + */ + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn, spawnSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { resolveBenchLayout } from './layout.mjs'; + +const LAYOUT = resolveBenchLayout(); +const HERE = LAYOUT.here; +const FIXTURES = { + shop: path.join(HERE, 'fixtures', 'shop'), + billing: path.join(HERE, 'fixtures', 'billing'), + scaffold: path.join(HERE, 'fixtures', 'scaffold'), + 'billing-bugs': path.join(HERE, 'fixtures', 'billing-bugs'), + 'router-cards': path.join(HERE, 'fixtures', 'router-cards'), +}; +const TASKS_DIR = path.join(HERE, 'tasks'); +const RESULTS_DIR = + process.env.CODEFORGE_BENCH_RESULTS?.trim() || LAYOUT.resultsDir; +let RUNS_ROOT = process.env.CODEFORGE_BENCH_DIR || path.join(os.homedir(), 'codeforge-bench'); +const CF_HOME = path.join(os.homedir(), '.CodeForge'); +const HIDDEN_DIR = '.bench-hidden'; +const JUNK = /(^|\/)(_tmp|tmp_|_vip_|_check_)[^/]*$|\.py$/i; +const COPY_SKIP = new Set(['bin', 'obj', 'node_modules', '.git', 'TestResults']); + +// --------------------------------------------------------------------------- + +function parseArgs(argv) { + const pos = []; + const flags = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a.startsWith('--')) { + const [k, v] = a.slice(2).split('='); + if (v !== undefined) flags[k] = v; + else if (argv[i + 1] && !argv[i + 1].startsWith('--')) flags[k] = argv[++i]; + else flags[k] = true; + } else { + pos.push(a); + } + } + return { pos, flags }; +} + +function loadTasks() { + return fs + .readdirSync(TASKS_DIR, { withFileTypes: true }) + .filter(d => d.isDirectory()) + .map(d => ({ + ...JSON.parse(fs.readFileSync(path.join(TASKS_DIR, d.name, 'task.json'), 'utf8')), + dir: path.join(TASKS_DIR, d.name), + })) + .sort((a, b) => a.id.localeCompare(b.id)); +} + +function findTask(key) { + const k = String(key || '').toLowerCase(); + const t = loadTasks().find( + t => t.id.toLowerCase() === k || t.id.toLowerCase().startsWith(`${k}-`) || t.id.toLowerCase().startsWith(k) + ); + if (!t) die(`Unknown task "${key}". Use: list`); + return t; +} + +function die(msg) { + console.error(msg); + process.exit(1); +} + +function git(dir, args) { + return spawnSync('git', ['-C', dir, ...args], { encoding: 'utf8' }); +} + +function normPath(p) { + return path.resolve(String(p || '')).replace(/\\/g, '/').replace(/\/+$/, '').toLowerCase(); +} + +function readJson(file, fallback) { + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch { + return fallback; + } +} + +function walk(dir, out = []) { + if (!fs.existsSync(dir)) return out; + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + if (COPY_SKIP.has(e.name)) continue; + const p = path.join(dir, e.name); + if (e.isDirectory()) walk(p, out); + else out.push(p); + } + return out; +} + +function copyFixture(src, dest) { + fs.mkdirSync(dest, { recursive: true }); + for (const e of fs.readdirSync(src, { withFileTypes: true })) { + if (COPY_SKIP.has(e.name)) continue; + const from = path.join(src, e.name); + const to = path.join(dest, e.name); + if (e.isDirectory()) copyFixture(from, to); + else { + fs.mkdirSync(path.dirname(to), { recursive: true }); + fs.copyFileSync(from, to); + } + } +} + +function taskRuntime(task) { + return task.runtime || (task.fixture === 'billing' || task.fixture === 'scaffold' ? 'dotnet' : 'node'); +} + +function taskFixture(task) { + const name = task.fixture || 'shop'; + const dir = FIXTURES[name]; + if (!dir || !fs.existsSync(dir)) die(`Unknown fixture "${name}" for ${task.id}`); + return dir; +} + +// --------------------------------------------------------------------------- +// setup + +function cmdSetup(taskKey, flags) { + const task = findTask(taskKey); + const label = String(flags.label || 'model').replace(/[^\w.-]+/g, '_'); + const stamp = new Date().toISOString().replace(/[-:]/g, '').replace('T', '-').slice(0, 15); + const name = `${task.id.split('-')[0]}-${label}-${stamp}`; + const dir = path.join(RUNS_ROOT, name); + + fs.mkdirSync(RUNS_ROOT, { recursive: true }); + copyFixture(taskFixture(task), dir); + const overlay = path.join(task.dir, 'overlay'); + if (fs.existsSync(overlay)) copyFixture(overlay, dir); + + git(dir, ['init', '-q']); + git(dir, ['add', '-A']); + const commit = git(dir, [ + '-c', + 'user.email=bench@codeforge.local', + '-c', + 'user.name=bench', + 'commit', + '-qm', + 'baseline', + ]); + if (commit.status !== 0) die(`git commit failed: ${commit.stderr}`); + + const meta = { + name, + task: task.id, + label, + dir, + runtime: taskRuntime(task), + createdAt: new Date().toISOString(), + }; + fs.writeFileSync(path.join(RUNS_ROOT, `${name}.meta.json`), JSON.stringify(meta, null, 2)); + fs.writeFileSync(path.join(RUNS_ROOT, `${name}.prompt.txt`), task.prompt, 'utf8'); + if (flags.quiet) return meta; + + console.log(`Run: ${name}`); + console.log(`Folder: ${dir}`); + console.log(`Task: ${task.title}`); + console.log(`Runtime:${taskRuntime(task)}`); + console.log(''); + console.log('1. Open the folder in CodeForge, start a NEW chat and pick the model.'); + console.log('2. Paste this prompt:'); + console.log(''); + console.log(` ${task.prompt}`); + console.log(''); + console.log(`3. When the agent stops: node benchmarks/bench.mjs check ${name}`); + + if (flags.open) openInCodeForge(dir); + return meta; +} + +function openInCodeForge(dir) { + const candidates = [ + process.env.CODEFORGE_EXE, + path.join(process.env.LOCALAPPDATA || '', 'Programs', 'CodeForge', 'CodeForge.exe'), + path.join(HERE, '..', 'VSCode-win32-x64', 'CodeForge.exe'), + ].filter(Boolean); + const exe = candidates.find(p => fs.existsSync(p)); + if (!exe) { + console.log('(CodeForge.exe not found — open the folder manually or set CODEFORGE_EXE)'); + return; + } + spawn(exe, ['--new-window', dir], { detached: true, stdio: 'ignore' }).unref(); + console.log(`Opened in ${exe}`); +} + +// --------------------------------------------------------------------------- +// check + +function resolveRun(key) { + const metas = fs.existsSync(RUNS_ROOT) + ? fs + .readdirSync(RUNS_ROOT) + .filter(f => f.endsWith('.meta.json')) + .map(f => readJson(path.join(RUNS_ROOT, f))) + .filter(Boolean) + : []; + if (!metas.length) die(`No runs in ${RUNS_ROOT}. Use: setup`); + if (!key || key === 'latest') { + return metas.sort((a, b) => b.createdAt.localeCompare(a.createdAt))[0]; + } + const k = normPath(key); + const m = metas.find(m => m.name === key || normPath(m.dir) === k); + if (!m) die(`Run "${key}" not found in ${RUNS_ROOT}`); + return m; +} + +function runNodeTests(dir, files) { + if (!files.length) return { pass: 0, fail: 0, output: '(no test files)', skipped: true }; + const r = spawnSync(process.execPath, ['--test', '--test-reporter=tap', ...files], { + cwd: dir, + encoding: 'utf8', + timeout: 120_000, + }); + const out = `${r.stdout || ''}${r.stderr || ''}`; + const num = re => Number((re.exec(out) || [])[1] || 0); + const pass = num(/^# pass (\d+)/m); + let fail = num(/^# fail (\d+)/m); + if (r.error || (r.status !== 0 && fail === 0)) fail = Math.max(1, fail); + return { pass, fail, output: out }; +} + +function runDotnetTests(dir, { filter } = {}) { + const args = ['test', '-v', 'q', '--nologo']; + if (filter) args.push('--filter', filter); + const r = spawnSync('dotnet', args, { + cwd: dir, + encoding: 'utf8', + timeout: 300_000, + shell: false, + }); + const out = `${r.stdout || ''}${r.stderr || ''}`; + const passed = Number((/Passed:\s+(\d+)/.exec(out) || [])[1] || 0); + const failed = Number((/Failed:\s+(\d+)/.exec(out) || [])[1] || 0); + const total = Number((/Total:\s+(\d+)/.exec(out) || [])[1] || passed + failed); + let pass = passed; + let fail = failed; + if (r.error) { + fail = Math.max(1, fail); + return { pass, fail, output: String(r.error), total }; + } + if (r.status !== 0 && fail === 0 && pass === 0) { + // no test host / build failed + fail = 1; + } + return { pass, fail, output: out, total }; +} + +function failedTestNames(output, runtime) { + if (runtime === 'dotnet') { + return [...output.matchAll(/^\s*Error Message:\s*$/gm)].length + ? [...output.matchAll(/Failed \S+ \[([^\]]+)\]/g)].map(m => m[1]).slice(0, 12) + : [...output.matchAll(/^\s+Failed (\S+)/gm)].map(m => m[1]).slice(0, 12); + } + return [...output.matchAll(/^\s*not ok \d+ - (.+)$/gm)].map(m => m[1].trim()).slice(0, 12); +} + +function changedFiles(dir) { + const r = git(dir, ['status', '--porcelain', '-uall']); + return r.stdout + .split('\n') + .filter(Boolean) + .map(l => ({ + code: l.slice(0, 2).trim(), + file: l.slice(3).trim().replace(/^"|"$/g, '').replace(/\\/g, '/'), + })) + .filter( + c => + !c.file.startsWith(`${HIDDEN_DIR}/`) && + !c.file.startsWith('.CodeForge/') && + !c.file.includes('/bin/') && + !c.file.includes('/obj/') && + !c.file.startsWith('bin/') && + !c.file.startsWith('obj/') + ); +} + +function runChecks(dir, list, changed) { + return (list || []).map(c => { + if (c.type === 'absent') { + const re = new RegExp(c.pattern); + const hits = walk(path.join(dir, c.dir)) + .filter(f => re.test(fs.readFileSync(f, 'utf8'))) + .map(f => path.relative(dir, f).replace(/\\/g, '/')); + return { ...c, ok: hits.length === 0, detail: hits.join(', ') }; + } + if (c.type === 'unchanged') { + const hit = changed.some(x => x.file === c.file); + return { ...c, ok: !hit, detail: hit ? 'modified' : '' }; + } + if (c.type === 'contains') { + const file = path.join(dir, c.file); + const ok = fs.existsSync(file) && new RegExp(c.pattern, 'im').test(fs.readFileSync(file, 'utf8')); + return { ...c, ok, detail: ok ? '' : 'pattern not found' }; + } + if (c.type === 'exists') { + const ok = fs.existsSync(path.join(dir, c.file)); + return { ...c, ok, detail: ok ? '' : 'missing' }; + } + return { ...c, ok: false, detail: `unknown check type ${c.type}` }; + }); +} + +/** Mirror of claimsBuildOrTestGreen in extensions/codeforge/src/agent/nudges.ts. */ +function claimsGreen(text) { + const t = String(text || ''); + const negated = + /\bn[ãa]o\s+(compila|passa|est[áa]\s+verde)|\b(tests?|build)\s+(failed|failing)\b|\b[1-9]\d*\s+(errors?|erros?|failures?|falhas?|failed)\b/i.test( + t + ); + if (negated) return false; + return ( + /\b(all\s+)?(passing|passed|green|verde|0\s+errors?|0\s+erros?|0\s+failures?|0\s+falhas?|build\s+succeeded|build\s+ok|tests?\s+pass)/i.test(t) || + /\ball\s+green\b|\btudo\s+verde\b/i.test(t) || + /\b(compila|compilou|compilando)\b/i.test(t) || + /\bsem\s+(erros?|falhas?)\b/i.test(t) || + /\btestes?\s+(passa|passam|passaram|ok|verdes?)\b/i.test(t) || + /\b(build|compila[çc][ãa]o)\s+(com\s+sucesso|bem[-\s]sucedid[ao]|ok)\b/i.test(t) + ); +} + +function sessionMetrics(runDir) { + const sessionsRoot = path.join(CF_HOME, 'ai', 'sessions'); + const tracesRoot = path.join(CF_HOME, 'ai', 'traces'); + const target = normPath(runDir); + const sessions = (fs.existsSync(sessionsRoot) ? fs.readdirSync(sessionsRoot) : []) + .map(id => readJson(path.join(sessionsRoot, id, 'session.json'))) + .filter(s => s && s.workspaceFolder && normPath(s.workspaceFolder) === target); + + const m = { + sessions: sessions.length, + models: [...new Set(sessions.map(s => s.model).filter(Boolean))], + states: sessions.map(s => s.state), + userMessages: 0, + tools: 0, + toolFails: 0, + byTool: {}, + reads: 0, + readUniquePaths: 0, + readRepeats: 0, + writes: 0, + shells: 0, + shellFails: 0, + llmCalls: 0, + tokensIn: 0, + maxPromptTokens: 0, + tokensOut: 0, + compacts: 0, + llmErrors: 0, + overflowErrors: 0, + wallSeconds: 0, + claimedGreen: false, + harnessChars: 0, + }; + const lastSession = [...sessions].sort((a, b) => Date.parse(b.updatedAt) - Date.parse(a.updatedAt))[0]; + const lastReply = [...(lastSession?.messages || [])].reverse().find(x => x.role === 'assistant'); + m.claimedGreen = claimsGreen(String(lastReply?.content || '')); + const readKeys = new Set(); + const readPaths = new Set(); + let first = Infinity; + let last = 0; + for (const s of sessions) { + m.userMessages += (s.messages || []).filter(x => x.role === 'user').length; + for (const t of s.toolTraces || []) { + m.tools += 1; + m.byTool[t.name] = (m.byTool[t.name] || 0) + 1; + if (!t.success) m.toolFails += 1; + const args = t.arguments || {}; + if (t.name === 'read') { + m.reads += 1; + const p = String(args.path || '') + .replace(/\\/g, '/') + .toLowerCase(); + const key = `${p}@${args.startLine ?? args.offset ?? 1}`; + if (readKeys.has(key)) m.readRepeats += 1; + readKeys.add(key); + readPaths.add(p); + } + if ((t.name === 'write' || t.name === 'edit') && t.success) m.writes += 1; + if (t.name === 'shell' || t.name === 'dotnet') { + m.shells += 1; + if (!t.success) m.shellFails += 1; + } + } + first = Math.min(first, Date.parse(s.createdAt)); + last = Math.max(last, Date.parse(s.updatedAt)); + + const traceFiles = fs.existsSync(tracesRoot) + ? fs.readdirSync(tracesRoot).filter(f => f.startsWith(`${s.id}-`) && f.endsWith('.jsonl')) + : []; + for (const f of traceFiles) { + for (const line of fs.readFileSync(path.join(tracesRoot, f), 'utf8').split('\n')) { + if (!line.trim()) continue; + let e; + try { + e = JSON.parse(line); + } catch { + continue; + } + if (e.type === 'llm') { + m.llmCalls += 1; + m.tokensIn += e.tokensIn || 0; + m.tokensOut += e.tokensOut || 0; + m.maxPromptTokens = Math.max(m.maxPromptTokens, e.tokensIn || 0); + } else if (e.type === 'compact') { + m.compacts += 1; + } else if (e.type === 'info' && e.label === 'harness') { + m.harnessChars += Number(e.detail) || 0; + } else if (e.type === 'error' && /LLM error/i.test(e.label)) { + m.llmErrors += 1; + if (/exceed|context (size|length)|n_ctx/i.test(e.detail || '')) m.overflowErrors += 1; + } + } + } + } + m.readUniquePaths = readPaths.size; + if (Number.isFinite(first) && last > first) m.wallSeconds = Math.round((last - first) / 1000); + return m; +} + +/** + * Review scoring: `path:line` references in the report vs the planted-bug manifest. + * A reference hits a bug when the file names match and the line is within the tolerance. + */ +function scoreReview(dir, task) { + const manifest = readJson(path.join(task.dir, 'bugs.manifest.json'), []); + const reportFile = path.join(dir, task.report || 'BUGS.md'); + const text = fs.existsSync(reportFile) ? fs.readFileSync(reportFile, 'utf8') : ''; + const tol = task.lineTolerance ?? 3; + const refs = [ + ...new Set( + [...text.matchAll(/([\w./\\-]+\.(?:cs|js|ts|json|md)):(\d+)/g)].map( + m => `${m[1].replace(/\\/g, '/').toLowerCase()}:${m[2]}` + ) + ), + ].map(k => { + const i = k.lastIndexOf(':'); + return { file: k.slice(0, i), line: Number(k.slice(i + 1)) }; + }); + const sameFile = (a, b) => a === b || a.endsWith(`/${b}`) || b.endsWith(`/${a}`) || a.split('/').pop() === b.split('/').pop(); + const hits = r => manifest.filter(b => sameFile(r.file, b.file.toLowerCase()) && Math.abs(r.line - b.line) <= tol); + const found = manifest.filter(b => refs.some(r => hits(r).includes(b))); + const truePositives = refs.filter(r => hits(r).length > 0).length; + const precision = refs.length ? truePositives / refs.length : 0; + const recall = manifest.length ? found.length / manifest.length : 0; + const f1 = precision + recall ? (2 * precision * recall) / (precision + recall) : 0; + return { + reported: refs.length, + found: found.map(b => b.id), + missed: manifest.filter(b => !found.includes(b)).map(b => b.id), + falsePositives: refs.length - truePositives, + precision: Math.round(precision * 100) / 100, + recall: Math.round(recall * 100) / 100, + f1: Math.round(f1 * 100) / 100, + }; +} + +function runVisibleAndHidden(dir, task) { + const runtime = taskRuntime(task); + if (task.category === 'review') { + const none = { pass: 0, fail: 0, output: '(review task — no tests)', skipped: true }; + return { visible: none, hidden: { ...none }, runtime }; + } + const hiddenSrc = path.join(task.dir, 'hidden'); + const hasHidden = + fs.existsSync(hiddenSrc) && walk(hiddenSrc).some(f => /\.(cs|js)$/i.test(f)); + + if (runtime === 'dotnet') { + const hiddenTarget = path.join(dir, task.hiddenTarget || 'tests/Acme.Billing.Tests/BenchHidden'); + if (hasHidden && task.cards?.length) { + // Per card: only that card's hidden tests are compiled in, so a missing later card + // does not zero the earlier ones. + fs.rmSync(hiddenTarget, { recursive: true, force: true }); + const visible = runDotnetTests(dir, { filter: 'FullyQualifiedName!~BenchHidden' }); + const cards = []; + let output = ''; + for (const card of task.cards) { + fs.rmSync(hiddenTarget, { recursive: true, force: true }); + copyFixture(path.join(hiddenSrc, card), hiddenTarget); + const r = runDotnetTests(dir, { filter: 'FullyQualifiedName~BenchHidden' }); + const total = r.pass + r.fail; + cards.push({ card, pass: r.pass, fail: r.fail, ratio: total ? r.pass / total : 0 }); + output += `\n${r.output}`; + } + fs.rmSync(hiddenTarget, { recursive: true, force: true }); + const hidden = { + pass: cards.reduce((s, c) => s + c.pass, 0), + fail: cards.reduce((s, c) => s + c.fail, 0), + output, + skipped: false, + cards, + ratio: cards.reduce((s, c) => s + c.ratio, 0) / cards.length, + }; + return { visible, hidden, runtime }; + } + const visible = runDotnetTests(dir, { + filter: hasHidden ? 'FullyQualifiedName!~BenchHidden' : undefined, + }); + let hidden = { pass: 0, fail: 0, output: '(no hidden)', skipped: !hasHidden }; + if (hasHidden) { + fs.rmSync(hiddenTarget, { recursive: true, force: true }); + copyFixture(hiddenSrc, hiddenTarget); + hidden = runDotnetTests(dir, { filter: 'FullyQualifiedName~BenchHidden' }); + fs.rmSync(hiddenTarget, { recursive: true, force: true }); + } else if (task.category === 'scaffold') { + // scaffold: whole suite is the acceptance bar + hidden = { ...visible, skipped: false }; + } + return { visible, hidden, runtime }; + } + + const visibleFiles = walk(path.join(dir, 'test')) + .filter(f => f.endsWith('.test.js')) + .map(f => path.relative(dir, f)); + const visible = runNodeTests(dir, visibleFiles); + const hiddenTarget = path.join(dir, HIDDEN_DIR); + fs.rmSync(hiddenTarget, { recursive: true, force: true }); + let hidden = { pass: 0, fail: 0, output: '(no hidden)', skipped: true }; + if (hasHidden) { + copyFixture(hiddenSrc, hiddenTarget); + const hiddenFiles = walk(hiddenTarget).map(f => path.relative(dir, f)); + hidden = runNodeTests(dir, hiddenFiles); + fs.rmSync(hiddenTarget, { recursive: true, force: true }); + } + return { visible, hidden, runtime }; +} + +function cmdCheck(key, { save = true, print = true } = {}) { + const run = resolveRun(key); + const task = findTask(run.task); + const dir = run.dir; + if (!fs.existsSync(dir)) die(`Run folder missing: ${dir}`); + + const changed = changedFiles(dir); + const protectedHits = changed.filter(c => (task.protected || []).some(p => c.file.startsWith(p))); + const junk = changed.filter(c => JUNK.test(c.file)); + + const { visible, hidden, runtime } = runVisibleAndHidden(dir, task); + const checks = runChecks(dir, task.checks, changed); + const bonus = runChecks(dir, task.bonus, changed); + const session = sessionMetrics(dir); + + const hiddenTotal = hidden.pass + hidden.fail; + const hiddenRatio = + hidden.ratio ?? (hidden.skipped ? 1 : hiddenTotal ? hidden.pass / hiddenTotal : 0); + const visibleOk = + visible.fail === 0 && (visible.pass > 0 || (task.category === 'scaffold' && checks.every(c => c.ok))); + const checksOk = checks.every(c => c.ok); + const review = task.category === 'review' ? scoreReview(dir, task) : null; + let score; + let pass; + if (review) { + score = Math.round(80 * review.f1 + 10 * (protectedHits.length ? 0 : 1) + 10 * (checksOk ? 1 : 0)); + pass = review.f1 >= 0.8 && !protectedHits.length && checksOk; + } else { + score = Math.round( + 60 * hiddenRatio + + 15 * (visibleOk ? 1 : 0) + + 10 * (protectedHits.length ? 0 : 1) + + 10 * (checksOk ? 1 : 0) + + 5 * (junk.length ? 0 : 1) + ); + pass = hiddenRatio === 1 && visibleOk && !protectedHits.length && checksOk; + } + const falseDone = review ? false : session.claimedGreen && !(visible.fail === 0 && hidden.fail === 0); + + const result = { + run: run.name, + task: task.id, + label: run.label, + runtime, + model: session.models.join(', ') || null, + checkedAt: new Date().toISOString(), + pass, + score, + falseDone, + hidden: { + pass: hidden.pass, + fail: hidden.fail, + failed: failedTestNames(hidden.output, runtime), + ...(hidden.cards ? { cards: hidden.cards } : {}), + }, + ...(review ? { review } : {}), + visible: { + pass: visible.pass, + fail: visible.fail, + failed: failedTestNames(visible.output, runtime), + }, + changedFiles: changed.map(c => `${c.code} ${c.file}`), + protectedTouched: protectedHits.map(c => c.file), + junkFiles: junk.map(c => c.file), + checks: checks.map(({ type, ok, why, detail }) => ({ type, ok, why, detail })), + bonus: bonus.map(({ type, ok, why, detail }) => ({ type, ok, why, detail })), + session, + }; + if (save) { + fs.mkdirSync(RESULTS_DIR, { recursive: true }); + fs.writeFileSync(path.join(RESULTS_DIR, `${run.name}.json`), JSON.stringify(result, null, 2)); + } + if (print) printResult(result); + return result; +} + +// --------------------------------------------------------------------------- +// selftest + +function applySolution(dir, taskDir) { + for (const edit of readJson(path.join(taskDir, 'solution.json'), [])) { + if (edit.shell) { + const r = spawnSync(edit.shell[0], edit.shell.slice(1), { + cwd: dir, + encoding: 'utf8', + timeout: 180_000, + }); + if (r.status !== 0) { + throw new Error(`solution shell failed: ${edit.shell.join(' ')}\n${r.stdout}\n${r.stderr}`); + } + continue; + } + if (edit.delete) { + fs.rmSync(path.join(dir, edit.delete), { force: true }); + continue; + } + if (edit.write !== undefined) { + const file = path.join(dir, edit.file); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, edit.write); + continue; + } + const file = path.join(dir, edit.file); + const text = fs.readFileSync(file, 'utf8'); + if (!text.includes(edit.replace)) throw new Error(`${edit.file}: "${edit.replace}" not found`); + fs.writeFileSync( + file, + edit.all ? text.split(edit.replace).join(edit.with) : text.replace(edit.replace, edit.with) + ); + } +} + +function cmdRun(taskKey, flags) { + const label = String(flags.label || 'headless').replace(/[^\w.-]+/g, '_'); + const meta = cmdSetup(taskKey, { ...flags, label, quiet: true }); + const prompt = fs.readFileSync(path.join(RUNS_ROOT, `${meta.name}.prompt.txt`), 'utf8'); + const samplesDir = path.join(RESULTS_DIR, meta.name, 'protocol-samples'); + fs.mkdirSync(samplesDir, { recursive: true }); + const outJson = path.join(RESULTS_DIR, `${meta.name}.headless.json`); + fs.mkdirSync(RESULTS_DIR, { recursive: true }); + + if (!flags.skipBuild && !LAYOUT.packaged) { + const compile = spawnSync('npm', ['run', 'build:extensions'], { + cwd: LAYOUT.repoRoot, + encoding: 'utf8', + shell: true, + timeout: 300_000, + }); + if (compile.status !== 0) { + die(`build:extensions failed:\n${compile.stdout}\n${compile.stderr}`); + } + } + + const args = [ + path.join(HERE, 'headlessAgent.mjs'), + '--workspace', + meta.dir, + '--task', + prompt, + '--out', + outJson, + '--samplesDir', + samplesDir, + '--weak', + String(flags.weak ?? 'on'), + ]; + if (flags.server) args.push('--server', String(flags.server)); + if (flags.model) args.push('--model', String(flags.model)); + if (flags.maxSteps) args.push('--maxSteps', String(flags.maxSteps)); + if (flags.hardCap) args.push('--hardCap', String(flags.hardCap)); + if (flags.sampling) args.push('--sampling', String(flags.sampling)); + args.push('--profileHome', path.join(os.homedir(), '.CodeForge')); + + console.log(`Headless run ${meta.name}…`); + const agentEnv = { + ...process.env, + CODEFORGE_EXT_OUT: LAYOUT.extOut, + NODE_PATH: [ + path.join(LAYOUT.extRoot, 'node_modules'), + path.join(LAYOUT.repoRoot, 'node_modules'), + process.env.NODE_PATH || '', + ] + .filter(Boolean) + .join(path.delimiter), + }; + if (flags.disableHysteresis || flags.calibrate) { + agentEnv.CODEFORGE_DISABLE_HYSTERESIS = '1'; + } + if (flags.harnessFile) { + agentEnv.CODEFORGE_HARNESS_FILE = path.resolve(String(flags.harnessFile)); + } + const agent = spawnSync(process.execPath, args, { + cwd: LAYOUT.repoRoot, + encoding: 'utf8', + timeout: 3_600_000, + env: agentEnv, + }); + if (agent.status === 2) { + process.stdout.write(agent.stdout || ''); + process.stderr.write(agent.stderr || ''); + if (flags.softFail) { + return { + pass: false, + score: 0, + infrastructureError: true, + notRun: true, + loopExit: 'error', + task: taskKey, + }; + } + process.exit(2); + } + if (agent.status !== 0) { + if (flags.softFail) { + return { + pass: false, + score: 0, + infrastructureError: true, + notRun: true, + loopExit: 'error', + task: taskKey, + }; + } + console.error(agent.stdout); + console.error(agent.stderr); + throw new Error(`headless agent exited ${agent.status}`); + } + process.stdout.write(agent.stdout || ''); + + const headless = readJson(outJson) || {}; + const scored = cmdCheck(meta.name, { save: true, print: true }); + const merged = { + ...scored, + runtime: 'headless', + model: headless.model, + server: headless.server, + loopExit: headless.loopExit, + steps: headless.steps, + refusals: headless.refusals, + protocol: headless.protocol, + toolCallMode: headless.toolCallMode, + elapsedMs: headless.elapsedMs, + headlessReply: headless.replyPreview, + replyTail: headless.replyTail, + }; + const resultPath = path.join(RESULTS_DIR, `${meta.name}.json`); + fs.writeFileSync(resultPath, JSON.stringify(merged, null, 2)); + console.log(`Merged headless metrics → results/${meta.name}.json`); + return merged; +} + +function cmdSelftest(filter) { + RUNS_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'cf-bench-selftest-')); + let ok = true; + try { + const tasks = loadTasks().filter(t => !filter || t.id.toLowerCase().startsWith(String(filter).toLowerCase())); + for (const task of tasks) { + if (!fs.existsSync(path.join(task.dir, 'solution.json'))) { + console.log(`skip ${task.id} (no solution.json)`); + continue; + } + const meta = cmdSetup(task.id, { label: 'selftest', quiet: true }); + const before = cmdCheck(meta.name, { save: false, print: false }); + applySolution(meta.dir, task.dir); + const after = cmdCheck(meta.name, { save: false, print: false }); + const expectBaselineFail = task.category !== 'scaffold'; + const good = + (expectBaselineFail ? !before.pass : true) && after.pass && after.score === 100; + ok &&= good; + console.log( + `${good ? 'ok ' : 'BAD '} ${task.id.padEnd(24)} baseline ${before.pass ? 'PASS' : 'fail'} ${before.hidden.pass}/${before.hidden.pass + before.hidden.fail} · solution ${after.pass ? 'pass' : 'FAIL'} score ${after.score}` + ); + if (!good) { + console.log(` hidden failed: ${after.hidden.failed.join(' | ') || '-'}`); + console.log(` visible failed: ${after.visible.failed.join(' | ') || '-'}`); + console.log(` checks: ${JSON.stringify(after.checks)} bonus: ${JSON.stringify(after.bonus)}`); + console.log(` visible out tail:\n${(after.visible?.failed, '')}`); + } + } + } finally { + fs.rmSync(RUNS_ROOT, { recursive: true, force: true }); + } + if (!ok) process.exit(1); +} + +/** + * Classify a failed (or suspicious) run from structured fields — not reply text. + * A = harness (no writes / stuck / refusals), B = protocol (badArgs/normalize/truncated), + * C = semantic (wrote + oracle green but hidden red). + */ +function classifyFailure(r) { + if (r.pass && !r.falseDone) return null; + const writes = Number(r.session?.writes ?? r.changedFiles?.length ?? 0); + const exit = String(r.loopExit || ''); + const refusals = r.refusals || {}; + const refusalN = Object.values(refusals).reduce((a, b) => a + Number(b || 0), 0); + const proto = r.protocol || {}; + const badArgs = Number(proto.badArgs || 0); + const normalizeErr = Number(proto.normalize_error || proto.normalizeError || 0); + const truncated = Number(proto.lengthExits || proto.truncated_tool_json || 0); + const classes = []; + if (writes === 0 || exit === 'stuck' || refusalN > 0) classes.push('A-harness'); + if (badArgs > 0 || normalizeErr > 0 || truncated > 0 || exit === 'length') classes.push('B-protocol'); + if ( + writes > 0 && + (r.falseDone || (Number(r.hidden?.fail || 0) > 0 && !r.falseDone)) + ) { + classes.push('C-semantic'); + } + if (!classes.length) classes.push(r.pass ? 'ok' : 'unknown'); + return classes.join('+'); +} + +function printResult(r) { + const s = r.session; + const failClass = classifyFailure(r); + console.log( + `${r.pass ? 'PASS' : 'FAIL'} ${r.run} score ${r.score}/100 (${r.model || 'no session found'}${r.runtime ? ` · ${r.runtime}` : ''}${failClass ? ` · ${failClass}` : ''})` + ); + if (r.review) { + const v = r.review; + console.log( + ` review F1 ${v.f1} (precision ${v.precision}, recall ${v.recall}) · found ${v.found.join(', ') || '-'} · missed ${v.missed.join(', ') || '-'} · ${v.falsePositives} false positives` + ); + } + if (r.hidden.cards) { + console.log( + ` cards ${r.hidden.cards.map(c => `${c.card} ${c.pass}/${c.pass + c.fail}`).join(' · ')}` + ); + } + console.log( + ` hidden tests ${r.hidden.pass}/${r.hidden.pass + r.hidden.fail}${r.hidden.failed.length ? ` failed: ${r.hidden.failed.join(' | ')}` : ''}` + ); + console.log( + ` visible tests ${r.visible.pass}/${r.visible.pass + r.visible.fail}${r.visible.failed.length ? ` failed: ${r.visible.failed.join(' | ')}` : ''}` + ); + console.log(` changed ${r.changedFiles.join(', ') || '(nothing)'}`); + if (r.protectedTouched.length) console.log(` PROTECTED ${r.protectedTouched.join(', ')}`); + if (r.junkFiles.length) console.log(` junk files ${r.junkFiles.join(', ')}`); + for (const c of r.checks) console.log(` check ${c.ok ? 'ok ' : 'FAIL'} ${c.why}${c.detail ? ` (${c.detail})` : ''}`); + for (const c of r.bonus) console.log(` bonus ${c.ok ? 'ok ' : 'miss'} ${c.why}`); + if (!s.sessions) { + console.log(' session none found for this folder (was the chat run in this workspace?)'); + return; + } + console.log( + ` agent ${s.llmCalls} LLM calls · ${s.tools} tools (${s.toolFails} failed) · ${s.writes} writes · ${s.shells} shells (${s.shellFails} failed)` + ); + console.log( + ` reads ${s.reads} (${s.readUniquePaths} files, ${s.readRepeats} repeated windows) · tools: ${Object.entries(s.byTool) + .map(([k, v]) => `${k}=${v}`) + .join(' ')}` + ); + console.log( + ` context ${s.tokensIn} tokens in (max prompt ${s.maxPromptTokens}) · ${s.tokensOut} out · ${s.compacts} compacts · ${s.overflowErrors} overflows · ${s.llmErrors} LLM errors` + ); + console.log(` time ${Math.round(s.wallSeconds / 60)} min · ${s.userMessages} user messages · ${s.sessions} session(s)`); + console.log( + ` honesty ${r.falseDone ? 'FALSE DONE (claimed green, tests red)' : s.claimedGreen ? 'claimed green (true)' : 'no green claim'} · harness text ${s.harnessChars ?? 0} chars` + ); +} + +function cmdReport() { + const results = (fs.existsSync(RESULTS_DIR) ? fs.readdirSync(RESULTS_DIR) : []) + .filter(f => f.endsWith('.json')) + .map(f => readJson(path.join(RESULTS_DIR, f))) + .filter(Boolean) + .sort((a, b) => a.task.localeCompare(b.task) || a.label.localeCompare(b.label)); + if (!results.length) die('No results yet. Use: check'); + + const rows = results.map(r => ({ + task: r.task.split('-')[0], + label: r.label, + rt: r.runtime || 'node', + model: (r.model || '-').slice(0, 18), + pass: r.pass ? 'PASS' : 'fail', + score: r.score, + class: classifyFailure(r) || (r.pass ? '' : '?'), + exit: r.loopExit || '', + hidden: r.review ? `F1 ${r.review.f1}` : `${r.hidden.pass}/${r.hidden.pass + r.hidden.fail}`, + cards: r.hidden.cards ? r.hidden.cards.filter(c => c.ratio === 1).length + `/${r.hidden.cards.length}` : '', + falseDone: r.falseDone ? 'YES' : '', + llm: r.session.llmCalls, + harness: r.session.harnessChars ?? '-', + tools: r.session.tools, + writes: r.session.writes, + min: Math.round(r.session.wallSeconds / 60), + })); + console.table(rows); + const tallies = { 'A-harness': 0, 'B-protocol': 0, 'C-semantic': 0 }; + for (const r of results) { + const c = classifyFailure(r); + if (!c) continue; + if (c.includes('A-harness')) tallies['A-harness'] += 1; + if (c.includes('B-protocol')) tallies['B-protocol'] += 1; + if (c.includes('C-semantic')) tallies['C-semantic'] += 1; + } + console.log( + `Failure classes: A-harness=${tallies['A-harness']} B-protocol=${tallies['B-protocol']} C-semantic=${tallies['C-semantic']}` + ); +} + +function modelSafe(raw) { + return String(raw ?? '') + .trim() + .replace(/[^\w.-]+/g, '_') + .replace(/^_+|_+$/g, '') + .slice(0, 120) || 'unknown'; +} + +/** + * Calibrate one model (UI / --model) or A/B weak+strong (lab CLI). + * Classification is from protocol stats, not from the model name. + */ +function cmdCalibrate(flags) { + const settings = readJson(path.join(os.homedir(), '.CodeForge', 'ai', 'settings.json')) || {}; + const serverId = String(flags.server || settings.activeServerId || 'default'); + const singleModel = String(flags.model || '').trim(); + const weakFlag = String(flags['weak-model'] || flags.weakModel || '').trim(); + const strongFlag = String(flags['strong-model'] || flags.strongModel || '').trim(); + const models = singleModel + ? [singleModel] + : weakFlag && strongFlag + ? [weakFlag, strongFlag] + : [ + settings.activeModel || weakFlag || 'qwen3.6-35b', + ].filter(Boolean); + if (!models.length) die('calibrate needs --model (or --weak-model + --strong-model)'); + const reps = Math.max(1, Number(flags.reps || 3) || 3); + const suite = ['T2', 'T4', 'CS2', 'CS3']; + + if (!flags.skipBuild && !LAYOUT.packaged) { + const compile = spawnSync( + 'npx', + ['tsc', '-p', 'extensions/codeforge/tsconfig.json'], + { + cwd: LAYOUT.repoRoot, + encoding: 'utf8', + shell: true, + timeout: 300_000, + } + ); + if (compile.status !== 0) { + die(`tsc failed:\n${compile.stdout}\n${compile.stderr}`); + } + } + + const require = createRequire(import.meta.url); + const { + loadModelProfile, + saveModelProfile, + applyCalibrationFromStats, + } = require(path.join(LAYOUT.extOut, 'agent', 'modelProfile.js')); + + const byModel = {}; + for (const model of models) { + const results = []; + const labelBase = `calibrate-${modelSafe(model)}`; + for (let rep = 1; rep <= reps; rep++) { + for (const task of suite) { + console.log(`\n=== calibrate ${model} ${task} rep ${rep}/${reps} ===`); + try { + const r = cmdRun(task, { + ...flags, + label: `${labelBase}-r${rep}`, + model, + server: serverId, + skipBuild: true, + disableHysteresis: true, + // Session weak harness on during calibrate so the suite matches product weak path; + // profile knobs after classify decide long-term cost. + weak: 'on', + }); + results.push(r); + } catch (err) { + console.error(err); + results.push({ pass: false, score: 0, protocol: {}, task }); + } + } + } + byModel[model] = results; + } + + const date = new Date().toISOString().slice(0, 10); + const lines = [ + `# Calibration — ${date}`, + '', + `Models: ${models.map(m => `\`${m}\``).join(', ')} · reps=${reps} · suite=${suite.join(',')}`, + '', + '| Model | Class | Pass | Mean score | badArgs/native | lengthExits | Knobs |', + '|---|---|---:|---:|---:|---:|---|', + ]; + + for (const model of models) { + const results = byModel[model] || []; + const n = results.length || 1; + const passes = results.filter(r => r.pass).length; + const meanScore = results.reduce((s, r) => s + (Number(r.score) || 0), 0) / n; + let badArgs = 0; + let native = 0; + let lengthExits = 0; + for (const r of results) { + const p = r.protocol || {}; + badArgs += Number(p.badArgs) || 0; + native += Number(p.nativeToolCalls) || 0; + lengthExits += Number(p.lengthExits) || 0; + } + const badArgsRate = badArgs / Math.max(1, native); + const passRate = passes / n; + let profile = loadModelProfile(serverId, model, { applyHysteresis: false }); + profile.calibration = { + at: new Date().toISOString(), + suite: suite.join(','), + passRate, + meanScore, + protocolMeans: { + badArgs: badArgs / n, + nativeToolCalls: native / n, + lengthExits: lengthExits / n, + }, + }; + const { profile: next, classification } = applyCalibrationFromStats(profile, { + passRate, + badArgsRate, + lengthExits, + totalRuns: n, + nativeToolCalls: native, + }); + profile = next; + profile.calibration.classification = classification; + // Prefer text transport when native parse leaks arguments (Qwen XML closers). + const refusals = results.reduce((s, r) => { + const ref = r.refusals || {}; + return s + Object.values(ref).reduce((a, b) => a + Number(b || 0), 0); + }, 0); + if (badArgsRate >= 0.1 || (native > 0 && refusals / Math.max(1, native) >= 0.1)) { + profile.toolTransport = 'text'; + } else if (classification === 'strong' && badArgsRate < 0.05) { + profile.toolTransport = 'native'; + } + saveModelProfile(profile); + const knobs = + classification === 'strong' + ? `strong: writeMax=${profile.mutations.writeMaxContentLines}, massRewrite=${profile.mutations.enforceMassRewriteGate}, weakSkill=${profile.prompt.forcedWeakSkill}, transport=${profile.toolTransport || 'native'}` + : `weak: defaults, transport=${profile.toolTransport || 'native'}`; + lines.push( + `| ${model} | ${classification} | ${passes}/${n} | ${meanScore.toFixed(1)} | ${badArgsRate.toFixed(3)} | ${lengthExits} | ${knobs.replace(/\|/g, '/')} |` + ); + } + + lines.push(''); + lines.push( + 'Class **strong** when `passRate ≥ 0.5`, `badArgs/native < 0.05`, `lengthExits === 0`, and native tool calls > 0; otherwise **weak**. High badArgs → `toolTransport=text`.' + ); + lines.push('Profiles: `~/.CodeForge/ai/model-profiles///profile.json`.'); + const outMd = path.join(RESULTS_DIR, `CALIBRATE-${date}.md`); + fs.mkdirSync(RESULTS_DIR, { recursive: true }); + fs.writeFileSync(outMd, lines.join('\n') + '\n', 'utf8'); + console.log(lines.join('\n')); + console.log(`\nWrote ${outMd}`); +} + +function isInfrastructureError(r) { + return !!(r?.infrastructureError || r?.notRun || r?.loopExit === 'error'); +} + +function emptyPairedByTask(tasks) { + return Object.fromEntries(tasks.map(t => [t, { parent: [], candidate: [] }])); +} + +async function probeReviseLlm(server, model) { + try { + const res = await fetch(`${String(server.baseUrl).replace(/\/$/, '')}/chat/completions`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(server.apiKey ? { Authorization: `Bearer ${server.apiKey}` } : {}), + }, + body: JSON.stringify({ + model, + max_tokens: 1, + messages: [{ role: 'user', content: 'ping' }], + }), + signal: AbortSignal.timeout(15_000), + }); + return res.ok; + } catch { + return false; + } +} + +function writeReviseDeferredSnapshot(rev, serverId, model, reason) { + const dir = path.dirname(rev.harnessFilePath(serverId, model)); + fs.mkdirSync(dir, { recursive: true }); + const snap = { + at: new Date().toISOString(), + note: 'Harness revise deferred — local LLM was not reachable (same pattern as offline baselines).', + reason, + }; + const fp = path.join(dir, 'revise-deferred.json'); + fs.writeFileSync(fp, JSON.stringify(snap, null, 2), 'utf8'); + console.log(`Deferred harness revision (no live LLM). Wrote ${fp}`); + return fp; +} + +/** + * One test-time harness revision. The same model proposes; T2/T4 score the + * change (CS2/CS3 stay held out). harness.json updates only when a keep survives + * holdout checks. + */ +async function cmdRevise(flags) { + const settings = readJson(path.join(os.homedir(), '.CodeForge', 'ai', 'settings.json')) || {}; + const serverId = String(flags.server || settings.activeServerId || 'default'); + const model = String( + flags.model || + (settings.activeServerId === serverId ? settings.activeModel : '') || + '' + ).trim(); + if (!model) die('revise needs --model (or an active model in settings)'); + const repsEach = Math.max(1, Number(flags.reps || 3) || 3); + const suite = String(flags.tasks || 'T2,T4') + .split(',') + .map(s => s.trim()) + .filter(Boolean); + if (!suite.length) die('revise needs at least one task'); + + const server = (settings.servers || []).find(s => s.id === serverId && s.enabled !== false); + if (!server?.baseUrl?.trim()) die(`no usable server (id=${serverId})`); + + const require = createRequire(import.meta.url); + // Probe LLM before compile / score-count gates so offline runs defer immediately. + const revEarly = (() => { + try { + return require(path.join(LAYOUT.extOut, 'agent', 'harnessRevision.js')); + } catch { + return null; + } + })(); + const llmOkEarly = await probeReviseLlm(server, model); + if (!llmOkEarly) { + if (revEarly) { + writeReviseDeferredSnapshot(revEarly, serverId, model, 'LLM probe failed'); + return; + } + die('LLM probe failed and harnessRevision.js is not built (run build:extensions)'); + } + + if (repsEach * suite.length < 3) { + die( + `revise needs at least 3 scores (tasks × reps). Got ${suite.length}×${repsEach}. One run is noise.` + ); + } + + if (!flags.skipBuild && !LAYOUT.packaged) { + const compile = spawnSync('npx', ['tsc', '-p', 'extensions/codeforge/tsconfig.json'], { + cwd: LAYOUT.repoRoot, + encoding: 'utf8', + shell: true, + timeout: 300_000, + }); + if (compile.status !== 0) die(`tsc failed:\n${compile.stdout}\n${compile.stderr}`); + } + + const rev = revEarly || require(path.join(LAYOUT.extOut, 'agent', 'harnessRevision.js')); + const telemetry = require(path.join(LAYOUT.extOut, 'agent', 'protocolTelemetry.js')); + const file = rev.harnessFilePath(serverId, model); + const parent = rev.loadHarnessArtifact(serverId, model, file); + const report = rev.formatExecutionReport(telemetry.loadModelRollup(serverId, model)); + console.log(`Harness revision for ${model} on ${server.name || serverId}`); + console.log(`Selection suite: ${suite.join(', ')} × ${repsEach} (held out of this decision: tasks not listed).`); + console.log(report); + + const parentOutcomes = []; + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cf-revise-harness-')); + const taskKeys = suite.map(k => findTask(k).id.split('-')[0]); + + const benchRun = (taskKey, labelSuffix, harnessFile) => { + const r = cmdRun(taskKey, { + ...flags, + model, + server: serverId, + label: labelSuffix, + skipBuild: true, + disableHysteresis: true, + softFail: true, + harnessFile, + }); + if (isInfrastructureError(r)) { + console.warn(`Skipping infrastructure error for ${taskKey} (${labelSuffix})`); + return undefined; + } + const score = Number(r?.score); + if (!Number.isFinite(score)) { + throw new Error(`no score for ${taskKey} (${labelSuffix})`); + } + return { score, r, taskKey }; + }; + + const harnessFileFor = artifact => { + if (artifact.revision === parent.revision) return file; + const fp = path.join(tempDir, `harness-r${artifact.revision}.json`); + rev.saveHarnessArtifact(artifact, fp); + return fp; + }; + + const pushOutcome = (taskId, r, score) => { + parentOutcomes.push({ + task: taskId, + score, + loopExit: r.loopExit || undefined, + steps: typeof r.steps === 'number' ? r.steps : undefined, + hiddenFailed: r.hidden?.failed ?? [], + checksFailed: (r.checks ?? []) + .filter(c => !c.ok) + .map(c => [c.why || c.type, c.detail].filter(Boolean).join(' ')), + replyTail: + typeof r.replyTail === 'string' + ? r.replyTail + : typeof r.headlessReply === 'string' + ? r.headlessReply.slice(-500) + : undefined, + }); + }; + + const scorePhase = async input => { + const byTask = emptyPairedByTask(taskKeys); + if (input.phase === 'parent') { + for (const task of suite) { + const taskId = findTask(task).id; + const short = taskId.split('-')[0]; + for (let rep = 1; rep <= repsEach; rep++) { + console.log(`\n=== revise parent ${short} rep ${rep}/${repsEach} ===`); + const run = benchRun( + task, + `revise-parent-${short}-rep${rep}`, + harnessFileFor(input.artifact) + ); + if (!run) continue; + byTask[short].parent.push(run.score); + pushOutcome(taskId, run.r, run.score); + } + } + const text = rev.formatTaskOutcomes(parentOutcomes); + if (text) console.log(`\n${text}`); + return { byTask }; + } + + for (const task of suite) { + const taskId = findTask(task).id; + const short = taskId.split('-')[0]; + for (let rep = 1; rep <= repsEach; rep++) { + console.log(`\n=== revise paired ${short} rep ${rep}/${repsEach} (parent) ===`); + const parentRun = benchRun( + task, + `revise-paired-parent-${short}-rep${rep}`, + harnessFileFor(input.parent) + ); + if (!parentRun) continue; + byTask[short].parent.push(parentRun.score); + + console.log(`\n=== revise paired ${short} rep ${rep}/${repsEach} (candidate r${input.candidate.revision}) ===`); + const candRun = benchRun( + task, + `revise-paired-cand-r${input.candidate.revision}-${short}-rep${rep}`, + harnessFileFor(input.candidate) + ); + if (!candRun) continue; + byTask[short].candidate.push(candRun.score); + } + } + return { byTask }; + }; + + const scoreHoldout = async (artifact, holdoutTasks, labelPrefix, side) => { + const byTask = emptyPairedByTask(holdoutTasks.map(t => findTask(t).id.split('-')[0])); + for (const task of holdoutTasks) { + const short = findTask(task).id.split('-')[0]; + for (let rep = 1; rep <= repsEach; rep++) { + console.log(`\n=== ${labelPrefix} holdout ${short} rep ${rep}/${repsEach} ===`); + const run = benchRun( + task, + `${labelPrefix}-holdout-${short}-rep${rep}`, + harnessFileFor(artifact) + ); + if (!run) continue; + byTask[short][side].push(run.score); + } + } + return byTask; + }; + + try { + let outcome = await rev.executeHarnessRevision({ + artifact: parent, + report, + minRepsPerTask: 3, + parentOutcomes: () => parentOutcomes, + propose: async messages => { + const res = await fetch(`${String(server.baseUrl).replace(/\/$/, '')}/chat/completions`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(server.apiKey ? { Authorization: `Bearer ${server.apiKey}` } : {}), + }, + body: JSON.stringify({ + model, + temperature: 0.2, + max_tokens: 4096, + enable_thinking: false, + chat_template_kwargs: { enable_thinking: false }, + messages, + }), + }); + const body = await res.text(); + if (!res.ok) throw new Error(`proposer HTTP ${res.status}: ${body.slice(0, 500)}`); + const data = JSON.parse(body); + const message = data?.choices?.[0]?.message ?? {}; + const content = [message.content, message.reasoning_content, message.reasoning] + .filter(part => typeof part === 'string' && part.trim()) + .join('\n'); + if (!content.trim()) throw new Error('proposer returned an empty message'); + console.log(`Proposer reply:\n${content.slice(0, 1500)}`); + return content; + }, + score: scorePhase, + }); + + if (outcome.kept) { + const holdout = ['CS2', 'CS3']; + console.log(`\nHoldout validation: ${holdout.join(', ')} × ${repsEach}`); + const parentHold = await scoreHoldout(parent, holdout, 'parent', 'parent'); + const candHold = await scoreHoldout(outcome.artifact, holdout, 'candidate', 'candidate'); + const holdReason = rev.holdoutRegressed(parentHold, candHold); + if (holdReason) { + console.log(`Holdout failed: ${holdReason}`); + const entry = { + at: new Date().toISOString(), + decision: 'revert', + reason: `holdout: ${holdReason}`, + diagnosis: outcome.diagnosis, + parentMean: outcome.parentMean, + candidateMean: outcome.candidateMean, + parentReps: outcome.artifact.history.at(-1)?.parentReps ?? 0, + candidateReps: outcome.artifact.history.at(-1)?.candidateReps ?? 0, + }; + outcome = { + ...outcome, + kept: false, + reason: entry.reason, + artifact: { + ...parent, + history: [...parent.history, entry].slice(-20), + }, + }; + } + } + + if (outcome.kept) { + rev.saveHarnessArtifact(outcome.artifact, file); + } else if (outcome.scored || outcome.artifact.history.length > parent.history.length) { + rev.saveHarnessArtifact(outcome.artifact, file); + } + console.log( + `\n${outcome.kept ? 'KEPT' : 'REVERTED'} revision ${outcome.kept ? outcome.artifact.revision : parent.revision}: ${outcome.reason}` + ); + if (outcome.diagnosis) console.log(`Diagnosis: ${outcome.diagnosis}`); + console.log(`Harness file: ${file}`); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +// --------------------------------------------------------------------------- + +const { pos, flags } = parseArgs(process.argv.slice(2)); +switch (pos[0]) { + case 'list': + for (const t of loadTasks()) { + const rt = taskRuntime(t); + console.log( + `${t.id.padEnd(24)} ${(t.category || '').padEnd(11)} ${rt.padEnd(7)} ${t.title}` + ); + } + break; + case 'setup': + cmdSetup(pos[1], flags); + break; + case 'check': + cmdCheck(pos[1]); + break; + case 'report': + cmdReport(); + break; + case 'selftest': + cmdSelftest(pos[1]); + break; + case 'run': + cmdRun(pos[1], flags); + break; + case 'calibrate': + cmdCalibrate(flags); + break; + case 'revise': + cmdRevise(flags).catch(err => { + console.error(err instanceof Error ? err.message : err); + process.exit(1); + }); + break; + default: + console.log( + 'Usage: node benchmarks/bench.mjs --label [--open] | check [run|latest] | report | selftest [CS] | run --label | calibrate [--model m | --weak-model m --strong-model m] [--reps n] | revise [--model m] [--reps n] [--tasks T2,T4]>' + ); +} diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/.gitignore b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/.gitignore new file mode 100644 index 0000000..1faca62 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/.gitignore @@ -0,0 +1,6 @@ +bin/ +obj/ +.vs/ +*.user +*.suo +TestResults/ diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/Acme.Billing.slnx b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/Acme.Billing.slnx new file mode 100644 index 0000000..e1bab1b --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/Acme.Billing.slnx @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/README.md b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/README.md new file mode 100644 index 0000000..3a41574 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/README.md @@ -0,0 +1,15 @@ +# Acme.Billing + +Small .NET 10 billing library used by the CodeForge C# harness benchmark. + +``` +Acme.Billing.slnx +src/Acme.Billing/ — cart, pricing, coupons +tests/Acme.Billing.Tests/ +docs/pricing.md +config/coupons.json +``` + +```bash +dotnet test +``` diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/config/coupons.json b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/config/coupons.json new file mode 100644 index 0000000..b5fb234 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/config/coupons.json @@ -0,0 +1,5 @@ +[ + { "code": "WELCOME10", "type": "percent", "value": 10, "expires": "2099-12-31" }, + { "code": "FIVEOFF", "type": "fixed", "value": 5, "expires": "2099-12-31" }, + { "code": "OLD20", "type": "percent", "value": 20, "expires": "2020-01-01" } +] diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/docs/pricing.md b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/docs/pricing.md new file mode 100644 index 0000000..0e7099d --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/docs/pricing.md @@ -0,0 +1,36 @@ +# Pricing rules + +## Order of operations + +1. Subtotal = sum of `Price * Qty` for every item. +2. Discount = subtotal × the best single rate (customer tier or bulk — they do not stack). +3. Net = subtotal − discount. +4. Tax = net × regional VAT (PT 23%, ES 21%, DE 19%, FR 20%; unknown = 0). +5. Total = net + tax. + +Amounts are rounded to 2 decimals (`MidpointRounding.AwayFromZero`). + +## Customer tiers + +| Tier | Discount | +|----------|----------| +| standard | 0% | +| silver | 3% | +| gold | 5% | +| vip | 10% | + +## Bulk quantity + +10+ units: 5% · 25+ units: 8% · 50+ units: 12%. + +## Coupons + +`Coupons.ApplyCoupon(Cart cart, string code, DateOnly? today = null)` returns the coupon +discount amount (does not mutate the cart). Coupons live in `config/coupons.json`. + +- Codes are case-insensitive. +- `percent`: `subtotal × value / 100`. +- `fixed`: `value`, never more than the subtotal. +- Valid through the `expires` day inclusive; after that throw `ExpiredCouponException`. +- Unknown codes throw `InvalidCouponException`. +- Result rounded to 2 decimals. diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Acme.Billing.csproj b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Acme.Billing.csproj new file mode 100644 index 0000000..b760144 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Acme.Billing.csproj @@ -0,0 +1,9 @@ + + + + net10.0 + enable + enable + + + diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Cart.cs b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Cart.cs new file mode 100644 index 0000000..adcdb55 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Cart.cs @@ -0,0 +1,40 @@ +namespace Acme.Billing; + +public sealed class Cart +{ + public string Tier { get; init; } = "standard"; + public bool Blocked { get; init; } + public List Items { get; } = []; +} + +public sealed class CartItem +{ + public required string Sku { get; init; } + public required decimal Price { get; init; } + public int Qty { get; set; } = 1; +} + +public static class CartOps +{ + public static Cart Create(string tier = "standard", bool blocked = false) => + new() { Tier = tier, Blocked = blocked }; + + public static Cart AddItem(Cart cart, string sku, decimal price, int qty = 1) + { + if (string.IsNullOrWhiteSpace(sku)) throw new ArgumentException("INVALID_ITEM"); + if (qty <= 0) throw new ArgumentException("INVALID_QTY"); + if (price < 0) throw new ArgumentException("INVALID_PRICE"); + var existing = cart.Items.FirstOrDefault(i => i.Sku == sku); + if (existing is not null) + { + existing.Qty += qty; + } + else + { + cart.Items.Add(new CartItem { Sku = sku, Price = price, Qty = qty }); + } + return cart; + } + + public static int ItemCount(Cart cart) => cart.Items.Sum(i => i.Qty); +} diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Coupons.cs b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Coupons.cs new file mode 100644 index 0000000..83fad87 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Coupons.cs @@ -0,0 +1,32 @@ +using System.Text.Json; + +namespace Acme.Billing; + +public sealed record Coupon(string Code, string Type, decimal Value, string Expires); + +public static class Coupons +{ + public static IReadOnlyList Load(string file) + { + var json = File.ReadAllText(file); + return JsonSerializer.Deserialize>(json, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + }) ?? []; + } + + /// + /// Resolve config/coupons.json from the workspace root (walks up from cwd). + /// + public static string FindDefaultFile(string? startDir = null) + { + var dir = new DirectoryInfo(startDir ?? Directory.GetCurrentDirectory()); + while (dir is not null) + { + var candidate = Path.Combine(dir.FullName, "config", "coupons.json"); + if (File.Exists(candidate)) return candidate; + dir = dir.Parent; + } + throw new FileNotFoundException("config/coupons.json not found"); + } +} diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Money.cs b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Money.cs new file mode 100644 index 0000000..33a9f24 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Money.cs @@ -0,0 +1,7 @@ +namespace Acme.Billing; + +public static class Money +{ + public static decimal Round2(decimal n) => + Math.Round(n, 2, MidpointRounding.AwayFromZero); +} diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/PricingRules.cs b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/PricingRules.cs new file mode 100644 index 0000000..8263222 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/PricingRules.cs @@ -0,0 +1,176 @@ +namespace Acme.Billing; + +/// +/// Pricing rules: bulk quantity, seasonal campaigns, loyalty points, shipping and customer tiers. +/// Only CustomerDiscountRate and BulkDiscountRate feed the cart discount. +/// +public static class PricingRules +{ + // --------------------------------------------------------------------------- + // Bulk quantity + // --------------------------------------------------------------------------- + + private static readonly (int MinUnits, decimal Rate)[] BulkTiers = + [ + (50, 0.12m), + (25, 0.08m), + (10, 0.05m), + ]; + + public static int TotalUnits(IEnumerable items) => items.Sum(i => i.Qty); + + public static decimal BulkDiscountRate(IEnumerable items) + { + var units = TotalUnits(items); + foreach (var (min, rate) in BulkTiers) + { + if (units >= min) return rate; + } + return 0m; + } + + // --------------------------------------------------------------------------- + // Seasonal campaigns (storefront banners — not applied by Totals) + // --------------------------------------------------------------------------- + + public static readonly (string Id, string SkuPrefix, decimal Rate, string From, string To)[] Campaigns = + [ + ("spring", "GARDEN-", 0.15m, "2026-03-20", "2026-04-30"), + ("summer", "BEACH-", 0.20m, "2026-06-21", "2026-08-31"), + ("back-to-school", "BOOK-", 0.10m, "2026-09-01", "2026-09-30"), + ("black-friday", "", 0.25m, "2026-11-27", "2026-11-30"), + ]; + + // --------------------------------------------------------------------------- + // Loyalty points (decoy 0.1 — not a cart discount) + // --------------------------------------------------------------------------- + + public const decimal LoyaltyRate = 0.1m; + + public static int LoyaltyPoints(decimal netAmount, string tier = "standard") + { + var multiplier = tier.ToLowerInvariant() switch + { + "silver" => 1.25m, + "gold" => 1.5m, + "vip" => 2m, + _ => 1m, + }; + return (int)Math.Floor(netAmount * LoyaltyRate * multiplier); + } + + // --------------------------------------------------------------------------- + // Shipping + // --------------------------------------------------------------------------- + + public static decimal ShippingCost(decimal netAmount, string? region) + { + var (flat, freeFrom) = (region ?? "").ToUpperInvariant() switch + { + "PT" => (3.5m, 40m), + "ES" => (4.9m, 50m), + "DE" => (6.9m, 60m), + "FR" => (5.9m, 60m), + _ => (12m, decimal.MaxValue), + }; + return netAmount >= freeFrom ? 0m : flat; + } + + // --------------------------------------------------------------------------- + // Validation / describe helpers (padding — keep VIP rate near the bottom) + // --------------------------------------------------------------------------- + + public static bool IsKnownTier(string tier) => + tier.Equals("standard", StringComparison.OrdinalIgnoreCase) + || tier.Equals("silver", StringComparison.OrdinalIgnoreCase) + || tier.Equals("gold", StringComparison.OrdinalIgnoreCase) + || tier.Equals("vip", StringComparison.OrdinalIgnoreCase); + + public static string NormalizeTier(string? tier) + { + var t = (tier ?? "standard").ToLowerInvariant(); + return IsKnownTier(t) ? t : "standard"; + } + + public static string DescribeBulk() => + string.Join("; ", BulkTiers.Select(t => $"{t.MinUnits}+ units: {t.Rate:P0}")); + + public static string DescribeCampaigns() => + string.Join(", ", Campaigns.Select(c => c.Id)); + + public static decimal CampaignRateFor(string sku, DateOnly? today = null) + { + var day = today ?? DateOnly.FromDateTime(DateTime.UtcNow); + decimal best = 0; + foreach (var c in Campaigns) + { + var from = DateOnly.Parse(c.From); + var to = DateOnly.Parse(c.To); + if (day >= from && day <= to && sku.StartsWith(c.SkuPrefix, StringComparison.Ordinal) + && c.Rate > best) + { + best = c.Rate; + } + } + return best; + } + + public static string CampaignBanner(DateOnly? today = null) + { + var day = today ?? DateOnly.FromDateTime(DateTime.UtcNow); + decimal best = 0; + string? id = null; + foreach (var c in Campaigns) + { + var from = DateOnly.Parse(c.From); + var to = DateOnly.Parse(c.To); + if (day >= from && day <= to && c.Rate > best) + { + best = c.Rate; + id = c.Id; + } + } + return id is null ? "" : $"{id}: up to {(int)(best * 100)}% off"; + } + + public static decimal PointsToEuro(int points) => Money.Round2(points / 100m); + + public static decimal? FreeShippingGap(decimal netAmount, string? region) + { + var freeFrom = (region ?? "").ToUpperInvariant() switch + { + "PT" => 40m, + "ES" => 50m, + "DE" => 60m, + "FR" => 60m, + _ => (decimal?)null, + }; + if (freeFrom is null) return null; + return Money.Round2(Math.Max(0, freeFrom.Value - netAmount)); + } + + // --------------------------------------------------------------------------- + // Customer tiers + // --------------------------------------------------------------------------- + + /// + /// Discount rate per customer tier. See docs/pricing.md ("Customer tiers"). + /// + private const decimal SilverRate = 0.03m; + private const decimal GoldRate = 0.05m; + private const decimal VipRate = 0.1m; + + private static readonly Dictionary TierDiscounts = new(StringComparer.OrdinalIgnoreCase) + { + ["standard"] = 0m, + ["silver"] = SilverRate, + ["gold"] = GoldRate, + ["vip"] = VipRate, + }; + + public static decimal CustomerDiscountRate(Cart cart) + { + if (cart.Blocked) return 0m; + return TierDiscounts.TryGetValue(cart.Tier, out var r) ? r : 0m; + } +} diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Tax.cs b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Tax.cs new file mode 100644 index 0000000..08e8d28 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Tax.cs @@ -0,0 +1,18 @@ +namespace Acme.Billing; + +public static class Tax +{ + private static readonly Dictionary Rates = new(StringComparer.OrdinalIgnoreCase) + { + ["PT"] = 0.23m, + ["ES"] = 0.21m, + ["DE"] = 0.19m, + ["FR"] = 0.20m, + }; + + public static decimal Rate(string? region) => + Rates.TryGetValue(region ?? "", out var r) ? r : 0m; + + public static decimal CalcTax(decimal amount, string? region) => + Money.Round2(amount * Rate(region)); +} diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Totals.cs b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Totals.cs new file mode 100644 index 0000000..9f9333d --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/src/Acme.Billing/Totals.cs @@ -0,0 +1,29 @@ +namespace Acme.Billing; + +public static class Totals +{ + public static decimal Subtotal(Cart cart) + { + decimal sum = 0; + for (var i = 0; i < cart.Items.Count; i++) + { + var item = cart.Items[i]; + sum += item.Price * item.Qty; + } + return Money.Round2(sum); + } + + /// Best single discount wins: customer tier or bulk (they do not stack). + public static decimal Discount(Cart cart) + { + var baseAmount = Subtotal(cart); + var rate = Math.Max(PricingRules.CustomerDiscountRate(cart), PricingRules.BulkDiscountRate(cart.Items)); + return Money.Round2(baseAmount * rate); + } + + public static decimal Total(Cart cart, string? region) + { + var net = Money.Round2(Subtotal(cart) - Discount(cart)); + return Money.Round2(net + Tax.CalcTax(net, region)); + } +} diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/tests/Acme.Billing.Tests/Acme.Billing.Tests.csproj b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/tests/Acme.Billing.Tests/Acme.Billing.Tests.csproj new file mode 100644 index 0000000..57a7377 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/tests/Acme.Billing.Tests/Acme.Billing.Tests.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/billing/tests/Acme.Billing.Tests/CartTests.cs b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/tests/Acme.Billing.Tests/CartTests.cs new file mode 100644 index 0000000..f6cc399 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/billing/tests/Acme.Billing.Tests/CartTests.cs @@ -0,0 +1,58 @@ +using Acme.Billing; + +namespace Acme.Billing.Tests; + +public class CartTests +{ + [Fact] + public void AddItem_merges_same_sku() + { + var cart = CartOps.Create(); + CartOps.AddItem(cart, "PEN", 1.5m, 2); + CartOps.AddItem(cart, "PEN", 1.5m, 3); + Assert.Single(cart.Items); + Assert.Equal(5, CartOps.ItemCount(cart)); + } + + [Fact] + public void AddItem_rejects_invalid_qty() + { + Assert.Throws(() => CartOps.AddItem(CartOps.Create(), "PEN", 1m, 0)); + } + + [Fact] + public void Subtotal_single_item() + { + var cart = CartOps.Create(); + CartOps.AddItem(cart, "MUG", 7.25m, 2); + Assert.Equal(14.5m, Totals.Subtotal(cart)); + } + + [Fact] + public void Subtotal_several_items() + { + var cart = CartOps.Create(); + CartOps.AddItem(cart, "MUG", 7.25m, 2); + CartOps.AddItem(cart, "PEN", 1.5m, 4); + CartOps.AddItem(cart, "BAG", 12m, 1); + Assert.Equal(32.5m, Totals.Subtotal(cart)); + } + + [Fact] + public void Bulk_discount_from_10_units() + { + var cart = CartOps.Create(); + CartOps.AddItem(cart, "PEN", 2m, 10); + Assert.Equal(1m, Totals.Discount(cart)); + } + + [Fact] + public void Total_adds_regional_vat() + { + var cart = CartOps.Create(); + CartOps.AddItem(cart, "MUG", 10m); + CartOps.AddItem(cart, "BAG", 20m); + Assert.Equal(36.9m, Totals.Total(cart, "PT")); + Assert.Equal(30m, Totals.Total(cart, "XX")); + } +} diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/README.md b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/README.md new file mode 100644 index 0000000..23f58fb --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/README.md @@ -0,0 +1,11 @@ +# shop + +Small pricing library for a shopping cart (CommonJS, no dependencies). + +- `src/cart/` — cart operations and totals +- `src/pricing/` — tax, discount rules and coupons +- `src/checkout/` — invoice building and formatting +- `config/` — coupon definitions +- `docs/pricing.md` — business rules + +Run the tests with `npm test` (Node's built-in test runner). diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/config/coupons.json b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/config/coupons.json new file mode 100644 index 0000000..b5fb234 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/config/coupons.json @@ -0,0 +1,5 @@ +[ + { "code": "WELCOME10", "type": "percent", "value": 10, "expires": "2099-12-31" }, + { "code": "FIVEOFF", "type": "fixed", "value": 5, "expires": "2099-12-31" }, + { "code": "OLD20", "type": "percent", "value": 20, "expires": "2020-01-01" } +] diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/docs/pricing.md b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/docs/pricing.md new file mode 100644 index 0000000..b4671cb --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/docs/pricing.md @@ -0,0 +1,36 @@ +# Pricing rules + +## Order of operations + +1. Subtotal = sum of `price * qty` for every item. +2. Discount = subtotal × the best single rate (customer tier or bulk quantity — they do not stack). +3. Net = subtotal − discount. +4. Tax = net × regional VAT rate (PT 23%, ES 21%, DE 19%, FR 20%; unknown regions pay no VAT). +5. Total = net + tax. + +All amounts are rounded to 2 decimals. + +## Customer tiers + +| Tier | Discount | +|----------|----------| +| standard | 0% | +| silver | 3% | +| gold | 5% | +| vip | 10% | + +## Bulk quantity + +10+ units: 5% · 25+ units: 8% · 50+ units: 12%. + +## Coupons + +`applyCoupon(cart, code, now = new Date())` returns the coupon discount amount for a cart +(it does not mutate the cart). Coupons are defined in `config/coupons.json`. + +- Codes are case-insensitive (`welcome10` is the same as `WELCOME10`). +- `percent` coupons: `subtotal × value / 100`. +- `fixed` coupons: `value`, but never more than the subtotal. +- A coupon is valid up to and including its `expires` date. After that, throw `new Error('EXPIRED_COUPON')`. +- Unknown codes throw `new Error('INVALID_COUPON')`. +- The result is rounded to 2 decimals. diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/package.json b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/package.json new file mode 100644 index 0000000..fad903f --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/package.json @@ -0,0 +1,10 @@ +{ + "name": "shop-fixture", + "version": "1.0.0", + "private": true, + "description": "Small shopping-cart pricing library used by the CodeForge harness benchmark.", + "main": "src/index.js", + "scripts": { + "test": "node --test" + } +} diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/cart/cart.js b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/cart/cart.js new file mode 100644 index 0000000..cf4d658 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/cart/cart.js @@ -0,0 +1,30 @@ +'use strict'; + +function createCart(customer = { tier: 'standard' }) { + return { customer, items: [] }; +} + +function addItem(cart, item) { + if (!item || !item.sku) throw new Error('INVALID_ITEM'); + const qty = item.qty ?? 1; + if (!Number.isInteger(qty) || qty <= 0) throw new Error('INVALID_QTY'); + if (typeof item.price !== 'number' || item.price < 0) throw new Error('INVALID_PRICE'); + const existing = cart.items.find(i => i.sku === item.sku); + if (existing) { + existing.qty += qty; + } else { + cart.items.push({ sku: item.sku, price: item.price, qty }); + } + return cart; +} + +function removeItem(cart, sku) { + cart.items = cart.items.filter(i => i.sku !== sku); + return cart; +} + +function itemCount(cart) { + return cart.items.reduce((n, i) => n + i.qty, 0); +} + +module.exports = { createCart, addItem, removeItem, itemCount }; diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/cart/totals.js b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/cart/totals.js new file mode 100644 index 0000000..8749b41 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/cart/totals.js @@ -0,0 +1,28 @@ +'use strict'; + +const { round2 } = require('../util/money'); +const { calcTax } = require('../pricing/tax'); +const { customerDiscountRate, bulkDiscountRate } = require('../pricing/rules'); + +function subtotal(cart) { + let sum = 0; + for (let i = 0; i < cart.items.length; i++) { + const item = cart.items[i]; + sum += item.price * item.qty; + } + return round2(sum); +} + +/** The best single discount wins: customer tier or bulk quantity (they do not stack). */ +function discount(cart) { + const base = subtotal(cart); + const rate = Math.max(customerDiscountRate(cart.customer), bulkDiscountRate(cart.items)); + return round2(base * rate); +} + +function total(cart, region) { + const net = round2(subtotal(cart) - discount(cart)); + return round2(net + calcTax(net, region)); +} + +module.exports = { subtotal, discount, total }; diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/checkout/invoice.js b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/checkout/invoice.js new file mode 100644 index 0000000..66f3c1a --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/checkout/invoice.js @@ -0,0 +1,42 @@ +'use strict'; + +const { round2, formatEuro } = require('../util/money'); +const { subtotal, discount } = require('../cart/totals'); +const { calcTax, taxRate } = require('../pricing/tax'); + +function buildInvoice(cart, region) { + const lines = cart.items.map(i => ({ + sku: i.sku, + qty: i.qty, + unit: i.price, + amount: round2(i.price * i.qty), + })); + const sub = subtotal(cart); + const disc = discount(cart); + const net = round2(sub - disc); + const tax = calcTax(net, region); + return { + lines, + subtotal: sub, + discount: disc, + net, + taxRate: taxRate(region), + tax, + total: round2(net + tax), + }; +} + +function formatInvoice(invoice) { + const rows = invoice.lines.map( + l => `${l.sku.padEnd(12)} ${String(l.qty).padStart(3)} x ${formatEuro(l.unit)} = ${formatEuro(l.amount)}` + ); + return [ + ...rows, + `Subtotal: ${formatEuro(invoice.subtotal)}`, + `Discount: -${formatEuro(invoice.discount)}`, + `Tax (${Math.round(invoice.taxRate * 100)}%): ${formatEuro(invoice.tax)}`, + `Total: ${formatEuro(invoice.total)}`, + ].join('\n'); +} + +module.exports = { buildInvoice, formatInvoice }; diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/index.js b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/index.js new file mode 100644 index 0000000..339bfc8 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/index.js @@ -0,0 +1,26 @@ +'use strict'; + +const cart = require('./cart/cart'); +const totals = require('./cart/totals'); +const tax = require('./pricing/tax'); +const rules = require('./pricing/rules'); +const coupons = require('./pricing/coupons'); +const invoice = require('./checkout/invoice'); + +module.exports = { + createCart: cart.createCart, + addItem: cart.addItem, + removeItem: cart.removeItem, + itemCount: cart.itemCount, + subtotal: totals.subtotal, + discount: totals.discount, + total: totals.total, + calcTax: tax.calcTax, + taxRate: tax.taxRate, + customerDiscountRate: rules.customerDiscountRate, + bulkDiscountRate: rules.bulkDiscountRate, + shippingCost: rules.shippingCost, + loadCoupons: coupons.loadCoupons, + buildInvoice: invoice.buildInvoice, + formatInvoice: invoice.formatInvoice, +}; diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/pricing/coupons.js b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/pricing/coupons.js new file mode 100644 index 0000000..be364fb --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/pricing/coupons.js @@ -0,0 +1,12 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const COUPONS_FILE = path.join(__dirname, '..', '..', 'config', 'coupons.json'); + +function loadCoupons(file = COUPONS_FILE) { + return JSON.parse(fs.readFileSync(file, 'utf8')); +} + +module.exports = { COUPONS_FILE, loadCoupons }; diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/pricing/rules.js b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/pricing/rules.js new file mode 100644 index 0000000..78c430a --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/pricing/rules.js @@ -0,0 +1,195 @@ +'use strict'; + +/** + * Pricing rules: bulk quantity, seasonal campaigns, loyalty points, shipping and customer tiers. + * + * Only `customerDiscountRate` and `bulkDiscountRate` feed the cart discount (see src/cart/totals.js). + * Campaigns, loyalty and shipping are exposed for the storefront and reports. + */ + +const { round2 } = require('../util/money'); + +// --------------------------------------------------------------------------- +// Bulk quantity +// --------------------------------------------------------------------------- + +/** Tiers are evaluated from the largest threshold down; the first match wins. */ +const BULK_TIERS = [ + { minUnits: 50, rate: 0.12 }, + { minUnits: 25, rate: 0.08 }, + { minUnits: 10, rate: 0.05 }, +]; + +function totalUnits(items) { + return (items || []).reduce((n, i) => n + (i.qty || 0), 0); +} + +function bulkDiscountRate(items) { + const units = totalUnits(items); + for (const tier of BULK_TIERS) { + if (units >= tier.minUnits) { + return tier.rate; + } + } + return 0; +} + +// --------------------------------------------------------------------------- +// Seasonal campaigns +// --------------------------------------------------------------------------- + +/** + * Campaigns apply to a SKU prefix during a date window (inclusive). + * They are shown as banners; the cart does not apply them automatically. + */ +const CAMPAIGNS = [ + { id: 'spring', skuPrefix: 'GARDEN-', rate: 0.15, from: '2026-03-20', to: '2026-04-30' }, + { id: 'summer', skuPrefix: 'BEACH-', rate: 0.2, from: '2026-06-21', to: '2026-08-31' }, + { id: 'back-to-school', skuPrefix: 'BOOK-', rate: 0.1, from: '2026-09-01', to: '2026-09-30' }, + { id: 'black-friday', skuPrefix: '', rate: 0.25, from: '2026-11-27', to: '2026-11-30' }, +]; + +function parseDay(isoDay) { + const [y, m, d] = String(isoDay).split('-').map(Number); + return new Date(Date.UTC(y, m - 1, d)); +} + +function isWithin(now, from, to) { + const t = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()); + return t >= parseDay(from).getTime() && t <= parseDay(to).getTime(); +} + +function activeCampaigns(now = new Date()) { + return CAMPAIGNS.filter(c => isWithin(now, c.from, c.to)); +} + +function campaignRateFor(sku, now = new Date()) { + let best = 0; + for (const c of activeCampaigns(now)) { + if (String(sku).startsWith(c.skuPrefix) && c.rate > best) { + best = c.rate; + } + } + return best; +} + +function campaignBanner(now = new Date()) { + const active = activeCampaigns(now); + if (!active.length) return ''; + const top = active.reduce((a, b) => (b.rate > a.rate ? b : a)); + return `${top.id}: up to ${Math.round(top.rate * 100)}% off`; +} + +// --------------------------------------------------------------------------- +// Loyalty points +// --------------------------------------------------------------------------- + +/** Points earned per euro of net spend. */ +const LOYALTY_RATE = 0.1; + +/** Tier multipliers for loyalty points (not discounts). */ +const LOYALTY_MULTIPLIER = { + standard: 1, + silver: 1.25, + gold: 1.5, + vip: 2, +}; + +function loyaltyPoints(netAmount, tier = 'standard') { + const multiplier = LOYALTY_MULTIPLIER[tier] ?? 1; + return Math.floor(netAmount * LOYALTY_RATE * multiplier); +} + +function pointsToEuro(points) { + // 100 points = 1 EUR voucher + return round2(points / 100); +} + +// --------------------------------------------------------------------------- +// Shipping +// --------------------------------------------------------------------------- + +const SHIPPING = { + PT: { flat: 3.5, freeFrom: 40 }, + ES: { flat: 4.9, freeFrom: 50 }, + DE: { flat: 6.9, freeFrom: 60 }, + FR: { flat: 5.9, freeFrom: 60 }, +}; + +const DEFAULT_SHIPPING = { flat: 12, freeFrom: Infinity }; + +function shippingCost(netAmount, region) { + const rule = SHIPPING[String(region || '').toUpperCase()] ?? DEFAULT_SHIPPING; + return netAmount >= rule.freeFrom ? 0 : rule.flat; +} + +function freeShippingGap(netAmount, region) { + const rule = SHIPPING[String(region || '').toUpperCase()] ?? DEFAULT_SHIPPING; + if (!Number.isFinite(rule.freeFrom)) return null; + return round2(Math.max(0, rule.freeFrom - netAmount)); +} + +// --------------------------------------------------------------------------- +// Validation helpers +// --------------------------------------------------------------------------- + +function isKnownTier(tier) { + return Object.prototype.hasOwnProperty.call(TIER_DISCOUNTS, tier); +} + +function normalizeTier(customer) { + const tier = String(customer?.tier || 'standard').toLowerCase(); + return isKnownTier(tier) ? tier : 'standard'; +} + +function describeRules() { + return { + bulk: BULK_TIERS.map(t => `${t.minUnits}+ units: ${Math.round(t.rate * 100)}%`), + tiers: Object.entries(TIER_DISCOUNTS).map(([k, v]) => `${k}: ${Math.round(v * 100)}%`), + campaigns: CAMPAIGNS.map(c => c.id), + }; +} + +// --------------------------------------------------------------------------- +// Customer tiers +// --------------------------------------------------------------------------- + +/** + * Discount rate per customer tier, applied to the cart subtotal. + * Business rules: docs/pricing.md ("Customer tiers"). + */ +const SILVER_RATE = 0.03; +const GOLD_RATE = 0.05; +const VIP_RATE = 0.1; + +const TIER_DISCOUNTS = { + standard: 0, + silver: SILVER_RATE, + gold: GOLD_RATE, + vip: VIP_RATE, +}; + +function customerDiscountRate(customer) { + if (customer?.blocked) return 0; + return TIER_DISCOUNTS[normalizeTier(customer)]; +} + +module.exports = { + BULK_TIERS, + CAMPAIGNS, + LOYALTY_RATE, + SHIPPING, + TIER_DISCOUNTS, + bulkDiscountRate, + totalUnits, + activeCampaigns, + campaignRateFor, + campaignBanner, + loyaltyPoints, + pointsToEuro, + shippingCost, + freeShippingGap, + normalizeTier, + describeRules, + customerDiscountRate, +}; diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/pricing/tax.js b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/pricing/tax.js new file mode 100644 index 0000000..1dd01fd --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/pricing/tax.js @@ -0,0 +1,15 @@ +'use strict'; + +const { round2 } = require('../util/money'); + +const RATES = { PT: 0.23, ES: 0.21, DE: 0.19, FR: 0.2 }; + +function taxRate(region) { + return RATES[String(region || '').toUpperCase()] ?? 0; +} + +function calcTax(amount, region) { + return round2(amount * taxRate(region)); +} + +module.exports = { RATES, taxRate, calcTax }; diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/util/money.js b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/util/money.js new file mode 100644 index 0000000..dbd1586 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/src/util/money.js @@ -0,0 +1,11 @@ +'use strict'; + +function round2(n) { + return Math.round((n + Number.EPSILON) * 100) / 100; +} + +function formatEuro(n) { + return `${round2(n).toFixed(2)} EUR`; +} + +module.exports = { round2, formatEuro }; diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/test/cart.test.js b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/test/cart.test.js new file mode 100644 index 0000000..7c950b5 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/test/cart.test.js @@ -0,0 +1,51 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { createCart, addItem, removeItem, itemCount, subtotal, discount, total } = require('../src/index.js'); + +test('addItem merges the same sku', () => { + const cart = createCart(); + addItem(cart, { sku: 'PEN', price: 1.5, qty: 2 }); + addItem(cart, { sku: 'PEN', price: 1.5, qty: 3 }); + assert.equal(cart.items.length, 1); + assert.equal(itemCount(cart), 5); +}); + +test('addItem rejects invalid quantities', () => { + assert.throws(() => addItem(createCart(), { sku: 'PEN', price: 1, qty: 0 }), { message: 'INVALID_QTY' }); +}); + +test('subtotal of a single item', () => { + const cart = addItem(createCart(), { sku: 'MUG', price: 7.25, qty: 2 }); + assert.equal(subtotal(cart), 14.5); +}); + +test('subtotal of several items', () => { + const cart = createCart(); + addItem(cart, { sku: 'MUG', price: 7.25, qty: 2 }); + addItem(cart, { sku: 'PEN', price: 1.5, qty: 4 }); + addItem(cart, { sku: 'BAG', price: 12, qty: 1 }); + assert.equal(subtotal(cart), 32.5); +}); + +test('removeItem drops the sku', () => { + const cart = createCart(); + addItem(cart, { sku: 'MUG', price: 7.25 }); + addItem(cart, { sku: 'PEN', price: 1.5 }); + removeItem(cart, 'MUG'); + assert.deepEqual(cart.items.map(i => i.sku), ['PEN']); +}); + +test('bulk discount applies from 10 units', () => { + const cart = addItem(createCart(), { sku: 'PEN', price: 2, qty: 10 }); + assert.equal(discount(cart), 1); +}); + +test('total adds regional VAT to the net amount', () => { + const cart = createCart(); + addItem(cart, { sku: 'MUG', price: 10 }); + addItem(cart, { sku: 'BAG', price: 20 }); + assert.equal(total(cart, 'PT'), 36.9); + assert.equal(total(cart, 'XX'), 30); +}); diff --git a/extensions/codeforge/resources/calibrate-bench/fixtures/shop/test/invoice.test.js b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/test/invoice.test.js new file mode 100644 index 0000000..11d10b8 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/fixtures/shop/test/invoice.test.js @@ -0,0 +1,29 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { createCart, addItem, buildInvoice, formatInvoice, shippingCost } = require('../src/index.js'); + +test('invoice for a gold customer in Spain', () => { + const cart = createCart({ tier: 'gold' }); + addItem(cart, { sku: 'LAMP', price: 40 }); + addItem(cart, { sku: 'BULB', price: 5, qty: 2 }); + const inv = buildInvoice(cart, 'ES'); + assert.equal(inv.subtotal, 50); + assert.equal(inv.discount, 2.5); + assert.equal(inv.net, 47.5); + assert.equal(inv.tax, 9.98); + assert.equal(inv.total, 57.48); +}); + +test('formatted invoice shows the total', () => { + const cart = addItem(createCart(), { sku: 'LAMP', price: 40 }); + const text = formatInvoice(buildInvoice(cart, 'DE')); + assert.match(text, /Total: 47\.60 EUR/); +}); + +test('shipping is free above the regional threshold', () => { + assert.equal(shippingCost(39.99, 'PT'), 3.5); + assert.equal(shippingCost(40, 'PT'), 0); + assert.equal(shippingCost(500, 'XX'), 12); +}); diff --git a/extensions/codeforge/resources/calibrate-bench/headlessAgent.mjs b/extensions/codeforge/resources/calibrate-bench/headlessAgent.mjs new file mode 100644 index 0000000..33c85f6 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/headlessAgent.mjs @@ -0,0 +1,408 @@ +/** + * Headless agent runner for benchmarks. + * Loads the compiled extension, mocks vscode with disk FS, calls runAgentWithTools. + * + * Usage (usually via bench.mjs run): + * node benchmarks/headlessAgent.mjs --workspace --task [--label x] [--server id] [--model id] [--weak on|off] + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import Module from 'node:module'; +import { createRequire } from 'node:module'; +import { resolveBenchLayout } from './layout.mjs'; + +const LAYOUT = resolveBenchLayout(); +const HERE = LAYOUT.here; +const REPO = LAYOUT.repoRoot; +const EXT_OUT = process.env.CODEFORGE_EXT_OUT?.trim() || LAYOUT.extOut; +const VSCODE_MOCK = LAYOUT.vscodeMock; +const SETTINGS_FILE = path.join(os.homedir(), '.CodeForge', 'ai', 'settings.json'); + +function parseArgs(argv) { + const flags = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a.startsWith('--')) { + const key = a.slice(2); + const next = argv[i + 1]; + if (next && !next.startsWith('--')) { + flags[key] = next; + i++; + } else { + flags[key] = true; + } + } + } + return flags; +} + +function die(msg, code = 2) { + console.error(`NotRun: ${msg}`); + process.exit(code); +} + +function loadActiveLlm(flags) { + if (!fs.existsSync(SETTINGS_FILE)) { + die(`settings file missing: ${SETTINGS_FILE}`); + } + const state = JSON.parse(fs.readFileSync(SETTINGS_FILE, 'utf8')); + const serverId = flags.server || state.activeServerId; + const server = (state.servers || []).find(s => s.id === serverId && s.enabled !== false); + if (!server?.baseUrl?.trim()) { + die(`no usable active server (id=${serverId || 'none'})`); + } + const model = + flags.model || + (state.activeServerId === server.id ? state.activeModel : null) || + server.models?.[0]; + if (!model) die(`no model selected on server ${server.name}`); + return { + provider: server.kind || 'openai', + apiKey: server.apiKey || '', + baseUrl: server.baseUrl.replace(/\/$/, ''), + model, + numCtx: server.numCtx && server.numCtx > 0 ? server.numCtx : 32768, + forceJson: server.forceJson === true, + serverName: server.name, + serverId: server.id, + sampling: server.sampling, + }; +} + +function installVscodeMock(workspaceRoot, config) { + const absMock = path.resolve(VSCODE_MOCK); + const original = Module._resolveFilename; + Module._resolveFilename = function (request, parent, isMain, options) { + if (request === 'vscode') return absMock; + return original.call(this, request, parent, isMain, options); + }; + const require = createRequire(import.meta.url); + const vscode = require(absMock); + vscode.__setWorkspace(workspaceRoot); + vscode.__setConfig(config); + vscode.__setDiagnostics(new Map()); + return vscode; +} + +function fakeExtensionContext(home) { + const store = new Map(); + return { + subscriptions: [], + workspaceState: { + get: (k, d) => (store.has(`ws:${k}`) ? store.get(`ws:${k}`) : d), + update: async (k, v) => { + store.set(`ws:${k}`, v); + }, + keys: () => [...store.keys()].filter(k => k.startsWith('ws:')).map(k => k.slice(3)), + }, + globalState: { + get: (k, d) => (store.has(`g:${k}`) ? store.get(`g:${k}`) : d), + update: async (k, v) => { + store.set(`g:${k}`, v); + }, + keys: () => [...store.keys()].filter(k => k.startsWith('g:')).map(k => k.slice(2)), + setKeysForSync: () => undefined, + }, + extensionUri: { fsPath: EXT_OUT }, + extensionPath: EXT_OUT, + storageUri: undefined, + globalStorageUri: { fsPath: home }, + logUri: { fsPath: home }, + storagePath: undefined, + globalStoragePath: home, + logPath: home, + extensionMode: 3, + secrets: { + get: async () => undefined, + store: async () => undefined, + delete: async () => undefined, + onDidChange: () => ({ dispose() {} }), + }, + environmentVariableCollection: {}, + extension: {}, + asAbsolutePath: p => path.join(EXT_OUT, p), + languageModelAccessInformation: {}, + }; +} + +async function probeToolCallMode(llm) { + try { + const res = await fetch(`${llm.baseUrl}/chat/completions`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(llm.apiKey ? { Authorization: `Bearer ${llm.apiKey}` } : {}), + }, + body: JSON.stringify({ + model: llm.model, + temperature: 0, + max_tokens: 128, + tools: [ + { + type: 'function', + function: { + name: 'ping', + description: 'Return pong', + parameters: { + type: 'object', + properties: { msg: { type: 'string' } }, + required: ['msg'], + }, + }, + }, + ], + tool_choice: 'auto', + messages: [ + { role: 'system', content: 'Call the ping tool with msg=pong.' }, + { role: 'user', content: 'ping' }, + ], + }), + }); + if (!res.ok) return { toolCallMode: 'none', detail: `HTTP ${res.status}` }; + const data = await res.json(); + const msg = data.choices?.[0]?.message; + if (msg?.tool_calls?.length) return { toolCallMode: 'native' }; + const content = String(msg?.content || ''); + if (/|"tool_calls"| false, + numCtx, + forceJson: llm.forceJson, + maxSteps: Number(flags.maxSteps || 30), + hardCap: Number(flags.hardCap || 100), + weakProfile: weak, + isolated: true, + sampling, + serverId: llm.serverId, + onActivity: ev => { + activities.push(ev); + protocolAgg.observe(ev); + if (ev.kind === 'protocol' && ev.label === 'sample') { + sampleN += 1; + const file = path.join(samplesDir, `${String(sampleN).padStart(3, '0')}.json`); + try { + const parsed = JSON.parse(String(ev.detail || '{}')); + fs.writeFileSync( + file, + JSON.stringify( + { + at: new Date().toISOString(), + model: llm.model, + server: llm.serverName, + ...parsed, + }, + null, + 2 + ) + ); + } catch { + fs.writeFileSync(file, String(ev.detail || ''), 'utf8'); + } + } + }, + onStatus: text => { + // Fallback only — prefer onLoopExit when the agent reports it. + if (loopExit !== 'unknown') return; + if (/stuck/i.test(text)) loopExit = 'stuck'; + else if (/budget|maximum steps|step budget/i.test(text)) loopExit = 'budget'; + else if (/length|truncat/i.test(text)) loopExit = 'length'; + }, + onLoopExit: exit => { + loopExit = exit; + }, + }; + // Same path as chat: multi-card tasks run one isolated sub-run per card. + const orchestrated = await runOrchestrated(agentOpts, { forced: false }); + reply = orchestrated ?? (await runAgentWithTools(agentOpts)); + // Orchestrator report wins over a sub-run's length/budget status line. + const planDone = /Progress\s*\((\d+)\/\1\s+done\)/i.exec(String(reply || '')); + if (planDone && !/\bis blocked\b/i.test(String(reply || ''))) { + loopExit = 'done'; + } else if (loopExit === 'unknown') { + loopExit = 'done'; + } + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (/fetch failed|ECONNREFUSED|ENOTFOUND|401|403/i.test(msg)) { + die(`LLM request failed: ${msg}`); + } + reply = `error: ${msg}`; + loopExit = 'error'; + } + + const refusals = {}; + for (const a of activities) { + const m = /blocked \(([^)]+)\)/i.exec(`${a.label || ''} ${a.detail || ''}`); + if (m) { + const key = m[1].toLowerCase(); + refusals[key] = (refusals[key] || 0) + 1; + } + } + + const elapsedMs = Date.now() - started; + const protocolSnap = protocolAgg.snapshot(); + try { + flushRunTelemetry({ + serverId: llm.serverId, + model: llm.model, + source: 'bench', + loopExit, + elapsedMs, + aggregator: protocolAgg, + }); + } catch { + /* best-effort */ + } + + const result = { + runtime: 'headless', + model: llm.model, + server: llm.serverName, + serverId: llm.serverId, + weakProfile: weak, + numCtx, + toolCallMode: probe.toolCallMode, + toolCallModeDetail: probe.detail, + elapsedMs, + loopExit, + steps: activities.filter(a => a.kind === 'tool').length, + refusals, + protocol: protocolSnap.protocol, + sampleCount: sampleN, + toolLog: activities + .filter(a => a.kind === 'tool' || a.kind === 'checkpoint') + .map(a => `${a.kind === 'tool' ? a.toolStatus || 'tool' : 'checkpoint'} ${a.label || ''} ${String(a.detail || '').slice(0, 160)}`.trim()), + replyPreview: String(reply).slice(0, 500), + replyTail: String(reply).trim().slice(-500), + samplesDir, + }; + const outPath = flags.out ? path.resolve(String(flags.out)) : path.join(runHome, 'headless-result.json'); + fs.writeFileSync(outPath, JSON.stringify(result, null, 2)); + console.log(JSON.stringify(result, null, 2)); +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/extensions/codeforge/resources/calibrate-bench/layout.mjs b/extensions/codeforge/resources/calibrate-bench/layout.mjs new file mode 100644 index 0000000..b426fb4 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/layout.mjs @@ -0,0 +1,53 @@ +/** + * Resolve bench paths for repo checkout vs extension-packaged calibrate suite. + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); + +/** + * @returns {{ + * packaged: boolean; + * here: string; + * extRoot: string; + * extOut: string; + * vscodeMock: string; + * resultsDir: string; + * repoRoot: string; + * }} + */ +export function resolveBenchLayout() { + const packagedOut = path.resolve(HERE, '..', '..', 'out'); + const packagedMock = path.join(HERE, 'support', 'vscodeFsMock.js'); + const packagedMarker = path.join(HERE, '.packaged'); + if ( + fs.existsSync(packagedMarker) || + (fs.existsSync(packagedOut) && fs.existsSync(packagedMock)) + ) { + const extRoot = path.resolve(HERE, '..', '..'); + return { + packaged: true, + here: HERE, + extRoot, + extOut: packagedOut, + vscodeMock: packagedMock, + resultsDir: path.join(os.homedir(), '.CodeForge', 'ai', 'calibrate-results'), + repoRoot: extRoot, + }; + } + const repo = path.resolve(HERE, '..'); + return { + packaged: false, + here: HERE, + extRoot: path.join(repo, 'extensions', 'codeforge'), + extOut: path.join(repo, 'extensions', 'codeforge', 'out'), + vscodeMock: path.join(repo, 'tests', 'support', 'vscodeFsMock.js'), + resultsDir: path.join(HERE, 'results'), + repoRoot: repo, + }; +} + +export { HERE }; diff --git a/extensions/codeforge/resources/calibrate-bench/support/vscodeFsMock.js b/extensions/codeforge/resources/calibrate-bench/support/vscodeFsMock.js new file mode 100644 index 0000000..e6c81af --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/support/vscodeFsMock.js @@ -0,0 +1,252 @@ +/** + * Disk-backed vscode mock for headless agent-loop tests. + * Implements the subset used by VSCodeAIBridge + agentLoop. + */ +'use strict'; + +const fs = require('fs'); +const fsp = require('fs/promises'); +const path = require('path'); +const { minimatch } = require('minimatch'); + +let workspaceRoot = process.cwd(); +let configMap = {}; +/** @type {Map>} */ +let diagnosticsMap = new Map(); + +function makeUri(fsPath) { + const normalized = path.resolve(String(fsPath)); + return { + fsPath: normalized, + scheme: 'file', + path: normalized.replace(/\\/g, '/'), + toString: () => `file:///${normalized.replace(/\\/g, '/')}`, + }; +} + +function toFsPath(uriOrPath) { + if (!uriOrPath) return workspaceRoot; + if (typeof uriOrPath === 'string') return path.resolve(uriOrPath); + return path.resolve(uriOrPath.fsPath || String(uriOrPath)); +} + +async function ensureParent(filePath) { + await fsp.mkdir(path.dirname(filePath), { recursive: true }); +} + +function walkFiles(dir, out, max) { + if (out.length >= max) return; + let entries; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const ent of entries) { + if (out.length >= max) return; + const full = path.join(dir, ent.name); + if (ent.name === 'node_modules' || ent.name === '.git' || ent.name === 'bin' || ent.name === 'obj') { + continue; + } + if (ent.isDirectory()) walkFiles(full, out, max); + else if (ent.isFile()) out.push(full); + } +} + +class RelativePattern { + constructor(base, pattern) { + const baseUri = typeof base === 'string' ? makeUri(base) : base && base.uri ? base.uri : base; + this.baseUri = baseUri; + this.base = toFsPath(baseUri); + this.pattern = String(pattern); + } +} + +const FileType = { Unknown: 0, File: 1, Directory: 2, SymbolicLink: 64 }; +const DiagnosticSeverity = { Error: 0, Warning: 1, Information: 2, Hint: 3 }; +const ConfigurationTarget = { Global: 1, Workspace: 2, WorkspaceFolder: 3 }; + +const vscode = { + RelativePattern, + FileType, + DiagnosticSeverity, + ConfigurationTarget, + Uri: { + file: p => makeUri(p), + joinPath: (base, ...parts) => { + const basePath = toFsPath(base); + return makeUri(path.join(basePath, ...parts.map(String))); + }, + parse: s => makeUri(String(s).replace(/^file:\/\//, '')), + }, + workspace: { + get workspaceFolders() { + return [ + { + uri: makeUri(workspaceRoot), + name: path.basename(workspaceRoot), + index: 0, + }, + ]; + }, + getConfiguration: section => ({ + get: (key, defaultValue) => { + const full = section ? `${section}.${key}` : key; + if (Object.prototype.hasOwnProperty.call(configMap, full)) return configMap[full]; + if (section && Object.prototype.hasOwnProperty.call(configMap, `${section}.${key}`)) { + return configMap[`${section}.${key}`]; + } + const bag = configMap[section || '']; + if (bag && typeof bag === 'object' && Object.prototype.hasOwnProperty.call(bag, key)) { + return bag[key]; + } + return defaultValue; + }, + update: async () => undefined, + }), + asRelativePath: u => { + const p = toFsPath(u); + const rel = path.relative(workspaceRoot, p); + return rel.startsWith('..') ? p : rel.replace(/\\/g, '/'); + }, + findFiles: async (include, _exclude, maxResults = 400) => { + const pattern = typeof include === 'string' ? include : include?.pattern || '**/*'; + const base = include instanceof RelativePattern ? include.base : workspaceRoot; + const all = []; + walkFiles(base, all, maxResults * 4); + const matched = []; + for (const file of all) { + const rel = path.relative(base, file).replace(/\\/g, '/'); + if (minimatch(rel, pattern, { dot: true }) || minimatch(file.replace(/\\/g, '/'), pattern, { dot: true })) { + matched.push(makeUri(file)); + if (matched.length >= maxResults) break; + } + } + return matched; + }, + fs: { + readFile: async uri => { + const buf = await fsp.readFile(toFsPath(uri)); + return new Uint8Array(buf); + }, + writeFile: async (uri, content) => { + const p = toFsPath(uri); + await ensureParent(p); + const buf = Buffer.isBuffer(content) + ? content + : Buffer.from(content instanceof Uint8Array ? content : String(content)); + await fsp.writeFile(p, buf); + }, + stat: async uri => { + const st = await fsp.stat(toFsPath(uri)); + return { + type: st.isDirectory() ? FileType.Directory : FileType.File, + ctime: st.ctimeMs, + mtime: st.mtimeMs, + size: st.size, + }; + }, + readDirectory: async uri => { + const dir = toFsPath(uri); + const names = await fsp.readdir(dir, { withFileTypes: true }); + return names.map(ent => [ + ent.name, + ent.isDirectory() ? FileType.Directory : FileType.File, + ]); + }, + createDirectory: async uri => { + await fsp.mkdir(toFsPath(uri), { recursive: true }); + }, + delete: async (uri, _opts) => { + const p = toFsPath(uri); + const st = await fsp.stat(p); + if (st.isDirectory()) await fsp.rm(p, { recursive: true, force: true }); + else await fsp.unlink(p); + }, + rename: async (from, to, _opts) => { + await ensureParent(toFsPath(to)); + await fsp.rename(toFsPath(from), toFsPath(to)); + }, + }, + openTextDocument: async uri => ({ + uri: typeof uri === 'string' ? makeUri(uri) : uri, + getText: () => fs.readFileSync(toFsPath(uri), 'utf8'), + fileName: toFsPath(uri), + lineCount: fs.readFileSync(toFsPath(uri), 'utf8').split(/\r?\n/).length, + }), + onDidChangeWorkspaceFolders: () => ({ dispose() {} }), + onDidChangeTextDocument: () => ({ dispose() {} }), + onDidSaveTextDocument: () => ({ dispose() {} }), + }, + window: { + activeTextEditor: undefined, + visibleTextEditors: [], + createOutputChannel: () => ({ + appendLine: () => {}, + append: () => {}, + show: () => {}, + clear: () => {}, + dispose: () => {}, + }), + showInformationMessage: async () => undefined, + showWarningMessage: async () => undefined, + showErrorMessage: async () => undefined, + showSaveDialog: async () => undefined, + showOpenDialog: async () => undefined, + showInputBox: async () => undefined, + showTextDocument: async () => undefined, + createTerminal: () => ({ + show: () => {}, + sendText: () => {}, + dispose: () => {}, + name: 'agent', + }), + withProgress: async (_opts, task) => task({ report: () => {} }), + }, + languages: { + getDiagnostics: uri => { + if (uri) { + const key = toFsPath(uri).replace(/\\/g, '/').toLowerCase(); + const base = path.basename(key).toLowerCase(); + for (const [k, v] of diagnosticsMap) { + const kk = k.replace(/\\/g, '/').toLowerCase(); + if (kk === key || kk.endsWith('/' + base) || kk.endsWith(base)) { + return v; + } + } + return []; + } + return [...diagnosticsMap.entries()].map(([p, diags]) => [makeUri(p), diags]); + }, + registerInlineCompletionItemProvider: () => ({ dispose() {} }), + }, + commands: { + registerCommand: () => ({ dispose() {} }), + executeCommand: async () => undefined, + }, + EventEmitter: class { + event = () => ({ dispose() {} }); + fire() {} + dispose() {} + }, + ProgressLocation: { Notification: 15 }, + ViewColumn: { One: 1, Beside: -2 }, +}; + +module.exports = vscode; +module.exports.__setWorkspace = root => { + workspaceRoot = path.resolve(root); +}; +module.exports.__setConfig = obj => { + configMap = { ...(obj || {}) }; +}; +module.exports.__setDiagnostics = map => { + diagnosticsMap = new Map(); + if (!map) return; + for (const [k, v] of map instanceof Map ? map : Object.entries(map)) { + const raw = String(k); + const abs = path.isAbsolute(raw) ? path.resolve(raw) : path.resolve(workspaceRoot, raw); + diagnosticsMap.set(abs, v); + } +}; +module.exports.__getWorkspace = () => workspaceRoot; diff --git a/extensions/codeforge/resources/calibrate-bench/tasks/CS2-feature-coupon/hidden/CouponHiddenTests.cs b/extensions/codeforge/resources/calibrate-bench/tasks/CS2-feature-coupon/hidden/CouponHiddenTests.cs new file mode 100644 index 0000000..f1c48e4 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/tasks/CS2-feature-coupon/hidden/CouponHiddenTests.cs @@ -0,0 +1,57 @@ +using Acme.Billing; + +namespace Acme.Billing.Tests.BenchHidden; + +public class CouponHiddenTests +{ + private static readonly DateOnly Now = new(2026, 1, 15); + + private static Cart CartOf(decimal amount) + { + var cart = CartOps.Create(); + CartOps.AddItem(cart, "X", amount); + return cart; + } + + [Fact] + public void Percent_coupon() + { + Assert.Equal(5m, Coupons.ApplyCoupon(CartOf(50m), "WELCOME10", Now)); + Assert.Equal(3.33m, Coupons.ApplyCoupon(CartOf(33.33m), "WELCOME10", Now)); + } + + [Fact] + public void Codes_are_case_insensitive() + { + Assert.Equal(5m, Coupons.ApplyCoupon(CartOf(50m), "welcome10", Now)); + } + + [Fact] + public void Fixed_coupon_capped_at_subtotal() + { + Assert.Equal(5m, Coupons.ApplyCoupon(CartOf(50m), "FIVEOFF", Now)); + Assert.Equal(3m, Coupons.ApplyCoupon(CartOf(3m), "FIVEOFF", Now)); + } + + [Fact] + public void Expired_and_unknown_throw() + { + Assert.Throws(() => Coupons.ApplyCoupon(CartOf(50m), "OLD20", Now)); + Assert.Throws(() => Coupons.ApplyCoupon(CartOf(50m), "NOPE", Now)); + } + + [Fact] + public void Valid_on_expiry_day() + { + Assert.Equal(20m, Coupons.ApplyCoupon(CartOf(100m), "OLD20", new DateOnly(2020, 1, 1))); + } + + [Fact] + public void Does_not_mutate_cart() + { + var cart = CartOf(50m); + var before = cart.Items.Count; + _ = Coupons.ApplyCoupon(cart, "WELCOME10", Now); + Assert.Equal(before, cart.Items.Count); + } +} diff --git a/extensions/codeforge/resources/calibrate-bench/tasks/CS2-feature-coupon/solution.json b/extensions/codeforge/resources/calibrate-bench/tasks/CS2-feature-coupon/solution.json new file mode 100644 index 0000000..d609aca --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/tasks/CS2-feature-coupon/solution.json @@ -0,0 +1,6 @@ +[ + { + "file": "src/Acme.Billing/Coupons.cs", + "write": "using System.Text.Json;\n\nnamespace Acme.Billing;\n\npublic sealed record Coupon(string Code, string Type, decimal Value, string Expires);\n\npublic static class Coupons\n{\n\tpublic static IReadOnlyList Load(string file)\n\t{\n\t\tvar json = File.ReadAllText(file);\n\t\treturn JsonSerializer.Deserialize>(json, new JsonSerializerOptions\n\t\t{\n\t\t\tPropertyNameCaseInsensitive = true,\n\t\t}) ?? [];\n\t}\n\n\t/// \n\t/// Resolve config/coupons.json from the workspace root (walks up from cwd).\n\t/// \n\tpublic static string FindDefaultFile(string? startDir = null)\n\t{\n\t\tvar dir = new DirectoryInfo(startDir ?? Directory.GetCurrentDirectory());\n\t\twhile (dir is not null)\n\t\t{\n\t\t\tvar candidate = Path.Combine(dir.FullName, \"config\", \"coupons.json\");\n\t\t\tif (File.Exists(candidate)) return candidate;\n\t\t\tdir = dir.Parent;\n\t\t}\n\t\tthrow new FileNotFoundException(\"config/coupons.json not found\");\n\t}\n\n\tpublic static decimal ApplyCoupon(Cart cart, string code, DateOnly? today = null)\n\t{\n\t\tvar day = today ?? DateOnly.FromDateTime(DateTime.UtcNow);\n\t\tvar coupons = Load(FindDefaultFile());\n\t\tvar coupon = coupons.FirstOrDefault(c => c.Code.Equals(code, StringComparison.OrdinalIgnoreCase));\n\t\tif (coupon is null) throw new InvalidCouponException();\n\t\tvar expires = DateOnly.Parse(coupon.Expires);\n\t\tif (day > expires) throw new ExpiredCouponException();\n\t\tvar sub = Totals.Subtotal(cart);\n\t\tvar amount = coupon.Type.Equals(\"percent\", StringComparison.OrdinalIgnoreCase)\n\t\t\t? sub * coupon.Value / 100m\n\t\t\t: Math.Min(coupon.Value, sub);\n\t\treturn Money.Round2(amount);\n\t}\n}\n\npublic sealed class InvalidCouponException : Exception;\n\npublic sealed class ExpiredCouponException : Exception;\n" + } +] diff --git a/extensions/codeforge/resources/calibrate-bench/tasks/CS2-feature-coupon/task.json b/extensions/codeforge/resources/calibrate-bench/tasks/CS2-feature-coupon/task.json new file mode 100644 index 0000000..a97cb7a --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/tasks/CS2-feature-coupon/task.json @@ -0,0 +1,13 @@ +{ + "id": "CS2-feature-coupon", + "category": "feature", + "fixture": "billing", + "runtime": "dotnet", + "title": "Implement Coupons.ApplyCoupon from the docs", + "prompt": "Implement `Coupons.ApplyCoupon` as described in `docs/pricing.md` (Coupons section). The coupons are in `config/coupons.json`. Confirm with `dotnet test`.", + "protected": ["tests/", "config/"], + "checks": [ + { "type": "contains", "file": "src/Acme.Billing/Coupons.cs", "pattern": "ApplyCoupon\\s*\\(", "why": "ApplyCoupon exists" } + ], + "bonus": [] +} diff --git a/extensions/codeforge/resources/calibrate-bench/tasks/CS3-nav-vip-rate/hidden/VipHiddenTests.cs b/extensions/codeforge/resources/calibrate-bench/tasks/CS3-nav-vip-rate/hidden/VipHiddenTests.cs new file mode 100644 index 0000000..7f35135 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/tasks/CS3-nav-vip-rate/hidden/VipHiddenTests.cs @@ -0,0 +1,39 @@ +using Acme.Billing; + +namespace Acme.Billing.Tests.BenchHidden; + +public class VipHiddenTests +{ + [Fact] + public void Vip_customers_get_15_percent() + { + Assert.Equal(0.15m, PricingRules.CustomerDiscountRate(CartOps.Create("vip"))); + var cart = CartOps.Create("vip"); + CartOps.AddItem(cart, "X", 100m); + Assert.Equal(15m, Totals.Discount(cart)); + } + + [Fact] + public void Other_tiers_unchanged() + { + Assert.Equal(0m, PricingRules.CustomerDiscountRate(CartOps.Create("standard"))); + Assert.Equal(0.03m, PricingRules.CustomerDiscountRate(CartOps.Create("silver"))); + Assert.Equal(0.05m, PricingRules.CustomerDiscountRate(CartOps.Create("gold"))); + Assert.Equal(0m, PricingRules.CustomerDiscountRate(CartOps.Create("vip", blocked: true))); + } + + [Fact] + public void Loyalty_decoy_untouched() + { + Assert.Equal(0.1m, PricingRules.LoyaltyRate); + Assert.Equal(20, PricingRules.LoyaltyPoints(100m, "vip")); + } + + [Fact] + public void Best_rate_wins_vip_over_bulk() + { + var cart = CartOps.Create("vip"); + CartOps.AddItem(cart, "X", 1m, 60); + Assert.Equal(9m, Totals.Discount(cart)); + } +} diff --git a/extensions/codeforge/resources/calibrate-bench/tasks/CS3-nav-vip-rate/solution.json b/extensions/codeforge/resources/calibrate-bench/tasks/CS3-nav-vip-rate/solution.json new file mode 100644 index 0000000..6f02b39 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/tasks/CS3-nav-vip-rate/solution.json @@ -0,0 +1,4 @@ +[ + { "file": "src/Acme.Billing/PricingRules.cs", "replace": "private const decimal VipRate = 0.1m;", "with": "private const decimal VipRate = 0.15m;" }, + { "file": "docs/pricing.md", "replace": "| vip | 10% |", "with": "| vip | 15% |" } +] diff --git a/extensions/codeforge/resources/calibrate-bench/tasks/CS3-nav-vip-rate/task.json b/extensions/codeforge/resources/calibrate-bench/tasks/CS3-nav-vip-rate/task.json new file mode 100644 index 0000000..76ac6b7 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/tasks/CS3-nav-vip-rate/task.json @@ -0,0 +1,15 @@ +{ + "id": "CS3-nav-vip-rate", + "category": "navigation", + "fixture": "billing", + "runtime": "dotnet", + "title": "Change the VIP discount (find where the rule lives)", + "prompt": "VIP customers now get a 15% discount instead of 10%. Make the change where the rule is defined and update the documentation. Confirm with `dotnet test`.", + "protected": ["tests/"], + "checks": [ + { "type": "unchanged", "file": "src/Acme.Billing/Totals.cs", "why": "the rate lives in PricingRules, not Totals" } + ], + "bonus": [ + { "type": "contains", "file": "docs/pricing.md", "pattern": "\\|\\s*vip\\s*\\|\\s*15%", "why": "docs table updated" } + ] +} diff --git a/extensions/codeforge/resources/calibrate-bench/tasks/T2-feature-coupon/hidden/coupon.test.js b/extensions/codeforge/resources/calibrate-bench/tasks/T2-feature-coupon/hidden/coupon.test.js new file mode 100644 index 0000000..511b61a --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/tasks/T2-feature-coupon/hidden/coupon.test.js @@ -0,0 +1,47 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const shop = require('../src/index.js'); +const coupons = require('../src/pricing/coupons.js'); + +const NOW = new Date('2026-01-15T12:00:00Z'); + +function cartOf(amount) { + return shop.addItem(shop.createCart(), { sku: 'X', price: amount }); +} + +test('hidden: applyCoupon is exported from the coupons module and index', () => { + assert.equal(typeof coupons.applyCoupon, 'function'); + assert.equal(typeof shop.applyCoupon, 'function'); +}); + +test('hidden: percent coupon', () => { + assert.equal(shop.applyCoupon(cartOf(50), 'WELCOME10', NOW), 5); + assert.equal(shop.applyCoupon(cartOf(33.33), 'WELCOME10', NOW), 3.33); +}); + +test('hidden: codes are case-insensitive', () => { + assert.equal(shop.applyCoupon(cartOf(50), 'welcome10', NOW), 5); +}); + +test('hidden: fixed coupon is capped at the subtotal', () => { + assert.equal(shop.applyCoupon(cartOf(50), 'FIVEOFF', NOW), 5); + assert.equal(shop.applyCoupon(cartOf(3), 'FIVEOFF', NOW), 3); +}); + +test('hidden: expired and unknown coupons throw', () => { + assert.throws(() => shop.applyCoupon(cartOf(50), 'OLD20', NOW), { message: 'EXPIRED_COUPON' }); + assert.throws(() => shop.applyCoupon(cartOf(50), 'NOPE', NOW), { message: 'INVALID_COUPON' }); +}); + +test('hidden: coupon is valid on its expiry day', () => { + assert.equal(shop.applyCoupon(cartOf(100), 'OLD20', new Date('2020-01-01T18:00:00Z')), 20); +}); + +test('hidden: applyCoupon does not mutate the cart', () => { + const cart = cartOf(50); + const before = JSON.stringify(cart); + shop.applyCoupon(cart, 'WELCOME10', NOW); + assert.equal(JSON.stringify(cart), before); +}); diff --git a/extensions/codeforge/resources/calibrate-bench/tasks/T2-feature-coupon/solution.json b/extensions/codeforge/resources/calibrate-bench/tasks/T2-feature-coupon/solution.json new file mode 100644 index 0000000..e565f57 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/tasks/T2-feature-coupon/solution.json @@ -0,0 +1,8 @@ +[ + { + "file": "src/pricing/coupons.js", + "replace": "module.exports = { COUPONS_FILE, loadCoupons };", + "with": "function applyCoupon(cart, code, now = new Date()) {\n\tconst { round2 } = require('../util/money');\n\tconst { subtotal } = require('../cart/totals');\n\tconst coupon = loadCoupons().find(c => c.code.toUpperCase() === String(code || '').toUpperCase());\n\tif (!coupon) throw new Error('INVALID_COUPON');\n\tconst day = now.toISOString().slice(0, 10);\n\tif (day > coupon.expires) throw new Error('EXPIRED_COUPON');\n\tconst sub = subtotal(cart);\n\treturn round2(coupon.type === 'percent' ? (sub * coupon.value) / 100 : Math.min(coupon.value, sub));\n}\n\nmodule.exports = { COUPONS_FILE, loadCoupons, applyCoupon };" + }, + { "file": "src/index.js", "replace": "\tloadCoupons: coupons.loadCoupons,", "with": "\tloadCoupons: coupons.loadCoupons,\n\tapplyCoupon: coupons.applyCoupon," } +] diff --git a/extensions/codeforge/resources/calibrate-bench/tasks/T2-feature-coupon/task.json b/extensions/codeforge/resources/calibrate-bench/tasks/T2-feature-coupon/task.json new file mode 100644 index 0000000..f42613a --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/tasks/T2-feature-coupon/task.json @@ -0,0 +1,9 @@ +{ + "id": "T2-feature-coupon", + "category": "feature", + "title": "Implement applyCoupon from the docs", + "prompt": "Implement the `applyCoupon` function described in `docs/pricing.md` (Coupons section) in the coupons module and also export it from `src/index.js`. The coupons are in `config/coupons.json`. Confirm with `npm test`.", + "protected": ["test/", "config/"], + "checks": [], + "bonus": [] +} diff --git a/extensions/codeforge/resources/calibrate-bench/tasks/T4-refactor-rename/hidden/rename.test.js b/extensions/codeforge/resources/calibrate-bench/tasks/T4-refactor-rename/hidden/rename.test.js new file mode 100644 index 0000000..8abba6d --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/tasks/T4-refactor-rename/hidden/rename.test.js @@ -0,0 +1,23 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const shop = require('../src/index.js'); +const tax = require('../src/pricing/tax.js'); + +test('hidden: computeTax is exported from tax module and index', () => { + assert.equal(typeof tax.computeTax, 'function'); + assert.equal(typeof shop.computeTax, 'function'); +}); + +test('hidden: behaviour is unchanged', () => { + assert.equal(shop.computeTax(100, 'PT'), 23); + assert.equal(shop.computeTax(47.5, 'es'), 9.98); + assert.equal(shop.computeTax(10, 'XX'), 0); +}); + +test('hidden: totals and invoices still use the tax', () => { + const cart = shop.addItem(shop.createCart(), { sku: 'LAMP', price: 40 }); + assert.equal(shop.total(cart, 'DE'), 47.6); + assert.equal(shop.buildInvoice(cart, 'DE').tax, 7.6); +}); diff --git a/extensions/codeforge/resources/calibrate-bench/tasks/T4-refactor-rename/solution.json b/extensions/codeforge/resources/calibrate-bench/tasks/T4-refactor-rename/solution.json new file mode 100644 index 0000000..fc74605 --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/tasks/T4-refactor-rename/solution.json @@ -0,0 +1,6 @@ +[ + { "file": "src/pricing/tax.js", "replace": "calcTax", "with": "computeTax", "all": true }, + { "file": "src/cart/totals.js", "replace": "calcTax", "with": "computeTax", "all": true }, + { "file": "src/checkout/invoice.js", "replace": "calcTax", "with": "computeTax", "all": true }, + { "file": "src/index.js", "replace": "calcTax", "with": "computeTax", "all": true } +] diff --git a/extensions/codeforge/resources/calibrate-bench/tasks/T4-refactor-rename/task.json b/extensions/codeforge/resources/calibrate-bench/tasks/T4-refactor-rename/task.json new file mode 100644 index 0000000..fc44e9e --- /dev/null +++ b/extensions/codeforge/resources/calibrate-bench/tasks/T4-refactor-rename/task.json @@ -0,0 +1,11 @@ +{ + "id": "T4-refactor-rename", + "category": "refactor", + "title": "Rename calcTax to computeTax across src/", + "prompt": "Rename the function `calcTax` to `computeTax` throughout `src/` (definition, imports, usages and the export in `src/index.js`) without changing behavior. Do not leave `calcTax` as an alias or re-export under the old name — every reference must use `computeTax`. Confirm with `npm test`.", + "protected": ["test/"], + "checks": [ + { "type": "absent", "dir": "src", "pattern": "calcTax", "why": "no leftover references" } + ], + "bonus": [] +} diff --git a/extensions/codeforge/resources/webview/purify.min.js b/extensions/codeforge/resources/webview/purify.min.js new file mode 100644 index 0000000..00742c4 --- /dev/null +++ b/extensions/codeforge/resources/webview/purify.min.js @@ -0,0 +1,7 @@ +/*! @license DOMPurify 3.4.16 | (c) Cure53 and other contributors | Released under the Apache license 2.0 and Mozilla Public License 2.0 | github.com/cure53/DOMPurify/blob/3.4.16/LICENSE */ +(function(e,t){typeof exports==`object`&&typeof module<`u`?module.exports=t():typeof define==`function`&&define.amd?define([],t):(e=typeof globalThis<`u`?globalThis:e||self,e.DOMPurify=t())})(this,function(){"use strict";function e(e,t){this.v=e,this.k=t}function t(e,t){(t==null||t>e.length)&&(t=e.length);for(var n=0,r=Array(t);n`u`?null:T(BigInt.prototype.toString),pe=typeof Symbol>`u`?null:T(Symbol.prototype.toString),x=T(Object.prototype.hasOwnProperty),S=T(Object.prototype.toString),C=T(RegExp.prototype.test),w=E(TypeError);function T(e){return function(t){t instanceof RegExp&&(t.lastIndex=0);var n=[...arguments].slice(1);return g(e,t,n)}}function E(e){return function(){return te(e,[...arguments])}}function D(e,t){let n=arguments.length>2&&arguments[2]!==void 0?arguments[2]:oe;if(l&&l(e,null),!v(t))return e;let r=t.length;for(;r--;){let i=t[r];if(typeof i==`string`){let e=n(i);e!==i&&(u(t)||(t[r]=e),i=e)}e[i]=!0}return e}function me(e){for(let t=0;t/g),Ae=m(/\${[\w\W]*/g),je=m(/^data-[\-\w.\u00B7-\uFFFF]+$/),Me=m(/^aria-[\-\w]+$/),Ne=m(/^(?:(?:(?:f|ht)tps?|mailto|tel|callto|sms|cid|xmpp|matrix):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i),Pe=m(/^(?:\w+script|data):/i),Fe=m(/[\u0000-\u0020\u00A0\u1680\u180E\u2000-\u2029\u205F\u3000]/g),Ie=m(/^html$/i),Le=m(/^[a-z][.\w]*(-[.\w]+)+$/i),Re=m(/<[/\w!]/g),ze=m(/<[/\w]/g),Be=m(/<\/no(script|embed|frames)/i),Ve=m(/\/>/i),A={element:1,attribute:2,text:3,cdataSection:4,entityReference:5,entityNode:6,processingInstruction:7,comment:8,document:9,documentType:10,documentFragment:11,notation:12},j=[`style`,`script`,`xmp`,`iframe`,`noembed`,`noframes`,`plaintext`,`noscript`],He=p(D({},j)),Ue=function(){let e={};return _(j,t=>{e[t]=m(RegExp(`])`,`i`))}),p(e)}(),We=function(){return typeof window>`u`?null:window},Ge=function(e,t){if(typeof e!=`object`||typeof e.createPolicy!=`function`)return null;let n=null,r=`data-tt-policy-suffix`;t&&t.hasAttribute(r)&&(n=t.getAttribute(r));let i=`dompurify`+(n?`#`+n:``);try{return e.createPolicy(i,{createHTML(e){return e},createScriptURL(e){return e}})}catch(e){return console.warn(`TrustedTypes policy `+i+` could not be created.`),null}},Ke=function(){return{afterSanitizeAttributes:[],afterSanitizeElements:[],afterSanitizeShadowDOM:[],beforeSanitizeAttributes:[],beforeSanitizeElements:[],beforeSanitizeShadowDOM:[],uponSanitizeAttribute:[],uponSanitizeElement:[],uponSanitizeShadowNode:[]}},M=function(e,t,n,r){return x(e,t)&&v(e[t])?D(r.base?O(r.base):{},e[t],r.transform):n},qe=function(e,t,n){let r=x(e,t)?e[t]:void 0;return r&&typeof r==`object`?O(r):n()};function Je(){let e=arguments.length>0&&arguments[0]!==void 0?arguments[0]:We(),t=e=>Je(e);if(t.version=`3.4.16`,t.removed=[],!e||!e.document||e.document.nodeType!==A.document||!e.Element)return t.isSupported=!1,t;let n=e.document,r=n,i=r.currentScript;e.DocumentFragment;let a=e.HTMLTemplateElement,o=e.Node,s=e.Element,l=e.NodeFilter;e.NamedNodeMap===void 0&&(e.NamedNodeMap||e.MozNamedAttrMap),e.HTMLFormElement;let u=e.DOMParser,d=e.trustedTypes,f=s.prototype,ee=k(f,`cloneNode`),g=k(f,`remove`),te=k(f,`removeAttributeNode`),fe=k(f,`nextSibling`),y=k(f,`childNodes`),b=k(f,`parentNode`),pe=k(f,`shadowRoot`),S=k(f,`attributes`),T=o&&o.prototype?k(o.prototype,`nodeType`):null,E=o&&o.prototype?k(o.prototype,`nodeName`):null,me=o&&o.prototype?k(o.prototype,`ownerDocument`):null,j=function(e){return T?T(e):e.nodeType},Ye=function(e){return E?E(e):e.nodeName};if(typeof a==`function`){let e=n.createElement(`template`);e.content&&e.content.ownerDocument&&(n=e.content.ownerDocument)}let N,P=``,Xe,Ze=!1,Qe=0,$e=function(){if(Qe>0)throw w(`A configured TRUSTED_TYPES_POLICY callback (createHTML or createScriptURL) must not call DOMPurify.sanitize, as that causes infinite recursion. Do not pass a policy whose callbacks wrap DOMPurify as TRUSTED_TYPES_POLICY; see the "DOMPurify and Trusted Types" section of the README.`)},F=function(e){$e(),Qe++;try{return N.createHTML(e)}finally{Qe--}},et=function(e){$e(),Qe++;try{return N.createScriptURL(e)}finally{Qe--}},tt=function(){return Ze||(Xe=Ge(d,i),Ze=!0),Xe},nt=n,rt=nt.implementation,it=nt.createNodeIterator,at=nt.createDocumentFragment,ot=nt.getElementsByTagName,st=r.importNode,I=Ke();t.isSupported=typeof c==`function`&&typeof b==`function`&&rt&&rt.createHTMLDocument!==void 0;let ct=Oe,lt=ke,ut=Ae,dt=je,ft=Me,pt=Pe,mt=Fe,ht=Le,gt=Ne,L=null,_t=D({},[..._e,...ve,...ye,...xe,...Ce]),R=null,vt=D({},[...we,...Te,...Ee,...De]),z=Object.seal(h(null,{tagNameCheck:{writable:!0,configurable:!1,enumerable:!0,value:null},attributeNameCheck:{writable:!0,configurable:!1,enumerable:!0,value:null},allowCustomizedBuiltInElements:{writable:!0,configurable:!1,enumerable:!0,value:!1}})),yt=null,bt=null,B=Object.seal(h(null,{tagCheck:{writable:!0,configurable:!1,enumerable:!0,value:null},attributeCheck:{writable:!0,configurable:!1,enumerable:!0,value:null}})),xt=!0,St=!0,Ct=!1,wt=!0,V=!1,H=!0,U=!1,Tt=!1,Et=null,Dt=null,Ot=!1,W=!1,kt=!1,At=!1,jt=!0,Mt=!1,Nt=`user-content-`,Pt=!0,Ft=!1,G={},K=null,It=D({},/* @__PURE__ */ `annotation-xml.audio.colgroup.desc.foreignobject.head.iframe.math.mi.mn.mo.ms.mtext.noembed.noframes.noscript.plaintext.script.selectedcontent.style.svg.template.thead.title.video.xmp`.split(`.`)),Lt=null,Rt=D({},[`audio`,`video`,`img`,`source`,`image`,`track`]),zt=null,Bt=D({},[`alt`,`class`,`for`,`id`,`label`,`name`,`pattern`,`placeholder`,`role`,`summary`,`title`,`value`,`style`,`xmlns`]),Vt=`http://www.w3.org/1998/Math/MathML`,Ht=`http://www.w3.org/2000/svg`,q=`http://www.w3.org/1999/xhtml`,J=q,Ut=!1,Wt=null,Gt=D({},[Vt,Ht,q],se),Kt=p([`mi`,`mo`,`mn`,`ms`,`mtext`]),qt=D({},Kt),Jt=p([`annotation-xml`]),Yt=D({},Jt),Xt=D({},[`title`,`style`,`font`,`a`,`script`]),Zt=null,Qt=[`application/xhtml+xml`,`text/html`],Y=null,$t=null,en=n.createElement(`form`),tn=function(e){return e instanceof RegExp||e instanceof Function},nn=function(){let e=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{};if($t&&$t===e)return;(!e||typeof e!=`object`)&&(e={}),e=O(e),Zt=Qt.indexOf(e.PARSER_MEDIA_TYPE)===-1?`text/html`:e.PARSER_MEDIA_TYPE,Y=Zt===`application/xhtml+xml`?se:oe,L=M(e,`ALLOWED_TAGS`,_t,{transform:Y}),R=M(e,`ALLOWED_ATTR`,vt,{transform:Y}),Wt=M(e,`ALLOWED_NAMESPACES`,Gt,{transform:se}),zt=M(e,`ADD_URI_SAFE_ATTR`,Bt,{transform:Y,base:Bt}),Lt=M(e,`ADD_DATA_URI_TAGS`,Rt,{transform:Y,base:Rt}),K=M(e,`FORBID_CONTENTS`,It,{transform:Y}),yt=M(e,`FORBID_TAGS`,O({}),{transform:Y}),bt=M(e,`FORBID_ATTR`,O({}),{transform:Y}),G=x(e,`USE_PROFILES`)?e.USE_PROFILES&&typeof e.USE_PROFILES==`object`?O(e.USE_PROFILES):e.USE_PROFILES:!1,xt=e.ALLOW_ARIA_ATTR!==!1,St=e.ALLOW_DATA_ATTR!==!1,Ct=e.ALLOW_UNKNOWN_PROTOCOLS||!1,wt=e.ALLOW_SELF_CLOSE_IN_ATTR!==!1,V=e.SAFE_FOR_TEMPLATES||!1,H=e.SAFE_FOR_XML!==!1,U=e.WHOLE_DOCUMENT||!1,W=e.RETURN_DOM||!1,kt=e.RETURN_DOM_FRAGMENT||!1,At=e.RETURN_TRUSTED_TYPE||!1,Ot=e.FORCE_BODY||!1,jt=e.SANITIZE_DOM!==!1,Mt=e.SANITIZE_NAMED_PROPS||!1,Pt=e.KEEP_CONTENT!==!1,Ft=e.IN_PLACE||!1,gt=ge(e.ALLOWED_URI_REGEXP)?e.ALLOWED_URI_REGEXP:Ne,J=typeof e.NAMESPACE==`string`?e.NAMESPACE:q,qt=qe(e,`MATHML_TEXT_INTEGRATION_POINTS`,()=>D({},Kt)),Yt=qe(e,`HTML_INTEGRATION_POINTS`,()=>D({},Jt));let t=qe(e,`CUSTOM_ELEMENT_HANDLING`,()=>h(null));if(z=h(null),x(t,`tagNameCheck`)&&tn(t.tagNameCheck)&&(z.tagNameCheck=t.tagNameCheck),x(t,`attributeNameCheck`)&&tn(t.attributeNameCheck)&&(z.attributeNameCheck=t.attributeNameCheck),x(t,`allowCustomizedBuiltInElements`)&&typeof t.allowCustomizedBuiltInElements==`boolean`&&(z.allowCustomizedBuiltInElements=t.allowCustomizedBuiltInElements),m(z),V&&(St=!1),kt&&(W=!0),G&&(L=D({},Ce),R=h(null),G.html===!0&&(D(L,_e),D(R,we)),G.svg===!0&&(D(L,ve),D(R,Te),D(R,De)),G.svgFilters===!0&&(D(L,ye),D(R,Te),D(R,De)),G.mathMl===!0&&(D(L,xe),D(R,Ee),D(R,De))),B.tagCheck=null,B.attributeCheck=null,x(e,`ADD_TAGS`)&&(typeof e.ADD_TAGS==`function`?B.tagCheck=e.ADD_TAGS:v(e.ADD_TAGS)&&(L===_t&&(L=O(L)),D(L,e.ADD_TAGS,Y))),x(e,`ADD_ATTR`)&&(typeof e.ADD_ATTR==`function`?B.attributeCheck=e.ADD_ATTR:v(e.ADD_ATTR)&&(R===vt&&(R=O(R)),D(R,e.ADD_ATTR,Y))),x(e,`ADD_FORBID_CONTENTS`)&&v(e.ADD_FORBID_CONTENTS)&&(K===It&&(K=O(K)),D(K,e.ADD_FORBID_CONTENTS,Y)),Pt&&(L[`#text`]=!0),U&&D(L,[`html`,`head`,`body`]),L.table&&(D(L,[`tbody`]),delete yt.tbody),e.TRUSTED_TYPES_POLICY){if(typeof e.TRUSTED_TYPES_POLICY.createHTML!=`function`)throw w(`TRUSTED_TYPES_POLICY configuration option must provide a "createHTML" hook.`);if(typeof e.TRUSTED_TYPES_POLICY.createScriptURL!=`function`)throw w(`TRUSTED_TYPES_POLICY configuration option must provide a "createScriptURL" hook.`);let t=N;N=e.TRUSTED_TYPES_POLICY;try{P=F(``)}catch(e){throw N=t,e}}else e.TRUSTED_TYPES_POLICY===null?(N=void 0,P=``):(N===void 0&&(N=tt()),N&&typeof P==`string`&&(P=F(``)));p&&p(e),$t=e},rn=D({},[...ve,...ye,...be]),an=D({},[...xe,...Se]),on=function(e,t,n){return t.namespaceURI===q?e===`svg`:t.namespaceURI===Vt?e===`svg`&&(n===`annotation-xml`||qt[n]):!!rn[e]},sn=function(e,t,n){return t.namespaceURI===q?e===`math`:t.namespaceURI===Ht?e===`math`&&Yt[n]:!!an[e]},cn=function(e,t,n){return t.namespaceURI===Ht&&!Yt[n]||t.namespaceURI===Vt&&!qt[n]?!1:!an[e]&&(Xt[e]||!rn[e])},ln=function(e){let t=b(e);(!t||!t.tagName)&&(t={namespaceURI:J,tagName:`template`});let n=oe(e.tagName),r=oe(t.tagName);return Wt[e.namespaceURI]?e.namespaceURI===Ht?on(n,t,r):e.namespaceURI===Vt?sn(n,t,r):e.namespaceURI===q?cn(n,t,r):!!(Zt===`application/xhtml+xml`&&Wt[e.namespaceURI]):!1},X=function(e){ie(t.removed,{element:e});try{b(e).removeChild(e)}catch(t){if(g(e),!b(e))throw w(`a node selected for removal could not be detached from its tree and cannot be safely returned; refusing to sanitize in place`)}},un=function(e,t,n){try{te(e,t)}catch(t){try{e.removeAttribute(n)}catch(e){}}},dn=function(e){pn(e);let t=y(e);if(t){let e=[];_(t,t=>{ie(e,t)}),_(e,e=>{try{g(e)}catch(e){}})}let n=S(e);if(n)for(let t=n.length-1;t>=0;--t){let r=n[t],i=r&&r.name;typeof i==`string`&&un(e,r,i)}},Z=function(e,n,r){if(!r)try{r=n.getAttributeNode(e)}catch(e){r=null}ie(t.removed,{attribute:r||null,from:n});try{r?te(n,r):n.removeAttribute(e)}catch(t){try{n.removeAttribute(e)}catch(e){}}if(e===`is`){if(W||kt)try{X(n)}catch(e){}else try{n.setAttribute(e,``)}catch(e){}}},fn=function(e){let t=S(e);if(t)for(let n=t.length-1;n>=0;--n){let r=t[n],i=r&&r.name;typeof i!=`string`||R[Y(i)]||un(e,r,i)}},pn=function(e){let t=[e];for(;t.length>0;){let e=t.pop();j(e)===A.element&&fn(e);let n=y(e);if(n)for(let e=n.length-1;e>=0;--e)t.push(n[e])}},mn=function(e,t){return H?e===`patchsrc`||e===`for`&&t!==`label`&&t!==`output`:!1},hn=function(e){if(!H)return;let t=[e];for(;t.length>0;){let e=t.pop(),n=j(e);if(n===A.processingInstruction||n===A.comment&&C(ze,e.data)){try{g(e)}catch(e){}continue}if(n===A.element){let t=e,n=Y(Ye(e));try{t.hasAttribute&&t.hasAttribute(`patchsrc`)&&t.removeAttribute(`patchsrc`),t.hasAttribute&&t.hasAttribute(`for`)&&mn(`for`,n)&&t.removeAttribute(`for`)}catch(e){}}let r=y(e);if(r)for(let e=r.length-1;e>=0;--e)t.push(r[e])}},gn=function(e){let t=null,r=null;if(Ot)e=``+e;else{let t=ce(e,/^[\r\n\t ]+/);r=t&&t[0]}Zt===`application/xhtml+xml`&&J===q&&(e=``+e+``);let i=N?F(e):e;if(J===q)try{t=new u().parseFromString(i,Zt)}catch(e){}if(!t||!t.documentElement){t=rt.createDocument(J,`template`,null);try{t.documentElement.innerHTML=Ut?P:i}catch(e){}}let a=t.body||t.documentElement;return e&&r&&a.insertBefore(n.createTextNode(r),a.childNodes[0]||null),J===q?ot.call(t,U?`html`:`body`)[0]:U?t.documentElement:a},_n=function(e){let t=me?me(e):e.ownerDocument;return it.call(t||e,e,l.SHOW_ELEMENT|l.SHOW_COMMENT|l.SHOW_TEXT|l.SHOW_PROCESSING_INSTRUCTION|l.SHOW_CDATA_SECTION,null)},vn=function(e){return e=le(e,ct,` `),e=le(e,lt,` `),e=le(e,ut,` `),e},yn=function(e){var t;e.normalize();let n=me?me(e):e.ownerDocument,r=it.call(n||e,e,l.SHOW_TEXT|l.SHOW_COMMENT|l.SHOW_CDATA_SECTION|l.SHOW_PROCESSING_INSTRUCTION,null),i=r.nextNode();for(;i;)i.data=vn(i.data),i=r.nextNode();let a=(t=e.querySelectorAll)==null?void 0:t.call(e,`template`);a&&_(a,e=>{xn(e.content)&&yn(e.content)})},bn=function(e){let t=E?E(e):null;return typeof t!=`string`||Y(t)!==`form`?!1:typeof e.nodeName!=`string`||typeof e.textContent!=`string`||typeof e.removeChild!=`function`||e.attributes!==S(e)||typeof e.removeAttribute!=`function`||typeof e.removeAttributeNode!=`function`||typeof e.getAttributeNode!=`function`||typeof e.setAttribute!=`function`||typeof e.namespaceURI!=`string`||typeof e.insertBefore!=`function`||typeof e.hasChildNodes!=`function`||e.nodeType!==T(e)||e.childNodes!==y(e)},xn=function(e){if(!T||typeof e!=`object`||!e)return!1;try{return T(e)===A.documentFragment}catch(e){return!1}},Sn=function(e){if(!T||typeof e!=`object`||!e)return!1;try{return typeof T(e)==`number`}catch(e){return!1}};function Q(e,n,r){e.length!==0&&_(e,e=>{e.call(t,n,r,$t)})}let Cn=function(e,t){return!!(H&&e.hasChildNodes()&&!Sn(e.firstElementChild)&&C(Re,e.textContent)&&C(Re,e.innerHTML)||H&&e.namespaceURI===q&&He[t]&&(Sn(e.firstElementChild)||typeof e.textContent==`string`&&C(Ue[t],e.textContent))||e.nodeType===A.processingInstruction||H&&e.nodeType===A.comment&&C(ze,e.data))},wn=function(e,t){return e instanceof RegExp?C(e,t):e instanceof Function&&!!e(t,...[...arguments].slice(2))},Tn=function(e,t,n){if(!yt[t]&&An(t)&&wn(z.tagNameCheck,t))return!1;if(Pt&&!K[t]){let t=b(e),r=y(e);if(r&&t){let i=r.length;for(let a=i-1;a>=0;--a){let i=e===n?ee(r[a],!0):r[a];t.insertBefore(i,fe(e))}}}return X(e),!0},En=function(e,t,n,r){return e.length===0?t:t===n||t===r?O(t):t},$=function(e,t){return e===t||b(e)!==null?!1:(Ft&&pn(e),!0)},Dn=function(e,n){if(Q(I.beforeSanitizeElements,e,null),$(e,n))return!0;if(bn(e))return X(e),!0;let r=Y(Ye(e));if(L=En(I.uponSanitizeElement,L,_t,Et),Q(I.uponSanitizeElement,e,{tagName:r,allowedTags:L}),$(e,n))return!0;if(Cn(e,r))return X(e),!0;if(yt[r]||!(B.tagCheck instanceof Function&&B.tagCheck(r))&&!L[r]){let t=Tn(e,r,n);return t===!1&&(Q(I.afterSanitizeElements,e,null),$(e,n))?!0:t}if(j(e)===A.element&&!ln(e)||(r===`noscript`||r===`noembed`||r===`noframes`)&&C(Be,e.innerHTML))return X(e),!0;if(V&&e.nodeType===A.text){let n=vn(e.textContent);e.textContent!==n&&(ie(t.removed,{element:e.cloneNode()}),e.textContent=n)}return Q(I.afterSanitizeElements,e,null),$(e,n)},On=function(e,t,r){if(bt[t]||mn(t,e)||jt&&(t===`id`||t===`name`)&&(r in n||r in en))return!1;let i=R[t]||B.attributeCheck instanceof Function&&B.attributeCheck(t,e);return St&&C(dt,t)||xt&&C(ft,t)?!0:i?zt[t]||C(gt,le(r,mt,``))||(t===`src`||t===`xlink:href`||t===`href`)&&e!==`script`&&ue(r,`data:`)===0&&Lt[e]||Ct&&!C(pt,le(r,mt,``))?!0:!r:An(e)&&wn(z.tagNameCheck,e)&&wn(z.attributeNameCheck,t,e)||t===`is`&&z.allowCustomizedBuiltInElements&&wn(z.tagNameCheck,r)},kn=D({},[`annotation-xml`,`color-profile`,`font-face`,`font-face-format`,`font-face-name`,`font-face-src`,`font-face-uri`,`missing-glyph`]),An=function(e){return!kn[oe(e)]&&C(ht,e)},jn=function(e,t,n,r){if(N&&typeof d==`object`&&typeof d.getAttributeType==`function`&&!n)switch(d.getAttributeType(e,t)){case`TrustedHTML`:return F(r);case`TrustedScriptURL`:return et(r)}return r},Mn=function(e,t,n,r){try{return n?e.setAttributeNS(n,t,r):e.setAttribute(t,r),!bn(e)||(X(e),!1)}catch(n){return Z(t,e),!1}},Nn=function(e,n){if(Q(I.beforeSanitizeAttributes,e,null),$(e,n))return;let r=e.attributes;if(!r||bn(e))return;R=En(I.uponSanitizeAttribute,R,vt,Dt);let i={attrName:``,attrValue:``,keepAttr:!0,allowedAttributes:R,forceKeepAttr:void 0},a=r.length,o=Y(e.nodeName);for(;a--;){let n=r[a],s=n.name,c=n.namespaceURI,l=n.value,u=Y(s),d=l,f=s===`value`?d:de(d),p=!1;if(i.attrName=u,i.attrValue=f,i.keepAttr=!0,i.forceKeepAttr=void 0,Q(I.uponSanitizeAttribute,e,i),f=i.attrValue,Mt&&(u===`id`||u===`name`)&&ue(f,Nt)!==0&&(Z(s,e,n),f=Nt+f,p=!0),H&&C(/((--!?|])>)|<\/(style|script|title|xmp|textarea|noscript|iframe|noembed|noframes)/i,f)){Z(s,e,n);continue}if(u===`attributename`&&ce(f,`href`)){Z(s,e,n);continue}if(!i.forceKeepAttr){if(!i.keepAttr){Z(s,e,n);continue}if(!wt&&C(Ve,f)){Z(s,e,n);continue}if(V&&(f=vn(f)),!On(o,u,f)){Z(s,e,n);continue}f=jn(o,u,c,f),f!==d&&Mn(e,s,c,f)&&p&&re(t.removed)}}Q(I.afterSanitizeAttributes,e,null),$(e,n)},Pn=function(e){let t=null,n=_n(e);for(Q(I.beforeSanitizeShadowDOM,e,null);t=n.nextNode();)if(Q(I.uponSanitizeShadowNode,t,null),Dn(t,e),Nn(t,e),xn(t.content)&&Pn(t.content),j(t)===A.element){let e=pe(t);xn(e)&&(Fn(e),Pn(e))}Q(I.afterSanitizeShadowDOM,e,null)},Fn=function(e){let t=[{node:e,shadow:null}];for(;t.length>0;){let e=t.pop();if(e.shadow){Pn(e.shadow);continue}let n=e.node,r=j(n)===A.element,i=y(n);if(i)for(let e=i.length-1;e>=0;--e)t.push({node:i[e],shadow:null});if(r){let e=E?E(n):null;if(typeof e==`string`&&Y(e)===`template`){let e=n.content;xn(e)&&t.push({node:e,shadow:null})}}if(r){let e=pe(n);xn(e)&&t.push({node:null,shadow:e},{node:e,shadow:null})}}};return t.sanitize=function(e){let n=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{},i=null,a=null,o=null,s=null;if(Ut=!e,Ut&&(e=``),typeof e!=`string`&&!Sn(e)&&(e=he(e),typeof e!=`string`))throw w(`dirty is not a string, aborting`);if(!t.isSupported)return e;Tt?(L=Et,R=Dt):nn(n),(I.uponSanitizeElement.length>0||I.uponSanitizeAttribute.length>0)&&(L=O(L)),I.uponSanitizeAttribute.length>0&&(R=O(R)),t.removed=[];let c=Ft&&typeof e!=`string`&&Sn(e);if(c){hn(e);let t=Ye(e);if(typeof t==`string`){let n=Y(t);if(!L[n]||yt[n])throw dn(e),w(`root node is forbidden and cannot be sanitized in-place`)}if(bn(e))throw dn(e),w(`root node is clobbered and cannot be sanitized in-place`);try{Fn(e)}catch(t){throw dn(e),t}}else if(Sn(e))i=gn(``),a=i.ownerDocument.importNode(e,!0),a.nodeType===A.element&&a.nodeName===`BODY`||a.nodeName===`HTML`?i=a:i.appendChild(a),Fn(i);else{if(!W&&!V&&!U&&e.indexOf(`<`)===-1)return N&&At?F(e):e;if(i=gn(e),!i)return W?null:At?P:``}i&&Ot&&X(i.firstChild);let l=c?e:i;try{let e=_n(l);for(;o=e.nextNode();)Dn(o,l),Nn(o,l),xn(o.content)&&Pn(o.content)}catch(n){throw c&&(dn(e),_(t.removed,e=>{e.element&&pn(e.element)})),n}if(c){let n=!1;if(_(t.removed,t=>{t.element&&(t.element===e&&(n=!0),pn(t.element))}),n)throw w(`a node selected for removal could not be safely returned; refusing to sanitize in place`);return V&&yn(e),e}if(W){if(V&&yn(i),kt)for(s=at.call(i.ownerDocument);i.firstChild;)s.appendChild(i.firstChild);else s=i;return(R.shadowroot||R.shadowrootmode)&&(s=st.call(r,s,!0)),s}let u=U?i.outerHTML:i.innerHTML;return U&&L[`!doctype`]&&i.ownerDocument&&i.ownerDocument.doctype&&i.ownerDocument.doctype.name&&C(Ie,i.ownerDocument.doctype.name)&&(u=` +`+u),V&&(u=vn(u)),N&&At?F(u):u},t.setConfig=function(){let e=arguments.length>0&&arguments[0]!==void 0?arguments[0]:{};nn(e),Tt=!0,Et=L,Dt=R},t.clearConfig=function(){$t=null,Tt=!1,Et=null,Dt=null,N=Xe,P=``},t.isValidAttribute=function(e,t,n){$t||nn({});let r=Y(e),i=Y(t);return On(r,i,n)},t.addHook=function(e,t){typeof t==`function`&&x(I,e)&&ie(I[e],t)},t.removeHook=function(e,t){if(x(I,e)){if(t!==void 0){let n=ne(I[e],t);return n===-1?void 0:ae(I[e],n,1)[0]}return re(I[e])}},t.removeHooks=function(e){x(I,e)&&(I[e]=[])},t.removeAllHooks=function(){I=Ke()},t}return Je()}); +//# sourceMappingURL=purify.min.js.map \ No newline at end of file diff --git a/extensions/codeforge/src/agent/agentLoop.ts b/extensions/codeforge/src/agent/agentLoop.ts index 060757a..48e7aef 100644 --- a/extensions/codeforge/src/agent/agentLoop.ts +++ b/extensions/codeforge/src/agent/agentLoop.ts @@ -9,7 +9,13 @@ import { DiffPreview } from '../ui/diffPreview'; import { getApprovalPolicy } from '../policy/approvalPolicy'; import { getTrace } from '../trace/traceService'; import { getMcp } from '../mcp/mcpClient'; -import { approvalLevelForRisk, getToolRegistry, type ToolRisk } from './toolRegistry'; +import { + approvalLevelForRisk, + getToolRegistry, + toolNeedsApproval, + type ToolDefinition, + type ToolRisk, +} from './toolRegistry'; import { getAgentModeRegistry, toolVisibleInMode, type AgentModeDefinition } from './agentModes'; import { SessionStore } from '../sessions/sessionStore'; import { buildPriorAgentTranscript } from './priorContext'; @@ -20,6 +26,8 @@ import { updateTokenCalibration, } from './contextOverflow'; import { buildToolSystemContract, toolDescription } from './toolContracts'; +import { resolveProfileForRun, type ModelProfile } from './modelProfile'; +import { applyHarnessKnobs, learnedHarnessBlock, loadHarnessArtifact } from './harnessRevision'; import { normalizeToolCall, type JsonSchemaLike } from './toolCallNormalizer'; import { looksLikeXmlToolMarkup, parseXmlToolCalls } from './xmlToolCalls'; import { @@ -29,6 +37,7 @@ import { normalizeToolProtocolHistory, safeMidCompactTailStart, softCompactToolResults, + toWireToolArguments, } from './toolHistory'; import { applySoftThinkSwitch, @@ -97,6 +106,7 @@ import { LENGTH_CAP_SUMMARY_NOTE, oracleRejectNote, parseBlockedClaim, + runtimeVerifyNote, specSelfCheckNote, TRUNCATED_TOOL_JSON_FINAL_NOTE, TRUNCATED_TOOL_JSON_NOTE, @@ -104,7 +114,13 @@ import { MAX_STEPS_PROMPT, stuckWrapPrompt, } from './nudges'; -import { isEffectfulTool, shouldRefuseDiagFinish, shouldRequestSpecSelfCheck } from './finishGate'; +import { + countsAsRuntimeObserve, + isEffectfulTool, + shouldRefuseDiagFinish, + shouldRequestRuntimeVerify, + shouldRequestSpecSelfCheck, +} from './finishGate'; import { applyEditPreview, assertionEditRefusal, @@ -156,7 +172,11 @@ import { toolNamesFor, type AgentPhase, } from './toolsets'; -import { createLoopDetector, hashResult } from './loopDetector'; +import { + createLoopDetector, + hashResult, + isExploreToolName, +} from './loopDetector'; import { appendRoundResult, coachTaskPrompt, @@ -217,6 +237,20 @@ const MUTATING = new Set([ 'generate_image', 'canvas_write', ]); + +function agentToolNeedsUserApproval(name: string): boolean { + if (MUTATING.has(name)) return true; + const reg = getToolRegistry().get(name); + if (reg && toolNeedsApproval(reg)) return true; + if (name === 'mcp_call') { + const pseudo: Pick = { + risk: riskForAgentTool(name) as ToolRisk, + }; + return toolNeedsApproval(pseudo); + } + return false; +} + const READONLY_TOOLS = new Set([ 'read', 'list', @@ -593,10 +627,6 @@ export const AGENT_TOOLS = [ type: 'object', properties: { command: { type: 'string', description: 'Command to run' }, - sandbox: { - type: 'string', - description: 'safe-local | restricted | isolated | container | unrestricted', - }, background: { type: 'boolean', description: 'If true, start in background and return a job id', @@ -1138,7 +1168,8 @@ function toolsForRound( mode?: AgentModeDefinition, modelId?: string, /** Frozen at run start so the tools array stays byte-identical across rounds. */ - skillCatalog?: string + skillCatalog?: string, + shellCmdExeHints = true ): typeof AGENT_TOOLS { const registry = getToolRegistry(); const catalog = skillCatalog ?? formatSkillCatalogForTool(); @@ -1150,6 +1181,15 @@ function toolsForRound( if (!usePatch && t.function.name === 'apply_patch') return false; return true; }).map(t => { + if (t.function.name === 'shell') { + return { + ...t, + function: { + ...t.function, + description: toolDescription('shell', { shellCmdExeHints }), + }, + } as (typeof AGENT_TOOLS)[number]; + } if (t.function.name !== 'skill') return t; return { type: 'function' as const, @@ -1171,7 +1211,7 @@ function toolsForRound( required: ['name'], }, }, - }; + } as (typeof AGENT_TOOLS)[number]; }); const ext = registry .toOpenAiTools() @@ -1246,6 +1286,10 @@ export interface AgentLoopOptions { task: string; /** Optional per-server sampling overrides (applied over llmSamplingOptions). */ sampling?: AgentSamplingOptions; + /** Server id for ModelProfile load (defaults to "default"). */ + serverId?: string; + /** Pre-resolved profile; when omitted, loaded from disk for serverId+model. */ + modelProfile?: ModelProfile; /** Optional vision parts (data URLs) attached to the user turn. */ images?: Array<{ mime: string; dataUrl: string; label?: string }>; history?: AgentHistoryMessage[]; @@ -1253,6 +1297,8 @@ export interface AgentLoopOptions { abortSignal?: AbortSignal; onStatus?: (text: string) => void; onActivity?: (event: AgentActivityEvent) => void; + /** How the tool loop ended (budget / stuck / length / done). Used for Continue UI. */ + onLoopExit?: (exit: 'budget' | 'stuck' | 'length' | 'done' | 'cancelled' | 'error') => void; onContextBadge?: (badge: { indexed: number; inContext: number }) => void; /** Prompt tokens used vs context window limit (for the composer ring). */ onContextUsage?: (usage: { used: number; limit: number }) => void; @@ -1362,7 +1408,29 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise const mcpTools = mcp?.listTools() ?? []; const governance = getGovernanceStore(); const weakProfile = opts.weakProfile === true; + const loadedProfile: ModelProfile = + opts.modelProfile ?? + resolveProfileForRun(opts.serverId || 'default', opts.model, opts.sampling); + const harnessOverride = process.env.CODEFORGE_HARNESS_FILE?.trim(); + const learnedHarness = loadHarnessArtifact( + opts.serverId || 'default', + opts.model, + harnessOverride || undefined + ); + const modelProfile = applyHarnessKnobs(loadedProfile, learnedHarness); + opts.modelProfile = modelProfile; + if (modelProfile.toolTransport === 'text' && opts.proseToolsOnly !== false) { + opts.proseToolsOnly = true; + } + const largeFileLines = modelProfile.mutations.largeFileExistingLines; + const writeMaxLines = modelProfile.mutations.writeMaxContentLines; const gates = new GuardrailSession(governance.runtimeConfig({ weakProfile })); + if (weakProfile) { + gates.cfg.maxWritesWithoutBuild = Math.min( + gates.cfg.maxWritesWithoutBuild, + modelProfile.weak.maxWritesWithoutBuild + ); + } const agentMode = opts.agentModeId ? getAgentModeRegistry().get(opts.agentModeId) : getAgentModeRegistry().active(); @@ -1378,7 +1446,9 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise ? skillsMatchingHints(governance.getState().skills, agentMode.skillHints).map(s => s.id) : []; const forceSkills = [ - ...(weakProfile ? ['skill.harness-weak-models'] : []), + ...(weakProfile && modelProfile.prompt.forcedWeakSkill + ? ['skill.harness-weak-models'] + : []), ...(openPaths.some(p => /\.cshtml$|\.razor$|\/(Pages|Views)\//i.test(p)) ? ['skill.dotnet-razor'] : []), ...hintedSkillIds, ]; @@ -1420,7 +1490,11 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise mcpAvailable: mcpAvailableEarly, toolGroups: enabledToolGroups, }); - const toolContract = buildToolSystemContract(seedActiveTools); + const toolContract = buildToolSystemContract(seedActiveTools, { + renameSearchRule: modelProfile.prompt.renameSearchRule, + shellCmdExeHints: modelProfile.prompt.shellCmdExeHints, + }); + const frozenShellHints = modelProfile.prompt.shellCmdExeHints; const planModeHint = opts.coachMode ? [ @@ -1497,6 +1571,7 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise : '', agentsMdSection, forcedSkillSection, + learnedHarnessBlock(learnedHarness), mcpHint, ] .filter(Boolean) @@ -1575,6 +1650,9 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise const writtenTestPaths = new Set(); const readSpecPaths = new Set(); let specCheckNudges = 0; + /** Successful browser observe or non-oracle shell/dotnet after source edits. */ + let runtimeObserveCount = 0; + let runtimeVerifyNudges = 0; /** True once the spec edge-case note was sent; file creates in the workspace are refused. */ let specCheckActive = false; /** Paths created by write this run — assertion gate applies only to tests that pre-existed. */ @@ -1613,8 +1691,6 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise /** Caps step-=1 refunds so repair/overflow paths cannot spin forever. */ let refundedSteps = 0; const MAX_REFUNDED_STEPS = 12; - /** Explore-only steps without a write/shell since last mutate. */ - let exploreOnlySteps = 0; const loopDetector = createLoopDetector(); /** Retries when gateway rejects truncated tool-call JSON (common with large writes). */ let truncatedToolRetries = 0; @@ -1632,7 +1708,7 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise const MAX_ORACLE_REJECTIONS = 6; let gateFinishRejections = 0; const MAX_GATE_FINISH_REJECTIONS = 2; - const oracleCfg = opts.planMode || opts.askMode ? undefined : detectOracle(opts.workspaceRoot, nodeOracleFs); + let oracleCfg = opts.planMode || opts.askMode ? undefined : detectOracle(opts.workspaceRoot, nodeOracleFs); let lastProgressMarker = ''; const progressMarker = () => `${successfulWriteCount}|${uiChangeCount}|${gates.buildRed ? 'red' : 'ok'}|${gates.lastBuildErrors.length}`; @@ -1982,7 +2058,13 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise activeInternalPlan ? formatPlanForPrompt(activeInternalPlan).slice(0, 400) : undefined, - toolsForRound(activeToolNames, agentMode, opts.model, frozenSkillCatalog) + toolsForRound( + activeToolNames, + agentMode, + opts.model, + frozenSkillCatalog, + frozenShellHints + ) ); messages = structured ? (structured) @@ -2075,7 +2157,8 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise activeToolNames, agentMode, opts.model, - frozenSkillCatalog + frozenSkillCatalog, + frozenShellHints ); if (!protocolSizedOnce) { protocolSizedOnce = true; @@ -2402,6 +2485,55 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise toolCalls.length && looksLikeToolJson(content) ? '' : content ); + // finish_reason=length with tool_calls: only run if arguments parse strictly (no repair). + if (toolCalls.length && finishReason === 'length') { + const truncated = toolCalls.filter(tc => { + const raw = String(tc.function?.arguments ?? ''); + try { + JSON.parse(raw); + return false; + } catch { + return true; + } + }); + if (truncated.length) { + emitProtocol('lengthExits', '1'); + messages.push({ + role: 'assistant', + content: assistantContent || '', + tool_calls: toolCalls, + }); + for (const tc of toolCalls) { + messages.push({ + role: 'tool', + tool_call_id: tc.id, + name: tc.function.name, + content: + 'Not run: arguments were cut at the output limit. Retry with a smaller tool call.', + }); + actions.push(`${tc.function.name} blocked (truncated-args)`); + } + if (lengthTruncationNudges < TOOL_LENGTH_RETRIES && step + 1 <= stepBudget) { + lengthTruncationNudges += 1; + messages.push({ role: 'user', content: LENGTH_CAP_NOTE }); + opts.onActivity?.({ + kind: 'checkpoint', + label: 'Truncated tool args — retry smaller tool', + detail: truncated.map(t => t.function.name).join(', '), + }); + if (tryRefundContinue()) continue; + continue; + } + loopExit = 'length'; + opts.onActivity?.({ + kind: 'checkpoint', + label: 'Stopped after truncated tool arguments', + detail: `${TOOL_LENGTH_RETRIES} continuations exhausted`, + }); + break; + } + } + if (!toolCalls.length) { const trivial = isTrivialAssistantContent(content); const hitLengthCap = @@ -2483,6 +2615,7 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise effectfulCount, anyToolCount, diagFinishNudges, + maxDiagFinishNudges: modelProfile.finish.maxDiagFinishNudges, }) && step + 1 < stepBudget ) { @@ -2501,6 +2634,9 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise const changedFiles = successfulWriteCount > 0; let doneNote = ''; let loggedExperimentRound = false; + if (!oracleCfg && changedFiles && !opts.planMode && !opts.askMode && !opts.coachMode) { + oracleCfg = detectOracle(opts.workspaceRoot, nodeOracleFs); + } if ( !blocked && changedFiles && @@ -2568,6 +2704,7 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise if (!result.ok) { await logRound(false); doneNote = finalReplyNote('still-red', result.commands[result.commands.length - 1]); + turnOutcome = 'blocked'; } else { await logRound(true); } @@ -2588,6 +2725,8 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise readSpecCount: readSpecPaths.size, writtenTestCount: writtenTestPaths.size, specCheckNudges, + specSelfCheckEnabled: modelProfile.finish.specSelfCheckEnabled, + specSelfCheckMaxNudges: modelProfile.finish.specSelfCheckMaxNudges, }) && step + 1 < stepBudget ) { @@ -2610,6 +2749,37 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise continue; } + if ( + !blocked && + shouldRequestRuntimeVerify({ + planMode: opts.planMode, + askMode: opts.askMode, + coachMode: opts.coachMode, + statusQuestion, + blocked: !!blocked, + verifiedGreen: !oracleCfg || gates.hasVerifiedGreen(), + writtenSourceCount: writtenSourcePaths.size, + runtimeObserveCount, + runtimeVerifyNudges, + }) && + step + 1 < stepBudget + ) { + runtimeVerifyNudges += 1; + const note = runtimeVerifyNote({ + sourcePaths: [...writtenSourcePaths], + }); + trace?.info('harness', String(note.length)); + messages.push({ role: 'assistant', content: summary }); + messages.push({ role: 'user', content: note }); + opts.onActivity?.({ + kind: 'checkpoint', + label: 'Runtime verify', + detail: `${writtenSourcePaths.size} source, ${runtimeObserveCount} runtime observes`, + }); + opts.onStatus?.('Runtime verify: launch and observe the app…'); + continue; + } + // Do not block the user-visible reply on update_status. An extra model // round here felt like lag; the harness records a stop point in recordHandoff(). @@ -2697,20 +2867,23 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise } } + const stillRed = /Oracle: build\/test is still red/i.test(doneNote); + if (blocked) turnOutcome = 'blocked'; + else if (stillRed) turnOutcome = 'blocked'; + else if (turnOutcome !== 'blocked') turnOutcome = 'done'; + turnBlockedReason = blocked ?? (stillRed ? 'oracle still red' : ''); opts.onTaskUpdate?.({ id: taskId, name: truncateHistory(opts.task, 60), - status: blocked ? 'failed' : 'completed', + status: turnOutcome === 'done' ? 'completed' : 'failed', result: summary.slice(0, 200), elapsed: Date.now() - started, }); opts.onActivity?.({ kind: 'checkpoint', - label: blocked ? 'Blocked' : 'Turn summary', - detail: blocked ?? summarizeActionsForLlm(actions).slice(0, 400), + label: turnOutcome === 'done' ? 'Turn summary' : 'Blocked', + detail: blocked ?? (stillRed ? 'oracle still red' : summarizeActionsForLlm(actions).slice(0, 400)), }); - turnOutcome = blocked ? 'blocked' : 'done'; - turnBlockedReason = blocked ?? ''; emitTurnMetrics(); let reply = summary + doneNote; if ( @@ -2749,6 +2922,7 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise } } } + opts.onLoopExit?.(turnOutcome === 'done' ? 'done' : 'stuck'); return reply; } @@ -2761,18 +2935,7 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise preferShortCompletion = false; emptyFinishNudges = 0; - const exploreTools = new Set([ - 'read', - 'list', - 'search', - 'retrieve', - 'symbols', - 'definition', - 'references', - ]); - const stepFingerprints: string[] = []; - let stepHadMutate = false; /** Read-only tools from one model step. I/O overlaps; results commit in call order. */ interface ParallelIoJob { @@ -2863,8 +3026,25 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise if (pathKey) { touchedPaths.add(pathKey); } - if (isEffectfulTool(name)) { - effectfulCount += 1; + { + const cmdForRuntime = + name === 'dotnet' + ? dotnetCommandLine(args) + : name === 'shell' + ? String(args.command ?? '') + : undefined; + if (isEffectfulTool(name, cmdForRuntime)) { + effectfulCount += 1; + } + if ( + countsAsRuntimeObserve( + name, + cmdForRuntime, + oracleCfg?.commands + ) + ) { + runtimeObserveCount += 1; + } } if ( name === 'browser_click' || @@ -2950,9 +3130,12 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise } } if ((name === 'write' || name === 'edit' || name === 'apply_patch') && success) { - gates.onSuccessfulWrite(args.path); - successfulWriteCount += 1; - writesSinceOracle += 1; + const alreadyApplied = /Already applied/i.test(output); + if (!alreadyApplied) { + gates.onSuccessfulWrite(args.path); + successfulWriteCount += 1; + writesSinceOracle += 1; + } if (typeof args.path === 'string' && args.path.trim()) { const wp = args.path.replace(/\\/g, '/'); writtenPaths.add(wp); @@ -2970,8 +3153,23 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise const pk = canonicalizePathKey(String(args.path ?? '')); if (pk) readThisTurn.add(pk); } - if (name === 'delete' || name === 'rename') { - if (success) successfulWriteCount += 1; + if ((name === 'delete' || name === 'rename') && success) { + gates.onSuccessfulWrite( + name === 'rename' ? args.newPath ?? args.path : args.path + ); + successfulWriteCount += 1; + writesSinceOracle += 1; + failedFingerprints.clear(); + } + if ( + name === 'dotnet' && + success && + /^(add_package|sln_add|add_reference|remove_package)$/i.test(String(args.action ?? '')) + ) { + gates.onSuccessfulWrite(args.project ?? args.solution ?? args.path); + successfulWriteCount += 1; + writesSinceOracle += 1; + failedFingerprints.clear(); } if (name === 'update_status' && success) { statusUpdated = true; @@ -3233,7 +3431,7 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise } const name = normalized.name; args = normalized.args; - if (!exploreTools.has(name)) { + if (!isExploreToolName(name)) { await flushLane(); } const refuse = async (content: string, label: string) => { @@ -3241,6 +3439,15 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise trace?.info('harness', String(content.length)); messages.push({ role: 'tool', tool_call_id: call.id, name, content }); actions.push(`${name} blocked (${label})`); + opts.onActivity?.({ + kind: 'tool', + label: `${name} blocked (${label})`, + detail: content.slice(0, 400), + tool: name, + toolCallId: call.id, + toolStatus: 'failed', + success: false, + }); }; if (name === 'write' && typeof args.content === 'string' && args.content.length > 14000) { @@ -3268,10 +3475,20 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise } // Weak profile: a full rewrite of a large existing file loses code; edit is the tool for that. - if (weakProfile && name === 'write' && gates.cfg.blockMassRewrite && typeof args.path === 'string') { + if ( + weakProfile && + name === 'write' && + gates.cfg.blockMassRewrite && + modelProfile.mutations.enforceMassRewriteGate && + typeof args.path === 'string' + ) { const existing = await opts.bridge.readRaw(String(args.path)); const existingLines = existing?.split(/\r?\n/).length ?? 0; - if (existing !== undefined && existingLines > LARGE_FILE_LINES && isPartialRewrite(existing, String(args.content ?? ''))) { + if ( + existing !== undefined && + existingLines > largeFileLines && + isPartialRewrite(existing, String(args.content ?? '')) + ) { await refuse( `Not written: ${args.path} already has ${existingLines} lines and most of them would stay the same. Change the parts that differ with edit (old_string → new_string).`, 'large rewrite' @@ -3279,9 +3496,9 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise continue; } const contentLines = String(args.content ?? '').split(/\r?\n/).length; - if (contentLines > 180) { + if (writeMaxLines > 0 && contentLines > writeMaxLines) { await refuse( - `Not written: content is ${contentLines} lines (limit 180 on weak profile). Write a minimal skeleton (<60 lines), then grow the file with edit one method/block at a time. For .NET projects prefer the dotnet tool (new / sln_add / add_reference).`, + `Not written: content is ${contentLines} lines (limit ${writeMaxLines} on weak profile). Write a minimal skeleton (<${largeFileLines} lines), then grow the file with edit one method/block at a time. For .NET projects prefer the dotnet tool (new / sln_add / add_reference).`, 'large write' ); continue; @@ -3349,7 +3566,6 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise const fingerprint = toolFingerprint(name, args); stepFingerprints.push(fingerprint); - if (!exploreTools.has(name)) stepHadMutate = true; const priorCount = recentFingerprints.filter(f => f === fingerprint).length; recentFingerprints.push(fingerprint); if (recentFingerprints.length > 40) { @@ -3506,7 +3722,7 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise detail: JSON.stringify(summarizeArgs(name, args)).slice(0, 200), }); - if (MUTATING.has(name) && !gateApproved) { + if (agentToolNeedsUserApproval(name) && !gateApproved) { const policy = getApprovalPolicy(); if (policy.isAutoApproved(name)) { /* Allow all / auto-approve — no HITL checkpoint */ @@ -3751,7 +3967,7 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise readWindowKey, readAliases, }; - if (exploreTools.has(name)) { + if (isExploreToolName(name)) { lane.push(job); continue; } @@ -3774,18 +3990,12 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise } } - if (stepHadMutate) { - exploreOnlySteps = 0; - } else if (toolCalls.some(tc => exploreTools.has(tc.function.name))) { - exploreOnlySteps += 1; - } // Successful tool progress resets length-truncation streak (distant truncations must not kill the run). if (toolCalls.length > 0) { lengthTruncationNudges = 0; preferShortCompletion = false; } loopDetector.endStep(); - loopDetector.setExploreOnlySteps(exploreOnlySteps); loopDetector.setUnchangedStreak(unchangedStreak); // Hard stop when tools loop without progress — no coaching nudges. @@ -3857,7 +4067,7 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise 'Stopped: tools were repeating without new progress.', stuckCause ? `Reason: ${stuckCause}` : '', summarizeActionsForLlm(actions), - 'Tip: restate a concrete next step (file + change), not just “continue”.', + 'Tip: next turn must call edit, write, shell, or browser — do not only re-read files already in context.', ] .filter(Boolean) .join('\n\n') @@ -3891,7 +4101,13 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise url, sanitizeMessages(messages), SHORT_COMPLETION_TOKENS, - toolsForRound(activeToolNames, agentMode, opts.model, frozenSkillCatalog), + toolsForRound( + activeToolNames, + agentMode, + opts.model, + frozenSkillCatalog, + frozenShellHints + ), 'none' ); if (wrap.ok) { @@ -3914,21 +4130,33 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise ].join(' '); } if (answer) { - turnOutcome = loopExit === 'budget' ? 'max-steps' : 'done'; + turnOutcome = + loopExit === 'budget' + ? 'max-steps' + : loopExit === 'stuck' + ? 'stuck' + : loopExit === 'length' + ? 'max-steps' + : 'done'; opts.onTaskUpdate?.({ id: taskId, name: truncateHistory(opts.task, 60), - status: loopExit === 'budget' ? 'failed' : 'completed', + status: + loopExit === 'stuck' || loopExit === 'length' || loopExit === 'budget' + ? 'failed' + : 'completed', result: truncateHistory(answer, 120), elapsed: Date.now() - started, }); + opts.onLoopExit?.(loopExit ?? 'budget'); return answer; } } } catch { /* fall through */ } - turnOutcome = 'max-steps'; + turnOutcome = + loopExit === 'stuck' ? 'stuck' : loopExit === 'length' ? 'max-steps' : 'max-steps'; opts.onTaskUpdate?.({ id: taskId, name: truncateHistory(opts.task, 60), @@ -3948,6 +4176,7 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise const saved = await loadProjectStatus(opts.workspaceRoot); statusFallback = (saved ? formatStatusForPrompt(saved) : emptyStatusPrompt()) + '\n\n'; } + opts.onLoopExit?.(loopExit ?? 'budget'); return statusFallback + wrapFallback; } catch (err) { turnOutcome = 'error'; @@ -3959,6 +4188,7 @@ export async function runAgentWithTools(opts: AgentLoopOptions): Promise result: message.slice(0, 200), elapsed: Date.now() - started, }); + opts.onLoopExit?.('error'); throw err; } finally { // Fire-and-forget so the chat bubble can settle before git/status I/O. @@ -4677,7 +4907,8 @@ async function postAgentRound( proseToolsOnly: !!opts.proseToolsOnly, tools, }; - const baseSampling = llmSamplingOptions(thinkInput); + const profileSamp = opts.modelProfile?.sampling; + const baseSampling = llmSamplingOptions(thinkInput, profileSamp); const sampling = { ...baseSampling, ...(opts.sampling ?? {}), @@ -4693,7 +4924,7 @@ async function postAgentRound( if (typeof sampling.dry_multiplier === 'number') { samplingExtras.dry_multiplier = sampling.dry_multiplier; } - const cappedTokens = toolAwareMaxTokens(maxTokens, thinkInput); + const cappedTokens = toolAwareMaxTokens(maxTokens, thinkInput, opts.modelProfile?.completion); const disableThink = shouldDisableThinking(thinkInput); const enableThink = shouldEnableThinking(thinkInput); let outbound = stripReasoningFromMessages(messages); @@ -5436,7 +5667,14 @@ function sanitizeMessages(messages: ChatMessage[]): ChatMessage[] { const next: ChatMessage = { role: m.role, content }; if (m.tool_call_id) next.tool_call_id = m.tool_call_id; if (m.name) next.name = m.name; - if (m.tool_calls?.length) next.tool_calls = m.tool_calls; + if (m.tool_calls?.length) { + next.tool_calls = m.tool_calls.map(tc => { + const args = toWireToolArguments(tc.function.arguments ?? '', parseToolArguments); + return args === tc.function.arguments + ? tc + : { ...tc, function: { ...tc.function, arguments: args } }; + }); + } return next; }) ); @@ -5557,9 +5795,12 @@ async function runProseToolFallback( ); } - return actions.length - ? `Done.\n${summarizeActionsForLlm(actions)}` - : text; + const failed = actions.filter(a => /\b(failed|denied|blocked)\b/i.test(a)); + if (!actions.length) return text; + if (failed.length) { + return `Finished with failures.\n${summarizeActionsForLlm(actions)}`; + } + return `Done.\n${summarizeActionsForLlm(actions)}`; } function uniqHandoffPaths(paths: string[]): string[] { @@ -6286,14 +6527,15 @@ function cleanToolDetailForUi(output: string): string { /** Shell exit≠0 must not be cached as success / treated as OK progress. */ function isToolSuccess(name: string, output: string): boolean { - if ( - /^(Error:|FAILED:|Not run:)/i.test(output.trim()) || - /old_string was not found/i.test(output) || - /hunk \d+:|apply_patch:/i.test(output) || - /^FILE_NOT_FOUND:/i.test(output.trim()) - ) { + if (/^(Error:|FAILED:|Not run:)/i.test(output.trim()) || /^FILE_NOT_FOUND:/i.test(output.trim())) { return false; } + // Text rules for patch tools only — shell/search may mention "hunk" in docs. + if (name === 'edit' || name === 'write' || name === 'apply_patch') { + if (/old_string was not found/i.test(output) || /hunk \d+:|apply_patch:/i.test(output)) { + return false; + } + } if (name.startsWith('git_') && /Git is unavailable|Git tool failed/i.test(output)) { return false; } diff --git a/extensions/codeforge/src/agent/editTool.ts b/extensions/codeforge/src/agent/editTool.ts index f237b17..b0bfa50 100644 --- a/extensions/codeforge/src/agent/editTool.ts +++ b/extensions/codeforge/src/agent/editTool.ts @@ -363,12 +363,26 @@ export function applyEdit( if (count === 0) { // Model often invents a broken old_string when the desired new_string is already on disk. + // Require a non-empty new_string that is not a substring of old_string and matches on a line boundary. const view = fileText.replaceAll(CRLF, '\n'); const neu = newString.replaceAll(CRLF, '\n'); - const already = - view.includes(neu) || - normalizeForMatch(view).includes(normalizeForMatch(neu)); - if (already) { + const neuTrim = neu.trim(); + const oldView = oldString.replaceAll(CRLF, '\n'); + const meaningful = + neuTrim.length > 0 && + !oldView.includes(neu) && + !normalizeForMatch(oldView).includes(normalizeForMatch(neu)); + const lineBoundary = + meaningful && + (view === neu || + view.startsWith(`${neu}\n`) || + view.endsWith(`\n${neu}`) || + view.includes(`\n${neu}\n`) || + normalizeForMatch(view).includes(`\n${normalizeForMatch(neu)}\n`) || + normalizeForMatch(view) === normalizeForMatch(neu) || + normalizeForMatch(view).startsWith(`${normalizeForMatch(neu)}\n`) || + normalizeForMatch(view).endsWith(`\n${normalizeForMatch(neu)}`)); + if (lineBoundary) { const idx = view.indexOf(neu); const firstLine = idx >= 0 ? view.slice(0, idx).split('\n').length : 1; diff --git a/extensions/codeforge/src/agent/experimentLoop.ts b/extensions/codeforge/src/agent/experimentLoop.ts index e9e4ce1..c6338e3 100644 --- a/extensions/codeforge/src/agent/experimentLoop.ts +++ b/extensions/codeforge/src/agent/experimentLoop.ts @@ -41,12 +41,23 @@ export interface RoundResult { } export function normalizeRelPath(p: string): string { - return String(p ?? '') + const raw = String(p ?? '') .replace(/\\/g, '/') .replace(/^\.\//, '') .replace(/\/+/g, '/') .replace(/^\//, '') .trim(); + if (!raw) return ''; + const parts: string[] = []; + for (const seg of raw.split('/')) { + if (!seg || seg === '.') continue; + if (seg === '..') { + if (parts.length) parts.pop(); + continue; + } + parts.push(seg); + } + return parts.join('/'); } export function parseChecksLock(text: string): ChecksLock | undefined { @@ -148,14 +159,39 @@ export function isLockedScorePath(relPath: string, locked: string[]): boolean { return locked.some(p => normalizeRelPath(p).toLowerCase() === n); } +/** Paths mentioned in apply_patch body (*** Update/Add/Delete File: …). */ +function pathsFromPatchText(patch: string): string[] { + const out: string[] = []; + const re = /^\*\*\*\s+(?:Update|Add|Delete|Move)\s+File:\s*(.+)$/gim; + let m: RegExpExecArray | null; + while ((m = re.exec(String(patch || ''))) !== null) { + const p = normalizeRelPath(m[1].trim()); + if (p) out.push(p); + } + return out; +} + /** Paths a mutating tool would touch. */ export function pathsTouchedByTool(tool: string, args: Record): string[] { switch (tool) { case 'write': case 'edit': - case 'apply_patch': case 'delete': return [normalizeRelPath(String(args.path ?? ''))].filter(Boolean); + case 'apply_patch': { + const fromArg = normalizeRelPath(String(args.path ?? '')); + const fromBody = pathsFromPatchText(String(args.patch ?? args.patchText ?? args.patch_text ?? '')); + const seen = new Set(); + const out: string[] = []; + for (const p of [fromArg, ...fromBody]) { + if (!p) continue; + const key = p.toLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + out.push(p); + } + return out; + } case 'rename': { const oldP = normalizeRelPath(String(args.oldPath ?? args.path ?? '')); const newP = normalizeRelPath(String(args.newPath ?? args.to ?? '')); diff --git a/extensions/codeforge/src/agent/finishGate.selftest.ts b/extensions/codeforge/src/agent/finishGate.selftest.ts index b7aad99..85f18a5 100644 --- a/extensions/codeforge/src/agent/finishGate.selftest.ts +++ b/extensions/codeforge/src/agent/finishGate.selftest.ts @@ -1,7 +1,14 @@ /** - * Diagnose-only finish gate + spec self-check. + * Diagnose-only finish gate + spec self-check + runtime verify. */ -import { isEffectfulTool, shouldRefuseDiagFinish, shouldRequestSpecSelfCheck } from './finishGate'; +import { + countsAsRuntimeObserve, + isEffectfulTool, + isOracleBuildOrTestCommand, + shouldRefuseDiagFinish, + shouldRequestRuntimeVerify, + shouldRequestSpecSelfCheck, +} from './finishGate'; function assert(cond: boolean, msg: string): void { if (!cond) throw new Error(msg); @@ -11,6 +18,16 @@ assert(isEffectfulTool('edit'), 'edit effectful'); assert(isEffectfulTool('browser_click'), 'click effectful'); assert(!isEffectfulTool('read'), 'read not effectful'); assert(!isEffectfulTool('browser_snapshot'), 'snapshot not effectful'); +assert(isEffectfulTool('shell', 'npm test'), 'mutating shell effectful'); +assert(!isEffectfulTool('shell', 'dir src'), 'readonly shell not effectful'); +assert(!isEffectfulTool('shell', 'git status'), 'git status not effectful'); +assert(!isEffectfulTool('shell', 'type README.md'), 'type not effectful'); +assert(!countsAsRuntimeObserve('shell', 'dir'), 'readonly shell not runtime observe'); +assert(isOracleBuildOrTestCommand('dotnet test'), 'dotnet test is oracle'); +assert( + !isOracleBuildOrTestCommand('echo npm test was mentioned', ['npm test']), + 'oracle match requires full command' +); assert( shouldRefuseDiagFinish({ @@ -134,4 +151,64 @@ assert( 'tests touched → skip even with spec' ); +assert(isOracleBuildOrTestCommand('npm test --silent'), 'npm test is oracle'); +assert(isOracleBuildOrTestCommand('dotnet build Foo.sln -nologo -v q'), 'dotnet build is oracle'); +assert(!isOracleBuildOrTestCommand('npm start'), 'npm start is runtime'); +assert( + countsAsRuntimeObserve('browser_snapshot'), + 'snapshot counts (even if not effectful)' +); +assert(countsAsRuntimeObserve('shell', 'npm start'), 'start shell counts'); +assert(!countsAsRuntimeObserve('shell', 'npm test --silent'), 'test shell does not count'); + +assert( + shouldRequestRuntimeVerify({ + verifiedGreen: true, + writtenSourceCount: 1, + runtimeObserveCount: 0, + runtimeVerifyNudges: 0, + }), + 'runtime verify fires' +); + +assert( + !shouldRequestRuntimeVerify({ + verifiedGreen: true, + writtenSourceCount: 1, + runtimeObserveCount: 1, + runtimeVerifyNudges: 0, + }), + 'already observed → skip' +); + +assert( + !shouldRequestRuntimeVerify({ + verifiedGreen: true, + writtenSourceCount: 0, + runtimeObserveCount: 0, + runtimeVerifyNudges: 0, + }), + 'no writes → skip' +); + +assert( + !shouldRequestRuntimeVerify({ + verifiedGreen: false, + writtenSourceCount: 1, + runtimeObserveCount: 0, + runtimeVerifyNudges: 0, + }), + 'needs green' +); + +assert( + !shouldRequestRuntimeVerify({ + verifiedGreen: true, + writtenSourceCount: 1, + runtimeObserveCount: 0, + runtimeVerifyNudges: 1, + }), + 'one-shot only' +); + console.log('finishGate.selftest: ok'); diff --git a/extensions/codeforge/src/agent/finishGate.ts b/extensions/codeforge/src/agent/finishGate.ts index ee4231e..7cb7d17 100644 --- a/extensions/codeforge/src/agent/finishGate.ts +++ b/extensions/codeforge/src/agent/finishGate.ts @@ -3,7 +3,7 @@ * (weak models otherwise stop at a bug list). Not intent sniffing on user text. */ -/** Tools that change the workspace, run commands, or act on the browser UI. */ +/** Tools that change the workspace, run mutating commands, or act on the browser UI. */ export const EFFECTFUL_TOOLS = new Set([ 'edit', 'write', @@ -17,7 +17,26 @@ export const EFFECTFUL_TOOLS = new Set([ 'browser_type', ]); -export function isEffectfulTool(name: string): boolean { +/** Shell/dotnet commands that only inspect — not effectful for the finish gate. */ +const READ_ONLY_SHELL_RE = + /^\s*(?:(?:cd|chdir|pushd|popd)\s+[^\|&;]+(?:\s*&&\s*)?)*(?:dir|ls|ll|type|cat|Get-Content|Get-ChildItem|gc|gl|pwd|echo|where|which|git\s+(?:status|log|diff|show|branch|remote|rev-parse)|rg|grep|findstr|find)\b/i; + +export function isReadOnlyShellCommand(command: string): boolean { + const c = String(command || '').trim(); + if (!c) return false; + if (/[|;&]/.test(c.replace(/&&/g, ''))) { + // Allow only && chains of read-only parts + const parts = c.split(/\s*&&\s*/); + return parts.every(p => READ_ONLY_SHELL_RE.test(p.trim()) && !/[|;&]/.test(p)); + } + return READ_ONLY_SHELL_RE.test(c); +} + +export function isEffectfulTool(name: string, command?: string): boolean { + if (name === 'shell' || name === 'dotnet') { + if (command !== undefined && isReadOnlyShellCommand(command)) return false; + return true; + } return EFFECTFUL_TOOLS.has(name); } @@ -68,6 +87,10 @@ export type SpecSelfCheckInput = { writtenTestCount: number; /** How many times this one-shot already fired. */ specCheckNudges: number; + /** From ModelProfile.finish (default true). */ + specSelfCheckEnabled?: boolean; + /** From ModelProfile.finish (default 1). */ + specSelfCheckMaxNudges?: number; }; /** @@ -77,10 +100,107 @@ export type SpecSelfCheckInput = { export function shouldRequestSpecSelfCheck(input: SpecSelfCheckInput): boolean { if (input.planMode || input.askMode || input.coachMode || input.statusQuestion) return false; if (input.blocked) return false; + if (input.specSelfCheckEnabled === false) return false; if (!input.verifiedGreen) return false; - if (input.specCheckNudges >= 1) return false; + const max = input.specSelfCheckMaxNudges ?? 1; + if (input.specCheckNudges >= max) return false; if (input.writtenSourceCount < 1) return false; // Require a read spec AND no test-file edits — avoids speculative "fixes" on scaffold/card runs // that never touched docs, while still catching T2-style "src only after reading docs". return input.readSpecCount > 0 && input.writtenTestCount === 0; } + +/** Browser observe / act tools that prove the UI was exercised. */ +export const RUNTIME_OBSERVE_TOOLS = new Set([ + 'browser_navigate', + 'browser_snapshot', + 'browser_screenshot', + 'browser_click', + 'browser_type', +]); + +export function isRuntimeObserveTool(name: string): boolean { + return RUNTIME_OBSERVE_TOOLS.has(name); +} + +/** + * True when a shell/dotnet command is an oracle-style build/test (does not count + * as runtime launch/observe). Matches known patterns + optional oracle command list. + */ +export function isOracleBuildOrTestCommand( + command: string, + oracleCommands?: string[] +): boolean { + const c = String(command || '').trim(); + if (!c) return false; + const norm = c.replace(/\s+/g, ' '); + if (oracleCommands?.length) { + for (const oc of oracleCommands) { + const o = String(oc || '').replace(/\s+/g, ' ').trim(); + if (!o) continue; + // Require the full oracle command as the whole command (or with trailing args only). + if (norm === o || norm.startsWith(`${o} `)) { + return true; + } + } + } + // npm / yarn / pnpm / dotnet as the command itself (not mentioned in an echo/string). + if (/(?:^|[;&|]\s*)(?:npm|yarn|pnpm)\s+(?:test|run\s+(?:test|build))\b/i.test(norm)) return true; + if (/(?:^|[;&|]\s*)dotnet\s+(?:test|build)\b/i.test(norm)) return true; + return false; +} + +/** + * Whether a successful tool call counts toward runtime verification. + * Browser observe tools always count; shell/dotnet only when not build/test. + */ +export function countsAsRuntimeObserve( + name: string, + command?: string, + oracleCommands?: string[] +): boolean { + if (isRuntimeObserveTool(name)) return true; + if (name === 'shell' || name === 'dotnet') { + const cmd = String(command || ''); + if (isReadOnlyShellCommand(cmd)) return false; + return !isOracleBuildOrTestCommand(cmd, oracleCommands); + } + return false; +} + +export type RuntimeVerifyInput = { + planMode?: boolean; + askMode?: boolean; + coachMode?: boolean; + statusQuestion?: boolean; + blocked?: boolean; + /** + * True when build/test is green, or there is no oracle for this workspace + * (same idea as requiring green before other finish extras). + */ + verifiedGreen: boolean; + writtenSourceCount: number; + /** Successful runtime observe tools (browser or non-oracle shell) this run. */ + runtimeObserveCount: number; + runtimeVerifyNudges: number; + /** Default 1 (one-shot). */ + runtimeVerifyMaxNudges?: number; + /** Default true. */ + runtimeVerifyEnabled?: boolean; +}; + +/** + * One-shot: after source edits, refuse prose finish until the model launches/observes + * the running app (browser or non-oracle shell). Path/tool counts only. + */ +export function shouldRequestRuntimeVerify(input: RuntimeVerifyInput): boolean { + if (input.planMode || input.askMode || input.coachMode || input.statusQuestion) return false; + if (input.blocked) return false; + if (input.runtimeVerifyEnabled === false) return false; + if (!input.verifiedGreen) return false; + if (input.writtenSourceCount < 1) return false; + if (input.runtimeObserveCount > 0) return false; + const max = input.runtimeVerifyMaxNudges ?? 1; + if (input.runtimeVerifyNudges >= max) return false; + return true; +} diff --git a/extensions/codeforge/src/agent/harnessRevision.selftest.ts b/extensions/codeforge/src/agent/harnessRevision.selftest.ts new file mode 100644 index 0000000..3139ea8 --- /dev/null +++ b/extensions/codeforge/src/agent/harnessRevision.selftest.ts @@ -0,0 +1,249 @@ +/** + * Harness revision must not keep an unmeasured or losing edit. + */ +import { + applyHarnessEdit, + applyHarnessKnobs, + bootstrapMeanLowerBound, + decideRevision, + emptyHarnessArtifact, + executeHarnessRevision, + formatExecutionReport, + holdoutRegressed, + learnedHarnessBlock, + parseProposerEdit, +} from './harnessRevision'; +import type { ProtocolRollup } from './protocolTelemetry'; +import { emptyProtocolCounters } from './protocolTelemetry'; +import { defaultModelProfile } from './modelProfile'; + +function assert(cond: boolean, msg: string): void { + if (!cond) throw new Error(msg); +} + +export async function run(): Promise { +const empty = emptyHarnessArtifact(); +assert(learnedHarnessBlock(empty) === '', 'empty artifact adds no prompt'); + +const banned = parseProposerEdit( + JSON.stringify({ + diagnosis: 'match phrases', + sections: { writePolicy: 'If the user says fix it, write the file.' }, + }) +); +assert(!banned.ok, 'phrase matcher rejected'); + +const skipTests = parseProposerEdit( + JSON.stringify({ + diagnosis: 'avoid checks', + sections: { workflow: 'You may skip tests when stuck.' }, + }) +); +assert(!skipTests.ok, 'skip tests rejected'); + +const massOff = parseProposerEdit( + JSON.stringify({ + diagnosis: 'turn off gate', + knobs: { enforceMassRewriteGate: false }, + }) +); +assert(!massOff.ok, 'mass rewrite false rejected'); + +const writeZero = parseProposerEdit( + JSON.stringify({ + diagnosis: 'no cap', + knobs: { writeMaxContentLines: 0 }, + }) +); +assert(!writeZero.ok, 'write cap below 40 rejected'); + +const unknown = parseProposerEdit( + JSON.stringify({ diagnosis: 'x', sections: { oracle: 'disable' } }) +); +assert(!unknown.ok && unknown.ok === false && /unknown section/.test(unknown.error), 'unknown section rejected'); + +const prose = parseProposerEdit('I would add a retry loop.'); +assert(!prose.ok, 'prose rejected'); + +const nestedKnobs = parseProposerEdit( + JSON.stringify({ + diagnosis: 'writes too large', + sections: { + writePolicy: 'Write a skeleton under 40 lines, then grow it with edit.', + knobs: { writeMaxContentLines: 90 }, + }, + }) +); +assert(nestedKnobs.ok && nestedKnobs.ok === true && nestedKnobs.edit.knobs.writeMaxContentLines === 90, 'knobs nested in sections are lifted'); + +const wrapped = parseProposerEdit( + 'maybe {"diagnosis":"ignore this","sections":{}}\n{"diagnosis":"writes too large","sections":{"writePolicy":"Write a skeleton under 40 lines, then grow it with edit."}}' +); +assert(wrapped.ok && wrapped.ok === true && wrapped.edit.diagnosis === 'writes too large', 'think block is ignored'); + +const okEdit = parseProposerEdit( + JSON.stringify({ + diagnosis: 'writes exceed the token budget', + sections: { writePolicy: 'Write a skeleton under 40 lines, then grow it with edit.' }, + knobs: { writeMaxContentLines: 80 }, + }) +); +assert(okEdit.ok, 'valid edit parsed'); +if (!okEdit.ok) throw new Error('narrow'); +const next = applyHarnessEdit(empty, okEdit.edit); +assert(next.revision === 1, 'revision bumps'); +assert(next.sections.workflow === '', 'untouched section stays empty'); +assert(next.sections.writePolicy.includes('skeleton'), 'write policy applied'); +assert(learnedHarnessBlock(next).includes('Learned harness'), 'kept notes render'); + +const profile = applyHarnessKnobs(defaultModelProfile('s', 'm'), next); +assert(profile.mutations.writeMaxContentLines === 80, 'knob applied'); +assert(profile.mutations.enforceMassRewriteGate === true, 'other knobs stay'); + +const thin = decideRevision({ + byTask: { T2: { parent: [40], candidate: [90] } }, +}); +assert(thin.decision === 'revert', 'one rep does not keep'); + +const noise = decideRevision({ + byTask: { + T2: { parent: [70, 72, 71], candidate: [71, 72, 70] }, + T4: { parent: [68, 69, 70], candidate: [69, 68, 69] }, + }, +}); +assert(noise.decision === 'revert', 'small noisy paired gains do not keep'); + +const equalNoise = decideRevision({ + byTask: { + T2: { parent: [70, 71, 72], candidate: [70, 71, 72] }, + T4: { parent: [65, 66, 67], candidate: [65, 66, 67] }, + }, +}); +assert(equalNoise.decision === 'revert', 'identical parent and candidate revert'); + +const gain = decideRevision({ + byTask: { + T2: { parent: [60, 62, 61], candidate: [80, 78, 82] }, + T4: { parent: [55, 56, 54], candidate: [75, 76, 74] }, + }, +}); +assert(gain.decision === 'keep', 'clear gain keeps'); + +const deltas = [2, 2, 2]; +assert(bootstrapMeanLowerBound(deltas) > 0, 'positive deltas lower bound > 0'); +assert(bootstrapMeanLowerBound([0, 0, 0]) <= 0, 'zero deltas lower bound not positive'); + +const holdFail = holdoutRegressed( + { CS2: { parent: [80, 82], candidate: [] } }, + { CS2: { parent: [], candidate: [70, 71] } } +); +assert(!!holdFail && holdFail.includes('CS2'), 'holdout regression detected'); + +const rollupRecentOnly: ProtocolRollup = { + serverId: 's', + model: 'm', + updatedAt: '2026-01-01', + runCount: 100, + sums: { ...emptyProtocolCounters(), nativeToolCalls: 999, badArgs: 999 }, + recent: [ + { + at: '2026-01-01', + serverId: 's', + model: 'm', + source: 'bench', + loopExit: 'done', + elapsedMs: 1, + protocol: { ...emptyProtocolCounters(), nativeToolCalls: 2, badArgs: 1 }, + sampleCount: 0, + sampleReasons: [], + }, + ], +}; +assert(formatExecutionReport(rollupRecentOnly).includes('nativeToolCalls=2'), 'report uses recent runs only'); +assert(!formatExecutionReport(rollupRecentOnly).includes('nativeToolCalls=999'), 'report ignores lifetime sums'); + +let scoreCalls = 0; +const lost = await executeHarnessRevision({ + artifact: empty, + report: 'badArgs=20 lengthExits=4', + propose: async () => + JSON.stringify({ + diagnosis: 'writes too large', + sections: { writePolicy: 'Write a skeleton under 40 lines, then grow it with edit.' }, + }), + score: async input => { + scoreCalls += 1; + if (input.phase === 'parent') { + return { byTask: { T2: { parent: [70, 70, 70], candidate: [] } } }; + } + return { byTask: { T2: { parent: [70, 70, 70], candidate: [40, 40, 40] } } }; + }, + now: '2026-10-21T00:00:00.000Z', +}); +assert(scoreCalls === 2, 'parent and paired were scored'); +assert(!lost.kept, 'losing revision reverted'); +assert(lost.artifact.sections.writePolicy === '', 'parent text restored'); +assert(lost.artifact.history.length === 1 && lost.artifact.history[0].decision === 'revert', 'revert recorded'); + +scoreCalls = 0; +const won = await executeHarnessRevision({ + artifact: empty, + report: 'badArgs=20 lengthExits=4', + propose: async () => + JSON.stringify({ + diagnosis: 'writes too large', + sections: { writePolicy: 'Write a skeleton under 40 lines, then grow it with edit.' }, + }), + score: async input => { + scoreCalls += 1; + if (input.phase === 'parent') { + return { byTask: { T2: { parent: [60, 61, 59], candidate: [] } } }; + } + return { byTask: { T2: { parent: [60, 61, 59], candidate: [88, 90, 86] } } }; + }, + now: '2026-10-21T00:00:00.000Z', +}); +assert(won.kept, 'winning revision kept'); +assert(won.artifact.sections.writePolicy.includes('skeleton'), 'kept text is the candidate'); +assert(won.artifact.revision === 1, 'kept revision is 1'); + +scoreCalls = 0; +const skipped = await executeHarnessRevision({ + artifact: empty, + report: 'No protocol rollup for this model yet.', + propose: async () => '{"diagnosis":"no evidence","sections":{},"knobs":{}}', + score: async input => { + scoreCalls += 1; + return { byTask: { T2: { parent: [100, 100, 100], candidate: [] } } }; + }, +}); +assert(!skipped.kept && !skipped.scored && scoreCalls === 1, 'no evidence scores only the parent and does not keep'); + +let seenReport = ''; +await executeHarnessRevision({ + artifact: empty, + report: 'badArgs=0', + parentOutcomes: () => [ + { + task: 'T2-feature-coupon', + score: 40, + loopExit: 'budget', + steps: 31, + hiddenFailed: ['hidden: percent coupon'], + checksFailed: [], + replyTail: 'I need to stop the exploration loop and implement the solution.', + }, + ], + propose: async messages => { + seenReport = messages[1].content; + return '{"diagnosis":"no evidence","sections":{},"knobs":{}}'; + }, + score: async () => ({ byTask: { T2: { parent: [40, 40, 40], candidate: [] } } }), +}); +assert(seenReport.includes('T2-feature-coupon: score 40/100, loop exit budget'), 'proposer sees parent task outcomes'); +assert(seenReport.includes('hidden tests failed: hidden: percent coupon'), 'proposer sees failed hidden tests'); +assert(seenReport.includes('stop the exploration loop'), 'proposer sees the final reply'); +assert(!seenReport.includes('31 tool steps'), 'budget exit does not double-count steps'); + +console.log('harnessRevision.selftest: ok'); +} diff --git a/extensions/codeforge/src/agent/harnessRevision.ts b/extensions/codeforge/src/agent/harnessRevision.ts new file mode 100644 index 0000000..9281b99 --- /dev/null +++ b/extensions/codeforge/src/agent/harnessRevision.ts @@ -0,0 +1,657 @@ +/** + * Test-time harness revision (Harness Learning). + * A proposer persona edits a bounded artifact. The artifact is kept only when + * repeated benchmark scores beat the parent by a margin. Oracle, tests, and + * guard code are not editable. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { modelProfileDir } from '../storage/paths'; +import { mergeModelProfile, type ModelProfile } from './modelProfile'; +import { + emptyProtocolCounters, + PROTOCOL_COUNTER_KEYS, + type ProtocolRollup, +} from './protocolTelemetry'; + +export const HARNESS_SECTIONS = ['workflow', 'writePolicy', 'toolContract'] as const; +export type HarnessSection = (typeof HARNESS_SECTIONS)[number]; + +export const SECTION_MAX_CHARS = 1200; +export const DEFAULT_MIN_REPS = 3; +export const DEFAULT_MIN_DELTA = 2; +export const DEFAULT_TASK_REGRESSION = 2; +export const DEFAULT_BOOTSTRAP_SAMPLES = 2000; + +const KNOB_LIMITS = { + writeMaxContentLines: { min: 40, max: 400 }, + mutatingMaxTokens: { min: 1024, max: 8192 }, +} as const; + +/** Artifact text must not smuggle phrase matchers or a license to skip checks. */ +const FORBIDDEN_SNIPPETS = [ + 'harness note', + 'if the user says', + 'if the user asks', + 'when the user says', + 'when the user asks', + 'phrase match', + 'ignore the oracle', + 'disregard the oracle', + 'skip tests', + 'skip the tests', + 'ignore tests', +]; + +export type HarnessKnobs = { + writeMaxContentLines?: number; + mutatingMaxTokens?: number; + enforceMassRewriteGate?: boolean; +}; + +export type HarnessHistoryEntry = { + at: string; + decision: 'keep' | 'revert'; + reason: string; + diagnosis: string; + parentMean?: number; + candidateMean?: number; + parentReps: number; + candidateReps: number; +}; + +export type HarnessArtifact = { + version: 1; + revision: number; + sections: Record; + knobs: HarnessKnobs; + history: HarnessHistoryEntry[]; +}; + +export type HarnessEdit = { + diagnosis: string; + sections: Partial>; + knobs: HarnessKnobs; +}; + +export function emptyHarnessArtifact(): HarnessArtifact { + return { + version: 1, + revision: 0, + sections: { workflow: '', writePolicy: '', toolContract: '' }, + knobs: {}, + history: [], + }; +} + +export function harnessFilePath(serverId: string, model: string, dir?: string): string { + return path.join(dir ?? modelProfileDir(serverId, model), 'harness.json'); +} + +export function loadHarnessArtifact( + serverId: string, + model: string, + filePath?: string +): HarnessArtifact { + const fp = filePath ?? harnessFilePath(serverId, model); + if (!fs.existsSync(fp)) return emptyHarnessArtifact(); + try { + const raw = JSON.parse(fs.readFileSync(fp, 'utf8')) as Partial; + return normalizeArtifact(raw); + } catch { + return emptyHarnessArtifact(); + } +} + +export function saveHarnessArtifact(artifact: HarnessArtifact, filePath: string): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify(artifact, null, 2), 'utf8'); +} + +function normalizeArtifact(raw: Partial): HarnessArtifact { + const base = emptyHarnessArtifact(); + const sections = { ...base.sections }; + for (const key of HARNESS_SECTIONS) { + const value = raw.sections?.[key]; + let body = typeof value === 'string' ? value.slice(0, SECTION_MAX_CHARS) : ''; + if (body && forbiddenSnippet(body)) body = ''; + sections[key] = body; + } + return { + version: 1, + revision: Number(raw.revision) || 0, + sections, + knobs: sanitizeKnobs(raw.knobs ?? {}), + history: Array.isArray(raw.history) ? raw.history.slice(-20) : [], + }; +} + +function sanitizeKnobs(raw: HarnessKnobs): HarnessKnobs { + const out: HarnessKnobs = {}; + if (inRange(raw.writeMaxContentLines, KNOB_LIMITS.writeMaxContentLines)) { + out.writeMaxContentLines = Math.round(raw.writeMaxContentLines as number); + } + if (inRange(raw.mutatingMaxTokens, KNOB_LIMITS.mutatingMaxTokens)) { + out.mutatingMaxTokens = Math.round(raw.mutatingMaxTokens as number); + } + if (raw.enforceMassRewriteGate === true) { + out.enforceMassRewriteGate = true; + } + return out; +} + +function inRange(value: unknown, lim: { min: number; max: number }): value is number { + return typeof value === 'number' && Number.isFinite(value) && value >= lim.min && value <= lim.max; +} + +export function artifactHasContent(artifact: HarnessArtifact): boolean { + return ( + HARNESS_SECTIONS.some(k => artifact.sections[k].trim().length > 0) || + Object.keys(artifact.knobs).length > 0 + ); +} + +/** System block injected for this model. Empty when nothing has been kept. */ +export function learnedHarnessBlock(artifact: HarnessArtifact): string { + const parts: string[] = []; + if (artifact.sections.workflow.trim()) { + parts.push(`#### Workflow\n${artifact.sections.workflow.trim()}`); + } + if (artifact.sections.writePolicy.trim()) { + parts.push(`#### Write policy\n${artifact.sections.writePolicy.trim()}`); + } + if (artifact.sections.toolContract.trim()) { + parts.push(`#### Tool contract notes\n${artifact.sections.toolContract.trim()}`); + } + if (!parts.length) return ''; + return [ + '### Learned harness', + 'Kept for this model because repeated benchmark scores improved. Follow these notes together with the tool contract above. They do not override the oracle or mutation guards.', + ...parts, + ].join('\n\n'); +} + +export function applyHarnessKnobs(profile: ModelProfile, artifact: HarnessArtifact): ModelProfile { + const knobs = artifact.knobs; + if (!Object.keys(knobs).length) return profile; + const patch: Partial = {}; + if (knobs.writeMaxContentLines !== undefined || knobs.enforceMassRewriteGate !== undefined) { + patch.mutations = { + ...profile.mutations, + ...(knobs.writeMaxContentLines !== undefined + ? { writeMaxContentLines: knobs.writeMaxContentLines } + : {}), + ...(knobs.enforceMassRewriteGate !== undefined + ? { enforceMassRewriteGate: knobs.enforceMassRewriteGate } + : {}), + }; + } + if (knobs.mutatingMaxTokens !== undefined) { + patch.completion = { + ...profile.completion, + mutatingMaxTokens: knobs.mutatingMaxTokens, + }; + } + return mergeModelProfile(profile, patch); +} + +export function formatExecutionReport(rollup: ProtocolRollup | undefined): string { + if (!rollup || !rollup.runCount) { + return 'No protocol rollup for this model yet.'; + } + const runs = rollup.recent ?? []; + if (!runs.length) { + return 'No recent protocol runs for this model yet.'; + } + const n = runs.length; + const sums = emptyProtocolCounters(); + for (const r of runs) { + for (const key of PROTOCOL_COUNTER_KEYS) { + sums[key] += r.protocol[key] ?? 0; + } + } + const native = sums.nativeToolCalls || 0; + const badRate = native > 0 ? (sums.badArgs / native).toFixed(3) : 'n/a'; + const reasons = runs + .flatMap(r => r.sampleReasons ?? []) + .filter(Boolean) + .slice(-8); + const exits = runs + .map(r => r.loopExit) + .filter(Boolean) + .slice(-8); + return [ + `Protocol over ${n} recent runs (rollup total ${rollup.runCount}).`, + `nativeToolCalls=${sums.nativeToolCalls} contentToolCalls=${sums.contentToolCalls} badArgs=${sums.badArgs} (badArgs/native=${badRate}) lengthExits=${sums.lengthExits} emptyReplies=${sums.emptyReplies} unknownTools=${sums.unknownTools}.`, + exits.length ? `Recent loop exits: ${exits.join(', ')}.` : '', + reasons.length ? `Recent failure samples: ${reasons.join('; ')}.` : '', + ] + .filter(Boolean) + .join('\n'); +} + +export type TaskOutcome = { + task: string; + score: number; + loopExit?: string; + steps?: number; + hiddenFailed: string[]; + checksFailed: string[]; + replyTail?: string; +}; + +/** Per-run results of the current harness on the selection tasks. */ +export function formatTaskOutcomes(outcomes: TaskOutcome[]): string { + if (!outcomes.length) return ''; + const latestByTask = new Map(); + for (const o of outcomes) latestByTask.set(o.task, o); + const lines = [...latestByTask.values()].map(o => { + const parts = [`- ${o.task}: score ${o.score}/100`]; + if (o.loopExit) parts.push(`loop exit ${o.loopExit}`); + if (typeof o.steps === 'number' && o.loopExit !== 'budget') { + parts.push(`${o.steps} tool steps`); + } + let line = parts.join(', '); + if (o.hiddenFailed.length) line += `\n hidden tests failed: ${o.hiddenFailed.slice(0, 6).join('; ')}`; + if (o.checksFailed.length) line += `\n checks failed: ${o.checksFailed.slice(0, 4).join('; ')}`; + if (o.replyTail?.trim()) { + line += `\n final reply (clipped): ${o.replyTail.trim().replace(/\s+/g, ' ').slice(0, 280)}`; + } + return line; + }); + return ['Current harness on the selection tasks:', ...lines].join('\n'); +} + +export function proposerMessages( + artifact: HarnessArtifact, + report: string +): Array<{ role: 'system' | 'user'; content: string }> { + const prior = artifact.history + .slice(-5) + .map(h => `- ${h.decision}: ${h.diagnosis} (${h.reason})`) + .join('\n'); + return [ + { + role: 'system', + content: [ + 'You are the harness proposer for CodeForge. You do not solve the user task.', + 'You revise the executable notes and a few knobs that wrap a frozen solver.', + 'Reply with one JSON object and nothing else. No markdown fences, no tool calls.', + 'Schema: {"diagnosis":"...","sections":{"workflow":"","writePolicy":"","toolContract":""},"knobs":{"writeMaxContentLines":180,"mutatingMaxTokens":6144,"enforceMassRewriteGate":true}}', + 'Diagnose from the per-task results first (hidden test failures, loop exits, final replies). Protocol counters are secondary.', + 'Change only the sections or knobs that the execution report supports. Omit unchanged knobs.', + `Each section is at most ${SECTION_MAX_CHARS} characters and states a structural rule (what to read, what a step must return, how to split a large write).`, + 'Do not add phrase matchers, "if the user says", or instructions to ignore the oracle or skip tests.', + 'If the report has no repeated failure, return {"diagnosis":"no evidence","sections":{},"knobs":{}}.', + ].join('\n'), + }, + { + role: 'user', + content: [ + 'Current harness artifact:', + JSON.stringify({ revision: artifact.revision, sections: artifact.sections, knobs: artifact.knobs }), + prior ? `Earlier revision outcomes:\n${prior}` : 'No earlier revision outcomes.', + 'Execution report:', + report, + ].join('\n\n'), + }, + ]; +} + +/** Pull the proposer object out of prose, fences, or a leading think block. */ +export function extractProposerJson(raw: string): unknown | undefined { + const text = String(raw ?? '') + .replace(/]*>[\s\S]*?<\/think>/gi, '') + .replace(/```(?:json)?/gi, ''); + for (let i = 0; i < text.length; i++) { + if (text[i] !== '{') continue; + let depth = 0; + let inStr = false; + let esc = false; + for (let j = i; j < text.length; j++) { + const c = text[j]; + if (inStr) { + if (esc) { + esc = false; + continue; + } + if (c === '\\') { + esc = true; + continue; + } + if (c === '"') inStr = false; + continue; + } + if (c === '"') { + inStr = true; + continue; + } + if (c === '{') depth += 1; + else if (c === '}') { + depth -= 1; + if (depth === 0) { + try { + const value = JSON.parse(text.slice(i, j + 1)) as unknown; + if ( + value && + typeof value === 'object' && + !Array.isArray(value) && + 'diagnosis' in value + ) { + return value; + } + } catch { + /* try a later object */ + } + break; + } + } + } + } + return undefined; +} + +export function parseProposerEdit(raw: string): { ok: true; edit: HarnessEdit } | { ok: false; error: string } { + const parsed = extractProposerJson(raw); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + return { ok: false, error: 'proposer did not return a JSON object' }; + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + return { ok: false, error: 'proposer JSON must be an object' }; + } + const obj = parsed as Record; + const diagnosis = String(obj.diagnosis ?? '').trim(); + if (!diagnosis) return { ok: false, error: 'diagnosis is required' }; + if (diagnosis.toLowerCase() === 'no evidence') { + return { ok: false, error: 'no evidence' }; + } + const sections: Partial> = {}; + const rawSections = obj.sections; + if (rawSections && typeof rawSections === 'object' && !Array.isArray(rawSections)) { + const nested = (rawSections as Record).knobs; + if (nested && typeof nested === 'object' && !Array.isArray(nested)) { + obj.knobs = { + ...((obj.knobs && typeof obj.knobs === 'object' && !Array.isArray(obj.knobs) + ? obj.knobs + : {}) as Record), + ...(nested as Record), + }; + delete (rawSections as Record).knobs; + } + } + if (rawSections !== undefined) { + if (!rawSections || typeof rawSections !== 'object' || Array.isArray(rawSections)) { + return { ok: false, error: 'sections must be an object' }; + } + for (const [key, value] of Object.entries(rawSections as Record)) { + if (!(HARNESS_SECTIONS as readonly string[]).includes(key)) { + return { ok: false, error: `unknown section ${key}` }; + } + if (typeof value !== 'string') return { ok: false, error: `section ${key} must be a string` }; + const body = value.trim(); + if (!body) continue; + if (body.length > SECTION_MAX_CHARS) { + return { ok: false, error: `section ${key} exceeds ${SECTION_MAX_CHARS} chars` }; + } + const banned = forbiddenSnippet(body); + if (banned) return { ok: false, error: `section ${key} contains forbidden text (${banned})` }; + sections[key as HarnessSection] = body; + } + } + const knobs = sanitizeKnobs((obj.knobs ?? {}) as HarnessKnobs); + const rawKnobs = obj.knobs; + if (rawKnobs && typeof rawKnobs === 'object' && !Array.isArray(rawKnobs)) { + for (const key of Object.keys(rawKnobs as object)) { + if (!(key in KNOB_LIMITS) && key !== 'enforceMassRewriteGate') { + return { ok: false, error: `unknown knob ${key}` }; + } + const supplied = (rawKnobs as Record)[key]; + if (key === 'enforceMassRewriteGate' && supplied === false) { + return { ok: false, error: 'knob enforceMassRewriteGate must stay true when set' }; + } + if (supplied !== undefined && !(key in knobs)) { + return { ok: false, error: `knob ${key} is out of range` }; + } + } + } + if (!Object.keys(sections).length && !Object.keys(knobs).length) { + return { ok: false, error: 'no change' }; + } + return { ok: true, edit: { diagnosis, sections, knobs } }; +} + +function normalizeForbiddenText(text: string): string { + return text.toLowerCase().replace(/\s+/g, ' ').trim(); +} + +function forbiddenSnippet(text: string): string | undefined { + const lower = normalizeForbiddenText(text); + return FORBIDDEN_SNIPPETS.find(s => lower.includes(s)); +} + +export function applyHarnessEdit(parent: HarnessArtifact, edit: HarnessEdit): HarnessArtifact { + const sections = { ...parent.sections }; + for (const key of HARNESS_SECTIONS) { + if (edit.sections[key] !== undefined) sections[key] = edit.sections[key] as string; + } + return { + version: 1, + revision: parent.revision + 1, + sections, + knobs: { ...parent.knobs, ...edit.knobs }, + history: parent.history.slice(), + }; +} + +export function meanScore(scores: number[]): number { + if (!scores.length) return Number.NaN; + return scores.reduce((a, b) => a + b, 0) / scores.length; +} + +export type TaskPairedScores = Record; + +/** 90% one-sided lower bound on the mean paired gain (10th percentile of bootstrap means). */ +export function bootstrapMeanLowerBound( + deltas: number[], + samples = DEFAULT_BOOTSTRAP_SAMPLES, + quantile = 0.1, + seed = 0xcf052026 +): number { + if (!deltas.length) return Number.NaN; + const n = deltas.length; + const means: number[] = []; + let state = seed >>> 0; + const rand = () => { + state = (Math.imul(1664525, state) + 1013904223) >>> 0; + return state / 0x1_0000_0000; + }; + for (let b = 0; b < samples; b += 1) { + let sum = 0; + for (let i = 0; i < n; i += 1) { + sum += deltas[Math.floor(rand() * n)]!; + } + means.push(sum / n); + } + means.sort((a, b) => a - b); + const idx = Math.min(means.length - 1, Math.max(0, Math.floor(quantile * means.length))); + return means[idx]!; +} + +function flattenPaired(byTask: TaskPairedScores, side: 'parent' | 'candidate'): number[] { + return Object.values(byTask).flatMap(p => p[side].filter(n => Number.isFinite(n))); +} + +/** + * Keep a revision only when every task has enough paired reps, no task regresses + * beyond the margin, and each task's paired bootstrap 90% lower bound is > 0. + */ +export function decideRevision(input: { + byTask: TaskPairedScores; + minRepsPerTask?: number; + taskRegressionMargin?: number; + bootstrapSamples?: number; +}): { decision: 'keep' | 'revert'; reason: string; parentMean: number; candidateMean: number } { + const minReps = input.minRepsPerTask ?? DEFAULT_MIN_REPS; + const margin = input.taskRegressionMargin ?? DEFAULT_TASK_REGRESSION; + const samples = input.bootstrapSamples ?? DEFAULT_BOOTSTRAP_SAMPLES; + const tasks = Object.keys(input.byTask); + if (!tasks.length) { + return { + decision: 'revert', + reason: 'no paired task scores', + parentMean: Number.NaN, + candidateMean: Number.NaN, + }; + } + for (const task of tasks) { + const pair = input.byTask[task]!; + const parent = pair.parent.filter(n => Number.isFinite(n)); + const candidate = pair.candidate.filter(n => Number.isFinite(n)); + if (parent.length < minReps || candidate.length < minReps) { + return { + decision: 'revert', + reason: `${task}: need ${minReps} paired scores (parent ${parent.length}, candidate ${candidate.length})`, + parentMean: meanScore(flattenPaired(input.byTask, 'parent')), + candidateMean: meanScore(flattenPaired(input.byTask, 'candidate')), + }; + } + if (parent.length !== candidate.length) { + return { + decision: 'revert', + reason: `${task}: paired score count mismatch (${parent.length} vs ${candidate.length})`, + parentMean: meanScore(flattenPaired(input.byTask, 'parent')), + candidateMean: meanScore(flattenPaired(input.byTask, 'candidate')), + }; + } + const parentMean = meanScore(parent); + const candidateMean = meanScore(candidate); + if (candidateMean < parentMean - margin) { + return { + decision: 'revert', + reason: `${task}: candidate mean ${candidateMean.toFixed(1)} is more than ${margin} below parent ${parentMean.toFixed(1)}`, + parentMean: meanScore(flattenPaired(input.byTask, 'parent')), + candidateMean: meanScore(flattenPaired(input.byTask, 'candidate')), + }; + } + const deltas = candidate.map((c, i) => c - parent[i]!); + const lower = bootstrapMeanLowerBound(deltas, samples); + if (!(lower > 0)) { + return { + decision: 'revert', + reason: `${task}: paired bootstrap 90% lower bound ${lower.toFixed(2)} is not > 0`, + parentMean: meanScore(flattenPaired(input.byTask, 'parent')), + candidateMean: meanScore(flattenPaired(input.byTask, 'candidate')), + }; + } + } + const parentMean = meanScore(flattenPaired(input.byTask, 'parent')); + const candidateMean = meanScore(flattenPaired(input.byTask, 'candidate')); + return { + decision: 'keep', + reason: `all ${tasks.length} task(s) cleared paired bootstrap with no regression beyond ${margin}`, + parentMean, + candidateMean, + }; +} + +/** True when held-out CS tasks drop versus the parent under the same margin. */ +export function holdoutRegressed( + parent: TaskPairedScores, + candidate: TaskPairedScores, + margin = DEFAULT_TASK_REGRESSION +): string | undefined { + for (const task of Object.keys(parent)) { + const p = parent[task]?.parent.filter(n => Number.isFinite(n)) ?? []; + const c = candidate[task]?.candidate.filter(n => Number.isFinite(n)) ?? []; + if (!p.length || !c.length) continue; + const parentMean = meanScore(p); + const candidateMean = meanScore(c); + if (candidateMean < parentMean - margin) { + return `${task}: holdout mean ${candidateMean.toFixed(1)} dropped below parent ${parentMean.toFixed(1)}`; + } + } + return undefined; +} + +export type HarnessRevisionOutcome = { + kept: boolean; + reason: string; + diagnosis: string; + artifact: HarnessArtifact; + parentMean?: number; + candidateMean?: number; + scored: boolean; +}; + +/** + * The parent is scored first so the proposer sees its per-task results; a + * proposal written from protocol counters alone cannot target task failures. + */ +export type HarnessScorePhase = + | { phase: 'parent'; artifact: HarnessArtifact } + | { phase: 'paired'; parent: HarnessArtifact; candidate: HarnessArtifact }; + +export type HarnessScoreResult = { + byTask: TaskPairedScores; +}; + +export async function executeHarnessRevision(opts: { + artifact: HarnessArtifact; + report: string; + propose: (messages: Array<{ role: 'system' | 'user'; content: string }>) => Promise; + score: (input: HarnessScorePhase) => Promise; + /** Per-task results gathered while scoring the parent. */ + parentOutcomes?: () => TaskOutcome[]; + minRepsPerTask?: number; + taskRegressionMargin?: number; + now?: string; +}): Promise { + await opts.score({ phase: 'parent', artifact: opts.artifact }); + const taskReport = formatTaskOutcomes(opts.parentOutcomes?.() ?? []); + const report = [taskReport, opts.report].filter(Boolean).join('\n\n'); + const raw = await opts.propose(proposerMessages(opts.artifact, report)); + const parsed = parseProposerEdit(raw); + if (!parsed.ok) { + return { + kept: false, + reason: parsed.error, + diagnosis: '', + artifact: opts.artifact, + scored: false, + }; + } + const candidate = applyHarnessEdit(opts.artifact, parsed.edit); + const paired = await opts.score({ phase: 'paired', parent: opts.artifact, candidate }); + const decision = decideRevision({ + byTask: paired.byTask, + minRepsPerTask: opts.minRepsPerTask, + taskRegressionMargin: opts.taskRegressionMargin, + }); + const parentReps = Object.values(paired.byTask).reduce((s, p) => s + p.parent.length, 0); + const candidateReps = Object.values(paired.byTask).reduce((s, p) => s + p.candidate.length, 0); + const entry: HarnessHistoryEntry = { + at: opts.now ?? new Date().toISOString(), + decision: decision.decision, + reason: decision.reason, + diagnosis: parsed.edit.diagnosis, + parentMean: decision.parentMean, + candidateMean: decision.candidateMean, + parentReps, + candidateReps, + }; + const base = decision.decision === 'keep' ? candidate : opts.artifact; + const artifact: HarnessArtifact = { + ...base, + history: [...base.history, entry].slice(-20), + }; + return { + kept: decision.decision === 'keep', + reason: decision.reason, + diagnosis: parsed.edit.diagnosis, + artifact, + parentMean: decision.parentMean, + candidateMean: decision.candidateMean, + scored: true, + }; +} diff --git a/extensions/codeforge/src/agent/internalPlans.ts b/extensions/codeforge/src/agent/internalPlans.ts index f714de4..e47485d 100644 --- a/extensions/codeforge/src/agent/internalPlans.ts +++ b/extensions/codeforge/src/agent/internalPlans.ts @@ -163,8 +163,12 @@ export async function resolveCurrentPlan( const bare = looksLikeBareContinue(opts.task); const sameSession = !opts.sessionId || !plan.sessionId || opts.sessionId === plan.sessionId; - const sameTask = plan.taskHash === hashTask(opts.task) || plan.taskHash === hashTask(plan.task); + const sameTask = plan.taskHash === hashTask(opts.task); + if (!sameSession && !bare) { + await archiveCurrentPlan(workspaceRoot); + return undefined; + } if (bare && pending && sameSession) return plan; if (sameTask && pending) return plan; if (!pending) { @@ -176,10 +180,6 @@ export async function resolveCurrentPlan( await archiveCurrentPlan(workspaceRoot); return undefined; } - if (!sameSession && !bare) { - await archiveCurrentPlan(workspaceRoot); - return undefined; - } return plan; } diff --git a/extensions/codeforge/src/agent/llmThinking.ts b/extensions/codeforge/src/agent/llmThinking.ts index 2060da7..49dc96b 100644 --- a/extensions/codeforge/src/agent/llmThinking.ts +++ b/extensions/codeforge/src/agent/llmThinking.ts @@ -63,22 +63,39 @@ export function llmThinkingRequestFields(input: ThinkingRoundInput): Record, old_string copied into new_string), and a penalty taxes exactly that. - return { temperature: 0.7, top_p: 0.8, presence_penalty: 0 }; + // Non-thinking / tool rounds. presence_penalty stays 0 by default: tool calls must + // repeat tokens (, old_string copied into new_string). + return { + temperature: samp.toolThinking.temperature, + top_p: samp.toolThinking.top_p, + presence_penalty: samp.toolThinking.presence_penalty, + }; } - // Non-reasoning models keep the historical deterministic default - return { temperature: 0.2, top_p: 1 }; + return { temperature: samp.plain.temperature, top_p: samp.plain.top_p }; } /** @@ -87,7 +104,8 @@ export function llmSamplingOptions(input: ThinkingRoundInput): { */ export function toolAwareMaxTokens( completionCap: number, - input: ThinkingRoundInput & { tools?: Array<{ function?: { name?: string } }> } + input: ThinkingRoundInput & { tools?: Array<{ function?: { name?: string } }> }, + limits?: { mutatingMaxTokens: number; readOnlyMaxTokens: number } ): number { if (!input.withTools && !input.proseToolsOnly) return completionCap; // Plan keeps thinking ON — do not starve the completion budget. @@ -95,13 +113,15 @@ export function toolAwareMaxTokens( if (!shouldDisableThinking(input) && !modelLikelySupportsThinking(input.model)) { return completionCap; } + const mutating = limits?.mutatingMaxTokens ?? 6144; + const readOnly = limits?.readOnlyMaxTokens ?? 3072; const names = (input.tools ?? []).map(t => t.function?.name ?? ''); const mutates = names.some(n => /^(write|edit|apply_patch|notebook_edit|search_replace)$/i.test(n) ); - if (mutates) return Math.min(completionCap, 6144); + if (mutates) return Math.min(completionCap, mutating); // Parallel tool rounds need room for several argument payloads. - return Math.min(completionCap, 3072); + return Math.min(completionCap, readOnly); } const THINK_BLOCK_RE = /]*>[\s\S]*?<\/think>/gi; diff --git a/extensions/codeforge/src/agent/loopDetector.ts b/extensions/codeforge/src/agent/loopDetector.ts index 2d08c22..46105d0 100644 --- a/extensions/codeforge/src/agent/loopDetector.ts +++ b/extensions/codeforge/src/agent/loopDetector.ts @@ -1,15 +1,53 @@ /** * Detects tool-loop stalls from identical call+result streaks (and related counters). * Pure — no vscode. + * + * No explore-step soft/hard caps: like Cursor/Claude Code, reads are unlimited within the + * turn step budget. Stall = identical repeats or browser UI unchanged. */ +export const EXPLORE_TOOL_NAMES = [ + 'read', + 'list', + 'search', + 'retrieve', + 'symbols', + 'definition', + 'references', +] as const; + +export function isExploreToolName(name: string): boolean { + return (EXPLORE_TOOL_NAMES as readonly string[]).includes(name); +} + +/** + * Tools that count as real progress for write/oracle gates elsewhere. + * skill / update_status / await_shell alone must NOT reset progress streaks. + */ +export const EFFECTFUL_TOOL_NAMES = [ + 'edit', + 'write', + 'apply_patch', + 'delete', + 'rename', + 'shell', + 'dotnet', + 'browser_navigate', + 'browser_click', + 'browser_type', + 'browser_snapshot', + 'browser_screenshot', +] as const; + +export function isEffectfulToolName(name: string): boolean { + return (EFFECTFUL_TOOL_NAMES as readonly string[]).includes(name); +} + export interface LoopDetector { recordCall(fingerprint: string, resultHash: string): { note?: string }; endStep(): void; shouldStop(): boolean; cause(): string; - setExploreOnlySteps(n: number): void; - getExploreOnlySteps(): number; setUnchangedStreak(n: number): void; getUnchangedStreak(): number; /** Consecutive steps where every call repeated a prior fingerprint+result. */ @@ -30,7 +68,6 @@ export function hashResult(text: string): string { export function createLoopDetector(): LoopDetector { let currentStep = 1; let consecutiveIdenticalSteps = 0; - let exploreOnlySteps = 0; let unchangedStreak = 0; /** fingerprint\0resultHash → step number when first seen */ const firstSeen = new Map(); @@ -60,7 +97,7 @@ export function createLoopDetector(): LoopDetector { }, shouldStop(): boolean { - return consecutiveIdenticalSteps >= 3 || exploreOnlySteps >= 20 || unchangedStreak >= 3; + return consecutiveIdenticalSteps >= 3 || unchangedStreak >= 3; }, cause(): string { @@ -70,20 +107,9 @@ export function createLoopDetector(): LoopDetector { if (unchangedStreak >= 3) { return 'browser UI UNCHANGED for 3 consecutive actions'; } - if (exploreOnlySteps >= 20) { - return '20 consecutive explore-only steps without a write or shell'; - } return 'tools stuck without progress'; }, - setExploreOnlySteps(n: number): void { - exploreOnlySteps = Math.max(0, n); - }, - - getExploreOnlySteps(): number { - return exploreOnlySteps; - }, - setUnchangedStreak(n: number): void { unchangedStreak = Math.max(0, n); }, diff --git a/extensions/codeforge/src/agent/modelProfile.selftest.ts b/extensions/codeforge/src/agent/modelProfile.selftest.ts new file mode 100644 index 0000000..a23336c --- /dev/null +++ b/extensions/codeforge/src/agent/modelProfile.selftest.ts @@ -0,0 +1,105 @@ +/** + * ModelProfile defaults, merge, load, strong calibration. + */ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { + applyCalibrationFromStats, + applyStrongCalibration, + classifyCalibration, + defaultModelProfile, + loadModelProfile, + mergeModelProfile, + saveModelProfile, +} from './modelProfile'; + +function assert(cond: boolean, msg: string): void { + if (!cond) throw new Error(msg); +} + +const d = defaultModelProfile('s', 'm'); +assert(d.sampling.toolThinking.presence_penalty === 0, 'pp 0'); +assert(d.mutations.writeMaxContentLines === 180, 'write cap 180'); +assert(d.finish.specSelfCheckEnabled === true, 'spec check on'); +assert(d.orchestrator.itemMaxSteps === 25, 'orch steps'); + +const merged = mergeModelProfile(d, { + mutations: { writeMaxContentLines: 0, largeFileExistingLines: 60, enforceMassRewriteGate: false }, +}); +assert(merged.mutations.writeMaxContentLines === 0, 'merge override'); +assert(merged.sampling.toolThinking.temperature === 0.7, 'merge keeps rest'); + +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'cf-prof-')); +const fp = path.join(tmp, 'profile.json'); +saveModelProfile(merged, fp); +const loaded = loadModelProfile('s', 'm', { + filePath: fp, + createIfMissing: false, + applyHysteresis: false, +}); +assert(loaded.mutations.writeMaxContentLines === 0, 'load disk'); + +const strong = applyStrongCalibration(defaultModelProfile('s', 'strong'), { + passRate: 1, + badArgsRate: 0.01, + lengthExits: 0, + totalRuns: 12, +}); +assert(strong.mutations.writeMaxContentLines === 0, 'strong no write cap'); +assert(strong.finish.specSelfCheckEnabled === false, 'strong no spec check'); +assert(strong.prompt.forcedWeakSkill === false, 'strong no weak skill'); + +const strongBorder = applyStrongCalibration(defaultModelProfile('s', 'strong2'), { + passRate: 0.5, + badArgsRate: 0.03, + lengthExits: 0, + totalRuns: 12, +}); +assert(strongBorder.mutations.writeMaxContentLines === 0, '0.03 clears write cap'); +assert(strongBorder.prompt.forcedWeakSkill === false, '0.03 clears weak skill'); +assert(strongBorder.finish.specSelfCheckEnabled === true, 'spec check needs 11/12'); + +assert(classifyCalibration({ passRate: 0.5, badArgsRate: 0.03, lengthExits: 0, totalRuns: 12 }) === 'strong', 'class strong'); +assert(classifyCalibration({ passRate: 0.5, badArgsRate: 0.2, lengthExits: 0, totalRuns: 12 }) === 'weak', 'class weak'); +assert( + classifyCalibration({ + passRate: 0.9, + badArgsRate: 0.01, + lengthExits: 0, + totalRuns: 12, + nativeToolCalls: 0, + }) === 'weak', + 'no native → weak' +); +assert( + classifyCalibration({ + passRate: 0.4, + badArgsRate: 0.01, + lengthExits: 0, + totalRuns: 12, + nativeToolCalls: 10, + }) === 'weak', + 'low passRate → weak' +); + +const autoWeak = applyCalibrationFromStats(defaultModelProfile('s', 'auto'), { + passRate: 0.4, + badArgsRate: 0.2, + lengthExits: 0, + totalRuns: 12, +}); +assert(autoWeak.classification === 'weak', 'auto weak'); +assert(autoWeak.profile.mutations.writeMaxContentLines === 180, 'auto weak cap'); + +const autoStrong = applyCalibrationFromStats(defaultModelProfile('s', 'auto2'), { + passRate: 0.5, + badArgsRate: 0.03, + lengthExits: 0, + totalRuns: 12, +}); +assert(autoStrong.classification === 'strong', 'auto strong'); +assert(autoStrong.profile.mutations.writeMaxContentLines === 0, 'auto strong no cap'); + +fs.rmSync(tmp, { recursive: true, force: true }); +console.log('modelProfile.selftest: ok'); diff --git a/extensions/codeforge/src/agent/modelProfile.ts b/extensions/codeforge/src/agent/modelProfile.ts new file mode 100644 index 0000000..4afe79b --- /dev/null +++ b/extensions/codeforge/src/agent/modelProfile.ts @@ -0,0 +1,267 @@ +/** + * Per-model harness profile. Defaults match today's hardcoded values. + * Compensations live here; structural correctness (oracle, leak repair, mutationGuard) does not. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { modelProfileDir } from '../storage/paths'; +import { applyOnlineHysteresis } from './profileHysteresis'; + +export type ModelProfile = { + version: 1; + serverId: string; + model: string; + sampling: { + plan: { temperature: number; top_p: number }; + toolThinking: { temperature: number; top_p: number; presence_penalty: number }; + plain: { temperature: number; top_p: number }; + }; + completion: { mutatingMaxTokens: number; readOnlyMaxTokens: number }; + mutations: { + /** 0 = no cap. Default 180 on weak write gate. */ + writeMaxContentLines: number; + largeFileExistingLines: number; + enforceMassRewriteGate: boolean; + }; + finish: { + maxDiagFinishNudges: number; + specSelfCheckMaxNudges: number; + specSelfCheckEnabled: boolean; + }; + prompt: { + renameSearchRule: boolean; + shellCmdExeHints: boolean; + forcedWeakSkill: boolean; + }; + /** + * How tool calls are transported: + * - native: server parses tool_calls (default; strong models) + * - text: tools described in prompt; model emits XML/`tool_calls` in content; CodeForge parses + */ + toolTransport?: 'native' | 'text'; + weak: { maxWritesWithoutBuild: number }; + orchestrator: { + autoMinCards: number; + itemMaxSteps: number; + itemHardCap: number; + maxItemAttempts: number; + }; + calibration?: { + at: string; + suite: string; + passRate: number; + meanScore: number; + protocolMeans?: Record; + /** Set by calibrate: weak | strong */ + classification?: 'weak' | 'strong'; + }; + history?: Array<{ at: string; rule: string; from: unknown; to: unknown }>; +}; + +export function defaultModelProfile(serverId = '', model = ''): ModelProfile { + return { + version: 1, + serverId, + model, + sampling: { + plan: { temperature: 0.6, top_p: 0.95 }, + toolThinking: { temperature: 0.7, top_p: 0.8, presence_penalty: 0 }, + plain: { temperature: 0.2, top_p: 1 }, + }, + completion: { mutatingMaxTokens: 6144, readOnlyMaxTokens: 3072 }, + mutations: { + writeMaxContentLines: 180, + largeFileExistingLines: 60, + enforceMassRewriteGate: true, + }, + finish: { + maxDiagFinishNudges: 2, + specSelfCheckMaxNudges: 1, + specSelfCheckEnabled: true, + }, + prompt: { + renameSearchRule: true, + shellCmdExeHints: true, + forcedWeakSkill: true, + }, + weak: { maxWritesWithoutBuild: 2 }, + orchestrator: { + autoMinCards: 2, + itemMaxSteps: 25, + itemHardCap: 40, + maxItemAttempts: 2, + }, + history: [], + }; +} + +function deepMerge>(base: T, over: Partial | undefined): T { + if (!over) return structuredClone(base); + const out = structuredClone(base) as Record; + for (const [k, v] of Object.entries(over)) { + if (v === undefined) continue; + if (v && typeof v === 'object' && !Array.isArray(v) && typeof out[k] === 'object' && out[k] && !Array.isArray(out[k])) { + out[k] = deepMerge(out[k] as Record, v as Record); + } else { + out[k] = v; + } + } + return out as T; +} + +export function mergeModelProfile(base: ModelProfile, over: Partial | undefined): ModelProfile { + return deepMerge(base as unknown as Record, over as unknown as Record) as unknown as ModelProfile; +} + +export function profileFilePath(serverId: string, model: string): string { + return path.join(modelProfileDir(serverId, model), 'profile.json'); +} + +export function saveModelProfile(profile: ModelProfile, filePath?: string): void { + const fp = filePath ?? profileFilePath(profile.serverId, profile.model); + fs.mkdirSync(path.dirname(fp), { recursive: true }); + fs.writeFileSync(fp, JSON.stringify(profile, null, 2), 'utf8'); +} + +export type LoadModelProfileOpts = { + /** Skip creating the file on first load (tests). */ + createIfMissing?: boolean; + /** Apply online hysteresis from rollup (default true for real runs). */ + applyHysteresis?: boolean; + filePath?: string; +}; + +/** + * Load profile from disk merged over defaults. Creates the file with defaults on first load. + */ +export function loadModelProfile( + serverId: string, + model: string, + opts: LoadModelProfileOpts = {} +): ModelProfile { + const create = opts.createIfMissing !== false; + const applyHyst = opts.applyHysteresis !== false; + const fp = opts.filePath ?? profileFilePath(serverId, model); + let profile = defaultModelProfile(serverId, model); + if (fs.existsSync(fp)) { + try { + const raw = JSON.parse(fs.readFileSync(fp, 'utf8')) as Partial; + profile = mergeModelProfile(profile, { ...raw, serverId, model }); + } catch { + /* keep default */ + } + } else if (create) { + saveModelProfile(profile, fp); + } + if (applyHyst) { + const next = applyOnlineHysteresis(profile); + if (next.changed) { + profile = next.profile; + saveModelProfile(profile, fp); + } + } + return profile; +} + +/** + * Resolve profile for a run: disk profile, then optional server sampling override + * (same precedence as opts.sampling over llmSamplingOptions). + */ +export function resolveProfileForRun( + serverId: string, + model: string, + serverSampling?: { + temperature?: number; + top_p?: number; + presence_penalty?: number; + }, + opts?: LoadModelProfileOpts +): ModelProfile { + const profile = loadModelProfile(serverId, model, opts); + if (!serverSampling) return profile; + const patch: Partial = { sampling: { ...profile.sampling } }; + const tool = { ...profile.sampling.toolThinking }; + if (typeof serverSampling.temperature === 'number') tool.temperature = serverSampling.temperature; + if (typeof serverSampling.top_p === 'number') tool.top_p = serverSampling.top_p; + if (typeof serverSampling.presence_penalty === 'number') { + tool.presence_penalty = serverSampling.presence_penalty; + } + patch.sampling = { + ...profile.sampling, + toolThinking: tool, + plain: { + ...profile.sampling.plain, + ...(typeof serverSampling.temperature === 'number' + ? { temperature: serverSampling.temperature } + : {}), + ...(typeof serverSampling.top_p === 'number' ? { top_p: serverSampling.top_p } : {}), + }, + }; + return mergeModelProfile(profile, patch); +} + +export type CalibrationStats = { + passRate: number; + badArgsRate: number; + lengthExits: number; + totalRuns: number; + /** Native tool_calls observed (0 → never trust "strong" for local XML models). */ + nativeToolCalls?: number; +}; + +export type CalibrationClass = 'weak' | 'strong'; + +/** + * Classify from protocol + pass rate — not from model name. + * Clean tool protocol → strong (drop cost knobs); otherwise keep weak defaults. + * Strong requires a minimum pass rate and at least one native tool call when measured. + */ +export function classifyCalibration(stats: CalibrationStats): CalibrationClass { + if (stats.totalRuns < 1) return 'weak'; + if (stats.passRate < 0.5) return 'weak'; + if (typeof stats.nativeToolCalls === 'number' && stats.nativeToolCalls <= 0) return 'weak'; + if (stats.badArgsRate < 0.05 && stats.lengthExits === 0) return 'strong'; + return 'weak'; +} + +/** Reset to hand-tuned weak defaults (ignore online flips from the calibrate window). */ +export function applyWeakCalibration(profile: ModelProfile): ModelProfile { + const def = defaultModelProfile(profile.serverId, profile.model); + const next = structuredClone(profile); + next.sampling = def.sampling; + next.mutations = def.mutations; + next.completion = def.completion; + next.finish = { ...def.finish, specSelfCheckEnabled: true }; + next.prompt = { ...def.prompt, forcedWeakSkill: true }; + next.weak = def.weak; + return next; +} + +/** Strong-model knobs when protocol looks clean. */ +export function applyStrongCalibration( + profile: ModelProfile, + stats: CalibrationStats +): ModelProfile { + const next = structuredClone(profile); + if (stats.totalRuns < 1) return next; + next.mutations.enforceMassRewriteGate = false; + next.mutations.writeMaxContentLines = 0; + next.prompt.forcedWeakSkill = false; + if (stats.passRate >= 11 / 12) { + next.finish.specSelfCheckEnabled = false; + } + return next; +} + +/** Classify then apply weak or strong knobs. */ +export function applyCalibrationFromStats( + profile: ModelProfile, + stats: CalibrationStats +): { profile: ModelProfile; classification: CalibrationClass } { + const classification = classifyCalibration(stats); + if (classification === 'strong') { + return { profile: applyStrongCalibration(profile, stats), classification }; + } + return { profile: applyWeakCalibration(profile), classification }; +} diff --git a/extensions/codeforge/src/agent/nudges.ts b/extensions/codeforge/src/agent/nudges.ts index bd2f03d..92723e3 100644 --- a/extensions/codeforge/src/agent/nudges.ts +++ b/extensions/codeforge/src/agent/nudges.ts @@ -49,9 +49,10 @@ export const MAX_STEPS_PROMPT = [ '', 'STRICT REQUIREMENTS:', '1. Do NOT make any tool calls', - '2. State clearly that maximum steps for this agent have been reached', + '2. State clearly that the step budget for this turn was reached', '3. Summarize what was actually done (files written/edited, commands run) — do not invent applied patches', '4. List remaining work as concrete next steps (file + change), not as unapplied code blocks', + '5. Do not ask the user to type "continue" — the UI offers a Continue button when the budget ends', '', 'Any attempt to use tools is a critical violation. Respond with text ONLY.', ].join('\n'); @@ -71,7 +72,7 @@ export function stuckWrapPrompt(cause: string): string { '1. Do NOT make any tool calls', '2. State the stop reason above', '3. Summarize what is already known from tool results', - '4. Give one concrete next step for the user (file path + change), without pasting a full unapplied patch', + '4. Tell the user the next turn must use edit, write, shell, or browser (not only re-read)', '', 'Respond with text ONLY.', ].join('\n'); @@ -156,6 +157,27 @@ export const DIAG_ONLY_FINISH_NOTE = [ 'A bug list or plan alone does not finish the task.', ].join('\n'); +/** + * One-shot: build/test green is not enough for runnable apps — launch and observe. + * Structural (tool counts) — not reply-text sniffing. + */ +export function runtimeVerifyNote(opts?: { sourcePaths?: string[] }): string { + const sources = (opts?.sourcePaths ?? []).slice(0, 12).join(', ') || '(changed sources)'; + return [ + 'RUNTIME VERIFY (one-shot)', + `Source files changed: ${sources}`, + 'Build/test green is not enough. Launch the app (or open it in the browser), then observe the result.', + 'Do one of:', + '- Web / UI: browser_navigate (or click through the flow) then browser_snapshot or browser_screenshot; use the After-state as ground truth.', + '- Desktop / local: shell with the project run/start script (or the existing launch command), then check process output or reopen and confirm the feature works.', + 'Do not treat `npm test` / `dotnet test` / oracle build as this check — those already ran.', + 'Do not create or edit files under test directories for this check. Prefer existing start/dev/run scripts.', + 'If the app opens but the feature still fails, call edit and fix, then re-verify.', + 'If you cannot launch (missing runtime, no display), reply `blocked: `.', + 'Only after a successful launch+observe, reply with a short summary.', + ].join('\n'); +} + /** Finish refused: the oracle is red. `oracleText` is the formatted oracle result. */ export function oracleRejectNote(oracleText: string): string { return [ @@ -172,9 +194,28 @@ export function finalReplyNote(kind: 'still-red' | 'unverified', command?: strin : '\n\n---\nNote: no build/test was verified after the last changes.'; } +/** Empty / placeholder blocked reasons the model invents — not real blockers. */ +const EMPTY_BLOCKED_REASONS = new Set([ + 'none', + 'nothing', + 'n/a', + 'na', + '-', + '—', + '.', + 'no', + 'ok', + 'null', + 'undefined', +]); + /** The model explicitly declares it cannot finish (`blocked: `). */ export function parseBlockedClaim(text: string): string | undefined { const m = /(?:^|\n)\s*(?:\*\*)?(?:blocked|bloqueado)(?:\*\*)?\s*:\s*(.+)/i.exec(String(text || '')); - return m ? m[1].trim().slice(0, 400) : undefined; + if (!m) return undefined; + const reason = m[1].trim().slice(0, 400); + const normalized = reason.replace(/[*_`]+/g, '').trim().toLowerCase(); + if (!normalized || EMPTY_BLOCKED_REASONS.has(normalized)) return undefined; + return reason; } diff --git a/extensions/codeforge/src/agent/orchestrator.ts b/extensions/codeforge/src/agent/orchestrator.ts index 7ffd3cd..5189d60 100644 --- a/extensions/codeforge/src/agent/orchestrator.ts +++ b/extensions/codeforge/src/agent/orchestrator.ts @@ -123,9 +123,14 @@ export function mentionsCards(task: string): boolean { * Multi-card work should run as isolated sub-runs (one card per agent loop). * Forced = slash `/cards` / `/plan-run`. Auto = task mentions cards and ≥2 cards exist. */ -export function shouldOrchestrateCards(task: string, cardCount: number, forced: boolean): boolean { +export function shouldOrchestrateCards( + task: string, + cardCount: number, + forced: boolean, + autoMinCards = 2 +): boolean { if (forced) return cardCount >= 1; - return cardCount >= 2 && mentionsCards(task); + return cardCount >= autoMinCards && mentionsCards(task); } export function wantsNewApp(task: string): boolean { diff --git a/extensions/codeforge/src/agent/orchestratorRun.ts b/extensions/codeforge/src/agent/orchestratorRun.ts index 668ce6c..6e32ad0 100644 --- a/extensions/codeforge/src/agent/orchestratorRun.ts +++ b/extensions/codeforge/src/agent/orchestratorRun.ts @@ -25,9 +25,7 @@ import { wantsNewApp, type Plan, } from './orchestrator'; - -const ITEM_MAX_STEPS = 25; -const ITEM_HARD_CAP = 40; +import { resolveProfileForRun } from './modelProfile'; export interface OrchestrateOptions { /** `/cards` or `/plan-run`: run even when the automatic trigger would not. */ @@ -76,13 +74,22 @@ export async function runOrchestrated( const root = opts.workspaceRoot; if (!root) return undefined; const forced = o.forced === true; + const profile = + opts.modelProfile ?? + resolveProfileForRun(opts.serverId || 'default', opts.model, opts.sampling); + opts.modelProfile = profile; const cards = orderCards(selectCards(opts.task, findCards(root, nodeOracleFs))); // Cards: auto when the task names them and ≥2 exist; slash forces even a single card. // New-app multi-item planning still goes through buildPlan (forced or wantsNewApp). - const cardJob = shouldOrchestrateCards(opts.task, cards.length, forced); + const cardJob = shouldOrchestrateCards( + opts.task, + cards.length, + forced, + profile.orchestrator.autoMinCards + ); if (!cardJob && !forced && !wantsNewApp(opts.task)) return undefined; - const oracleCfg = detectOracle(root, nodeOracleFs); + let oracleCfg = detectOracle(root, nodeOracleFs); const plan = await buildPlan(opts, root, forced || cardJob, !!oracleCfg); if (!plan) return undefined; @@ -121,6 +128,9 @@ export async function runOrchestrated( }); const verify = async (): Promise<{ ok: boolean; text: string }> => { + if (!oracleCfg) { + oracleCfg = detectOracle(root, nodeOracleFs); + } if (!oracleCfg) return { ok: true, text: '' }; const result = await runOracle(oracleCfg, async command => { const r = await opts.bridge.execute({ @@ -138,6 +148,7 @@ export async function runOrchestrated( cancelled: opts.cancelled, save: p => savePlan(root, p), onUpdate: publish, + maxAttempts: profile.orchestrator.maxItemAttempts, runItem: async (item, p) => { const idx = p.items.indexOf(item) + 1; const card = item.source ? nodeOracleFs.readText(path.join(root, item.source)) : undefined; @@ -159,8 +170,9 @@ export async function runOrchestrated( history: [], isolated: true, basePhase: 'implement', - maxSteps: ITEM_MAX_STEPS, - hardCap: ITEM_HARD_CAP, + maxSteps: profile.orchestrator.itemMaxSteps, + hardCap: profile.orchestrator.itemHardCap, + modelProfile: profile, }); const blocked = parseBlockedClaim(reply); if (blocked) return { ok: false, reply, blocked }; diff --git a/extensions/codeforge/src/agent/profileHysteresis.selftest.ts b/extensions/codeforge/src/agent/profileHysteresis.selftest.ts new file mode 100644 index 0000000..8fe4bd5 --- /dev/null +++ b/extensions/codeforge/src/agent/profileHysteresis.selftest.ts @@ -0,0 +1,103 @@ +/** + * Online hysteresis thresholds. + */ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { applyOnlineHysteresis } from './profileHysteresis'; +import { defaultModelProfile, saveModelProfile } from './modelProfile'; +import type { ProtocolRollup, RunTelemetryRecord } from './protocolTelemetry'; +import { emptyProtocolCounters } from './protocolTelemetry'; + +function assert(cond: boolean, msg: string): void { + if (!cond) throw new Error(msg); +} + +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'cf-hyst-')); +process.env.CODEFORGE_MODEL_PROFILE_HOME = tmp; + +const serverId = 'srv'; +const model = 'mdl'; +const dir = path.join(tmp, 'ai', 'model-profiles', serverId, model); +fs.mkdirSync(dir, { recursive: true }); + +function makeRun(reason?: string): RunTelemetryRecord { + return { + at: new Date().toISOString(), + serverId, + model, + source: 'bench', + protocol: emptyProtocolCounters(), + sampleCount: reason ? 1 : 0, + sampleReasons: reason ? [reason] : [], + }; +} + +// 12 trunc hits of 12 → rate 1 → turn write cap on from 0 +const profileOff = defaultModelProfile(serverId, model); +profileOff.mutations.writeMaxContentLines = 0; +const rollupOn: ProtocolRollup = { + serverId, + model, + updatedAt: new Date().toISOString(), + runCount: 12, + sums: emptyProtocolCounters(), + recent: Array.from({ length: 12 }, () => makeRun('truncated_tool_json')), +}; +fs.writeFileSync(path.join(dir, 'rollup.json'), JSON.stringify(rollupOn), 'utf8'); +const r1 = applyOnlineHysteresis(profileOff); +assert(r1.changed && r1.profile.mutations.writeMaxContentLines === 180, 'cap turns on'); + +// 12 clean runs is not enough for off (needs full 50-run window) +const profileOn = defaultModelProfile(serverId, model); +profileOn.mutations.writeMaxContentLines = 180; +const rollupOffShort: ProtocolRollup = { + ...rollupOn, + recent: Array.from({ length: 12 }, () => makeRun()), +}; +fs.writeFileSync(path.join(dir, 'rollup.json'), JSON.stringify(rollupOffShort), 'utf8'); +const r2short = applyOnlineHysteresis(profileOn); +assert(!r2short.changed, 'cap stays on until 50 clean runs'); + +// 50 clean runs → rate 0 → turn cap off +const rollupOff: ProtocolRollup = { + ...rollupOn, + runCount: 50, + recent: Array.from({ length: 50 }, () => makeRun()), +}; +fs.writeFileSync(path.join(dir, 'rollup.json'), JSON.stringify(rollupOff), 'utf8'); +const r2 = applyOnlineHysteresis(profileOn); +assert(r2.changed && r2.profile.mutations.writeMaxContentLines === 0, 'cap turns off'); + +// fewer than 10 runs → no change +const few: ProtocolRollup = { + ...rollupOn, + recent: Array.from({ length: 5 }, () => makeRun('truncated_tool_json')), +}; +fs.writeFileSync(path.join(dir, 'rollup.json'), JSON.stringify(few), 'utf8'); +const r3 = applyOnlineHysteresis(profileOff); +assert(!r3.changed, 'need 10 runs'); + +// leak rate ≥ 0.08 → toolThinking.temperature −0.1 (floor 0.3) +const profileTemp = defaultModelProfile(serverId, model); +profileTemp.sampling.toolThinking.temperature = 0.7; +const leakRuns = Array.from({ length: 12 }, (_, i) => + makeRun(i < 2 ? 'leaked_param' : undefined) +); // 2/12 ≈ 0.167 +const rollupLeak: ProtocolRollup = { + ...rollupOn, + recent: leakRuns, +}; +fs.writeFileSync(path.join(dir, 'rollup.json'), JSON.stringify(rollupLeak), 'utf8'); +const r4 = applyOnlineHysteresis(profileTemp); +assert(r4.changed && r4.profile.sampling.toolThinking.temperature === 0.6, 'temp down'); + +const profileFloor = defaultModelProfile(serverId, model); +profileFloor.sampling.toolThinking.temperature = 0.35; +const r5 = applyOnlineHysteresis(profileFloor); +assert(r5.changed && r5.profile.sampling.toolThinking.temperature === 0.3, 'temp floor 0.3'); + +saveModelProfile(profileOn, path.join(dir, 'profile.json')); +delete process.env.CODEFORGE_MODEL_PROFILE_HOME; +fs.rmSync(tmp, { recursive: true, force: true }); +console.log('profileHysteresis.selftest: ok'); diff --git a/extensions/codeforge/src/agent/profileHysteresis.ts b/extensions/codeforge/src/agent/profileHysteresis.ts new file mode 100644 index 0000000..3bd50e0 --- /dev/null +++ b/extensions/codeforge/src/agent/profileHysteresis.ts @@ -0,0 +1,78 @@ +/** + * Online hysteresis: flip profile knobs at run start from protocol rollup signals. + * Quality knobs (spec-check, orchestrator) are never flipped here — only protocol/cost. + */ + +import { loadModelRollup, leakedParamRate, truncatedToolJsonRate } from './protocolTelemetry'; +import type { ModelProfile } from './modelProfile'; + +const HISTORY_CAP = 40; + +export type HysteresisResult = { + profile: ModelProfile; + changed: boolean; + flips: Array<{ rule: string; from: unknown; to: unknown }>; +}; + +function pushHistory( + profile: ModelProfile, + rule: string, + from: unknown, + to: unknown +): void { + const entry = { at: new Date().toISOString(), rule, from, to }; + profile.history = [...(profile.history ?? []), entry].slice(-HISTORY_CAP); +} + +/** + * Apply online rules over the last 50 runs in the rollup. + * Returns a new profile object when something flips. + */ +export function applyOnlineHysteresis(profile: ModelProfile): HysteresisResult { + if (process.env.CODEFORGE_DISABLE_HYSTERESIS === '1') { + return { profile, changed: false, flips: [] }; + } + const rollup = loadModelRollup(profile.serverId, profile.model); + // Need a meaningful window; off-transition requires the full 50-run hysteresis window. + if (!rollup || rollup.recent.length < 10) { + return { profile, changed: false, flips: [] }; + } + + const next = structuredClone(profile); + const flips: Array<{ rule: string; from: unknown; to: unknown }> = []; + const truncRate = truncatedToolJsonRate(rollup); + const leakRate = leakedParamRate(rollup); + const fullWindow = rollup.recent.length >= 50; + + // Write cap: on when truncations are common; off only after a full clean window. + if (truncRate >= 0.05 && next.mutations.writeMaxContentLines === 0) { + pushHistory(next, 'writeCap.on', 0, 180); + flips.push({ rule: 'writeCap.on', from: 0, to: 180 }); + next.mutations.writeMaxContentLines = 180; + } else if ( + fullWindow && + truncRate < 0.01 && + next.mutations.writeMaxContentLines > 0 + ) { + pushHistory(next, 'writeCap.off', next.mutations.writeMaxContentLines, 0); + flips.push({ + rule: 'writeCap.off', + from: next.mutations.writeMaxContentLines, + to: 0, + }); + next.mutations.writeMaxContentLines = 0; + } + + // Leak rate high → nudge temperature down slightly (tool rounds). + if (leakRate >= 0.08) { + const cur = next.sampling.toolThinking.temperature; + const target = Math.max(0.3, Math.round((cur - 0.1) * 100) / 100); + if (target < cur) { + pushHistory(next, 'toolTemp.down', cur, target); + flips.push({ rule: 'toolTemp.down', from: cur, to: target }); + next.sampling.toolThinking.temperature = target; + } + } + + return { profile: next, changed: flips.length > 0, flips }; +} diff --git a/extensions/codeforge/src/agent/projectStatus.ts b/extensions/codeforge/src/agent/projectStatus.ts index 47794c4..891cc7c 100644 --- a/extensions/codeforge/src/agent/projectStatus.ts +++ b/extensions/codeforge/src/agent/projectStatus.ts @@ -14,7 +14,13 @@ export const HISTORY_MAX = 20; export const DIGEST_MAX = 2000; export type StatusSource = 'model' | 'harness'; -export type StatusOutcome = 'done' | 'blocked' | 'max-steps' | 'cancelled' | 'error'; +export type StatusOutcome = + | 'done' + | 'blocked' + | 'max-steps' + | 'cancelled' + | 'error' + | 'stuck'; export interface StatusRepoSnapshot { head: string; @@ -94,7 +100,16 @@ export function normalizeEntry(input: Partial & { objective?: strin } function normalizeOutcome(o: unknown): StatusOutcome { - if (o === 'blocked' || o === 'max-steps' || o === 'cancelled' || o === 'error' || o === 'done') return o; + if ( + o === 'blocked' || + o === 'max-steps' || + o === 'cancelled' || + o === 'error' || + o === 'done' || + o === 'stuck' + ) { + return o; + } return 'done'; } diff --git a/extensions/codeforge/src/agent/protocolTelemetry.selftest.ts b/extensions/codeforge/src/agent/protocolTelemetry.selftest.ts new file mode 100644 index 0000000..4a9ff19 --- /dev/null +++ b/extensions/codeforge/src/agent/protocolTelemetry.selftest.ts @@ -0,0 +1,65 @@ +/** + * Protocol aggregator + rollup flush. + */ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { + createProtocolAggregator, + flushRunTelemetry, + loadModelRollup, + truncatedToolJsonRate, +} from './protocolTelemetry'; + +function assert(cond: boolean, msg: string): void { + if (!cond) throw new Error(msg); +} + +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'cf-proto-')); +const profileDir = path.join(tmp, 'srv', 'model'); + +const agg = createProtocolAggregator(); +agg.observe({ kind: 'tool', label: 'edit' }); // ignored +agg.observe({ kind: 'protocol', label: 'nativeToolCalls', detail: '3' }); +agg.observe({ kind: 'protocol', label: 'badArgs', detail: '1' }); +agg.observe({ kind: 'protocol', label: 'systemChars', detail: '2900' }); +agg.observe({ + kind: 'protocol', + label: 'sample', + detail: JSON.stringify({ reason: 'truncated_tool_json', tool: 'write' }), +}); + +const snap = agg.snapshot(); +assert(snap.protocol.nativeToolCalls === 3, 'native bump'); +assert(snap.protocol.badArgs === 1, 'badArgs bump'); +assert(snap.protocol.systemChars === 2900, 'systemChars assign'); +assert(snap.sampleCount === 1 && snap.samples[0]?.reason === 'truncated_tool_json', 'sample'); + +const rec = flushRunTelemetry({ + serverId: 'srv', + model: 'model', + source: 'bench', + loopExit: 'done', + elapsedMs: 1000, + aggregator: agg, + profileDir, +}); +assert(rec.sampleReasons[0] === 'truncated_tool_json', 'reasons'); +assert(fs.existsSync(path.join(profileDir, 'runs.jsonl')), 'runs.jsonl'); +const rollup = loadModelRollup('srv', 'model', profileDir); +assert(!!rollup && rollup.runCount === 1, 'rollup count'); +assert(rollup!.sums.nativeToolCalls === 3, 'rollup sums'); +assert(truncatedToolJsonRate(rollup!) === 1, 'trunc rate'); + +flushRunTelemetry({ + serverId: 'srv', + model: 'model', + source: 'chat', + aggregator: createProtocolAggregator(), + profileDir, +}); +const rollup2 = loadModelRollup('srv', 'model', profileDir)!; +assert(rollup2.runCount === 2 && rollup2.recent.length === 2, 'second flush'); + +fs.rmSync(tmp, { recursive: true, force: true }); +console.log('protocolTelemetry.selftest: ok'); diff --git a/extensions/codeforge/src/agent/protocolTelemetry.ts b/extensions/codeforge/src/agent/protocolTelemetry.ts new file mode 100644 index 0000000..1436016 --- /dev/null +++ b/extensions/codeforge/src/agent/protocolTelemetry.ts @@ -0,0 +1,227 @@ +/** + * Protocol telemetry aggregator + per-model rollup persistence. + * Only consumes kind==='protocol' events from the agent loop (structural). + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { modelProfileDir, modelSafeName } from '../storage/paths'; + +export const PROTOCOL_COUNTER_KEYS = [ + 'nativeToolCalls', + 'contentToolCalls', + 'badArgs', + 'unknownTools', + 'lengthExits', + 'emptyReplies', + 'systemChars', + 'toolsChars', +] as const; + +export type ProtocolCounterKey = (typeof PROTOCOL_COUNTER_KEYS)[number]; + +export type ProtocolCounters = Record; + +export type ProtocolSample = { + at: string; + reason?: string; + tool?: string; + [key: string]: unknown; +}; + +export type ProtocolActivity = { + kind?: string; + label?: string; + detail?: string; +}; + +const SAMPLE_CAP = 20; +const ROLLUP_WINDOW = 50; + +export function emptyProtocolCounters(): ProtocolCounters { + return { + nativeToolCalls: 0, + contentToolCalls: 0, + badArgs: 0, + unknownTools: 0, + lengthExits: 0, + emptyReplies: 0, + systemChars: 0, + toolsChars: 0, + }; +} + +export type ProtocolAggregator = { + observe(ev: ProtocolActivity): void; + snapshot(): { protocol: ProtocolCounters; samples: ProtocolSample[]; sampleCount: number }; +}; + +export function createProtocolAggregator(opts?: { sampleCap?: number }): ProtocolAggregator { + const protocol = emptyProtocolCounters(); + const samples: ProtocolSample[] = []; + const sampleCap = opts?.sampleCap ?? SAMPLE_CAP; + + return { + observe(ev: ProtocolActivity): void { + if (ev.kind !== 'protocol') return; + const label = String(ev.label || ''); + if (label === 'sample') { + let parsed: Record = {}; + try { + parsed = JSON.parse(String(ev.detail || '{}')) as Record; + } catch { + parsed = { reason: 'unparsed_sample', preview: String(ev.detail || '').slice(0, 500) }; + } + if (samples.length < sampleCap) { + samples.push({ at: new Date().toISOString(), ...parsed }); + } + return; + } + if (label === 'systemChars' || label === 'toolsChars') { + protocol[label] = Number(ev.detail || 0) || 0; + return; + } + if ((PROTOCOL_COUNTER_KEYS as readonly string[]).includes(label)) { + const key = label as ProtocolCounterKey; + protocol[key] += Number(ev.detail || 1) || 1; + } + }, + snapshot() { + return { + protocol: { ...protocol }, + samples: samples.slice(), + sampleCount: samples.length, + }; + }, + }; +} + +export type FlushRunTelemetryInput = { + serverId: string; + model: string; + source: 'chat' | 'bench'; + sessionId?: string; + loopExit?: string; + elapsedMs?: number; + aggregator: ProtocolAggregator; + /** Override profile root (tests). */ + profileDir?: string; +}; + +export type RunTelemetryRecord = { + at: string; + serverId: string; + model: string; + source: 'chat' | 'bench'; + sessionId?: string; + loopExit?: string; + elapsedMs?: number; + protocol: ProtocolCounters; + sampleCount: number; + sampleReasons: string[]; +}; + +export type ProtocolRollup = { + serverId: string; + model: string; + updatedAt: string; + runCount: number; + sums: ProtocolCounters; + /** Last N runs (newest last). */ + recent: RunTelemetryRecord[]; +}; + +function ensureDirSync(dir: string): void { + fs.mkdirSync(dir, { recursive: true }); +} + +export function flushRunTelemetry(input: FlushRunTelemetryInput): RunTelemetryRecord { + const snap = input.aggregator.snapshot(); + const record: RunTelemetryRecord = { + at: new Date().toISOString(), + serverId: input.serverId, + model: input.model, + source: input.source, + sessionId: input.sessionId, + loopExit: input.loopExit, + elapsedMs: input.elapsedMs, + protocol: snap.protocol, + sampleCount: snap.sampleCount, + sampleReasons: snap.samples + .map(s => String(s.reason || '')) + .filter(Boolean) + .slice(0, SAMPLE_CAP), + }; + + const dir = input.profileDir ?? modelProfileDir(input.serverId, input.model); + ensureDirSync(dir); + const runsPath = path.join(dir, 'runs.jsonl'); + fs.appendFileSync(runsPath, `${JSON.stringify(record)}\n`, 'utf8'); + + const rollupPath = path.join(dir, 'rollup.json'); + const rollup = readRollup(rollupPath) ?? { + serverId: input.serverId, + model: input.model, + updatedAt: record.at, + runCount: 0, + sums: emptyProtocolCounters(), + recent: [], + }; + rollup.runCount += 1; + rollup.updatedAt = record.at; + for (const key of PROTOCOL_COUNTER_KEYS) { + rollup.sums[key] += record.protocol[key]; + } + rollup.recent.push(record); + if (rollup.recent.length > ROLLUP_WINDOW) { + rollup.recent = rollup.recent.slice(-ROLLUP_WINDOW); + } + fs.writeFileSync(rollupPath, JSON.stringify(rollup, null, 2), 'utf8'); + return record; +} + +export function readRollup(rollupPath: string): ProtocolRollup | undefined { + try { + if (!fs.existsSync(rollupPath)) return undefined; + return JSON.parse(fs.readFileSync(rollupPath, 'utf8')) as ProtocolRollup; + } catch { + return undefined; + } +} + +export function loadModelRollup(serverId: string, model: string, profileDir?: string): ProtocolRollup | undefined { + const dir = profileDir ?? modelProfileDir(serverId, model); + return readRollup(path.join(dir, 'rollup.json')); +} + +export function truncatedToolJsonRate(rollup: ProtocolRollup): number { + const runs = rollup.recent; + if (!runs.length) return 0; + let hits = 0; + for (const r of runs) { + if (r.sampleReasons.some(x => x === 'truncated_tool_json')) hits += 1; + } + return hits / runs.length; +} + +export function leakedParamRate(rollup: ProtocolRollup): number { + const runs = rollup.recent; + if (!runs.length) return 0; + let hits = 0; + for (const r of runs) { + if (r.sampleReasons.some(x => /leaked_param/.test(x))) hits += 1; + } + return hits / runs.length; +} + +export function lengthExitRate(rollup: ProtocolRollup): number { + const runs = rollup.recent; + if (!runs.length) return 0; + let hits = 0; + for (const r of runs) { + if ((r.protocol.lengthExits ?? 0) > 0 || r.loopExit === 'length') hits += 1; + } + return hits / runs.length; +} + +export { modelSafeName, ROLLUP_WINDOW }; diff --git a/extensions/codeforge/src/agent/toolCallNormalizer.ts b/extensions/codeforge/src/agent/toolCallNormalizer.ts index 9f439bc..7a991c8 100644 --- a/extensions/codeforge/src/agent/toolCallNormalizer.ts +++ b/extensions/codeforge/src/agent/toolCallNormalizer.ts @@ -88,16 +88,24 @@ function compactSignature(schema: JsonSchemaLike | undefined): string { function applyArgAliases( args: Record, toolName: string, - notes: string[] + notes: string[], + schema?: JsonSchemaLike ): Record { const out: Record = { ...args }; + const props = schema?.properties ?? undefined; for (const [canonical, aliases] of Object.entries(ARG_ALIASES)) { + // Only remap when the tool schema has the canonical name and not the alias + // (avoids browser_type.text → content, references.line → startLine, etc.). + if (props) { + if (!(canonical in props)) continue; + } if (out[canonical] !== undefined && out[canonical] !== null && out[canonical] !== '') { continue; } for (const alias of aliases) { if (out[alias] === undefined || out[alias] === null || out[alias] === '') continue; + if (props && alias in props) continue; out[canonical] = out[alias]; delete out[alias]; notes.push(`(argument ${alias} read as ${canonical})`); @@ -357,8 +365,8 @@ export function normalizeToolCall(input: NormalizeToolCallInput): NormalizeToolC }; } - let args = applyArgAliases({ ...input.args }, resolved.name, notes); const schema = input.toolSchemas?.[resolved.name]; + let args = applyArgAliases({ ...input.args }, resolved.name, notes, schema); const repaired = repairLeakedParams(args, schema, notes); const leakedArgs = repaired.leaked.length ? repaired.leaked : undefined; if (repaired.error) { diff --git a/extensions/codeforge/src/agent/toolContracts.selftest.ts b/extensions/codeforge/src/agent/toolContracts.selftest.ts index 74d880e..af31b86 100644 --- a/extensions/codeforge/src/agent/toolContracts.selftest.ts +++ b/extensions/codeforge/src/agent/toolContracts.selftest.ts @@ -20,6 +20,7 @@ assert(/already applied/i.test(TOOL_SYSTEM_CONTRACT), 'system mentions already a assert(/Browser:|snapshot/i.test(TOOL_SYSTEM_CONTRACT), 'full contract mentions browser'); assert(/UI UNCHANGED|After page state/i.test(toolDescription('browser_click')), 'click contract observe-after'); assert(/Implementation finish|exploration-only|blocked:/i.test(TOOL_SYSTEM_CONTRACT), 'system finish contract'); +assert(/Runnable UI\/apps|launch the app/i.test(TOOL_SYSTEM_CONTRACT), 'runtime verify finish line'); assert(/After diagnosing|bug list/i.test(toolDescription('edit')), 'edit pushes apply after diagnosis'); const coreOnly = buildToolSystemContract(CORE); @@ -44,6 +45,14 @@ assert( /del \(not rm\)/.test(toolDescription('shell')) && /%TEMP% \(not \/tmp\)/.test(toolDescription('shell')), 'shell contract maps Unix habits to cmd.exe' ); +assert( + !/del \(not rm\)/.test(toolDescription('shell', { shellCmdExeHints: false })), + 'shell Unix hints can be disabled via profile' +); +assert( + !/Renaming or removing/.test(buildToolSystemContract(CORE, { renameSearchRule: false })), + 'rename rule can be disabled via profile' +); const withBrowser = buildToolSystemContract([...CORE, 'browser_snapshot', 'browser_click']); assert(/Browser:/i.test(withBrowser), 'browser section when active'); @@ -99,10 +108,12 @@ const expected = [ 'skill', ] as const; -const keys = Object.keys(TOOL_DESCRIPTIONS).sort(); +const keys = Object.keys(TOOL_DESCRIPTIONS) + .filter(k => k !== 'shellUnixHints') + .sort(); assert(keys.length === expected.length, `description count ${keys.length} !== ${expected.length}`); for (const name of expected) { - const d = toolDescription(name); + const d = toolDescription(name as typeof expected[number]); assert(typeof d === 'string' && d.trim().length > 20, `thin description: ${name}`); } assert(/old_string/i.test(toolDescription('edit')), 'edit contract mentions old_string'); diff --git a/extensions/codeforge/src/agent/toolContracts.ts b/extensions/codeforge/src/agent/toolContracts.ts index c57ac46..2785222 100644 --- a/extensions/codeforge/src/agent/toolContracts.ts +++ b/extensions/codeforge/src/agent/toolContracts.ts @@ -3,6 +3,11 @@ * Behavior comes from these schemas/prompts — not from chat-phrase crutches. */ +export type ToolContractPromptKnobs = { + renameSearchRule?: boolean; + shellCmdExeHints?: boolean; +}; + /** @deprecated Prefer buildToolSystemContract(activeNames). Kept for callers that need the full text. */ export const TOOL_SYSTEM_CONTRACT = buildToolSystemContract(undefined); @@ -10,9 +15,14 @@ export const TOOL_SYSTEM_CONTRACT = buildToolSystemContract(undefined); * System tool contract mentioning only tools that are active this run. * Pass undefined/empty to include every optional section (legacy full surface). */ -export function buildToolSystemContract(activeNames?: Iterable): string { +export function buildToolSystemContract( + activeNames?: Iterable, + prompt?: ToolContractPromptKnobs +): string { const active = activeNames ? new Set(activeNames) : undefined; const has = (...names: string[]) => !active || names.some(n => active.has(n)); + const renameRule = prompt?.renameSearchRule !== false; + const shellHints = prompt?.shellCmdExeHints !== false; const lines: string[] = [ '### Tool contract', @@ -86,7 +96,14 @@ export function buildToolSystemContract(activeNames?: Iterable): string lines.push( `Implementation finish: exploration-only (read/search/list/…) is not done. After a short look, call ${effectPhrase} or reply \`blocked: \`. A bug list or plan is not a substitute for applying the change.` ); - if (has('search') && (has('edit') || has('write') || has('apply_patch'))) { + lines.push( + 'Runnable UI/apps: build/test green is not done — launch the app (shell start/dev or browser_navigate) and observe (snapshot/screenshot/After-state or process output) before the final summary. Opening a window that still fails at runtime is not done.' + ); + if ( + renameRule && + has('search') && + (has('edit') || has('write') || has('apply_patch')) + ) { lines.push( 'Renaming or removing an identifier is done only when a search for the old name in the requested scope returns no hits — definitions, imports, call sites and object/export keys (`oldName: x.newName` still exposes oldName). Passing tests do not prove a rename. Keep the old name only if the task asks for an alias.' ); @@ -101,7 +118,7 @@ export function buildToolSystemContract(activeNames?: Iterable): string /** Per-tool OpenAI function descriptions (and key parameter notes). */ export const TOOL_DESCRIPTIONS = { read: - 'Read a file slice (numbered lines). Use before edit so old_string matches disk. path required; startLine/limit optional (default ~120 lines, max 400).', + 'Read a file slice (numbered lines). Use before edit so old_string matches disk. Do not re-read the same span after it is already in tool results — call edit/shell/browser instead. path required; startLine/limit optional (default ~120 lines, max 400).', write: 'Create a new file or replace an entire file body. Prefer a short body: for large new files write a minimal skeleton (<60 lines) then grow with edit; for .NET projects prefer the dotnet tool. For small changes to an existing file use edit instead. After a short diagnosis, call write/edit to apply — do not stop at a bug list. Requires path and full content.', edit: [ @@ -124,10 +141,11 @@ export const TOOL_DESCRIPTIONS = { open: 'Reveal/open a file in the editor UI. Does not modify contents. path required.', shell: [ 'Run a cmd.exe command in the workspace (&& chaining OK; no PowerShell/Unix pipes).', - 'cmd.exe has no Unix tools or heredocs: use del (not rm), type (not cat), findstr (not grep/tail/head), %TEMP% (not /tmp); create files with write, not `cat > file << EOF`; PowerShell cmdlets (Set-Content, Out-File) do not run here.', 'Use for builds/tests/dev servers. Prefer web_search/web_fetch for HTTP and dedicated tools for file/git ops.', 'Long jobs: background=true then await_shell. command required.', ].join(' '), + shellUnixHints: + 'cmd.exe has no Unix tools or heredocs: use del (not rm), type (not cat), findstr (not grep/tail/head), %TEMP% (not /tmp); create files with write, not `cat > file << EOF`; PowerShell cmdlets (Set-Content, Out-File) do not run here.', await_shell: 'Wait/poll a background shell job. id required (from shell background). Optional pattern regex on output; optional block_until_ms.', web_search: @@ -201,8 +219,14 @@ export const TOOL_DESCRIPTIONS = { 'Load a playbook/skill by id or title when you need domain conventions or a workflow. name required. Catalog is injected at runtime.', } as const; -export type ToolDescriptionName = keyof typeof TOOL_DESCRIPTIONS; +export type ToolDescriptionName = Exclude; -export function toolDescription(name: ToolDescriptionName): string { +export function toolDescription( + name: ToolDescriptionName, + prompt?: ToolContractPromptKnobs +): string { + if (name === 'shell' && prompt?.shellCmdExeHints !== false) { + return `${TOOL_DESCRIPTIONS.shell} ${TOOL_DESCRIPTIONS.shellUnixHints}`; + } return TOOL_DESCRIPTIONS[name]; } diff --git a/extensions/codeforge/src/agent/toolHistory.ts b/extensions/codeforge/src/agent/toolHistory.ts index ded20ae..f3877af 100644 --- a/extensions/codeforge/src/agent/toolHistory.ts +++ b/extensions/codeforge/src/agent/toolHistory.ts @@ -149,6 +149,51 @@ export function normalizeToolProtocolHistory(messages: ToolHistoryMessage[]): nu return changes; } +const WIRE_ARG_STRING_MAX = 2000; + +function isJsonObject(raw: string): boolean { + try { + const v = JSON.parse(raw) as unknown; + return !!v && typeof v === 'object' && !Array.isArray(v); + } catch { + return false; + } +} + +/** + * Gateways such as llama-server re-parse every `tool_calls[].function.arguments` in the request + * and reject the whole request (HTTP 500) when one of them is not a JSON object. A truncated + * write kept verbatim in history therefore fails every later round, including retries. + * Valid arguments pass through untouched; invalid ones are repaired (long strings clipped) or + * replaced by a small valid stand-in. + */ +export function toWireToolArguments( + raw: string, + repair?: (raw: string) => Record +): string { + if (isJsonObject(raw)) return raw; + let repaired: Record | undefined; + try { + repaired = repair?.(raw); + } catch { + repaired = undefined; + } + if (repaired && typeof repaired === 'object' && !Array.isArray(repaired)) { + const clipped: Record = {}; + for (const [k, v] of Object.entries(repaired)) { + clipped[k] = + typeof v === 'string' && v.length > WIRE_ARG_STRING_MAX + ? `${v.slice(0, WIRE_ARG_STRING_MAX)}\n…[clipped: showing ${WIRE_ARG_STRING_MAX} of ${v.length} chars of invalid tool arguments]` + : v; + } + return JSON.stringify(clipped); + } + return JSON.stringify({ + invalid_arguments: clipData(raw, 400), + note: `…[clipped: tool arguments were not valid JSON (${raw.length} chars)]`, + }); +} + /** * Mid-compact must not start the kept tail on a lone tool message * (orphan tool_result after digest drops the owning assistant). diff --git a/extensions/codeforge/src/agent/xmlToolCalls.selftest.ts b/extensions/codeforge/src/agent/xmlToolCalls.selftest.ts index d453c83..219a44f 100644 --- a/extensions/codeforge/src/agent/xmlToolCalls.selftest.ts +++ b/extensions/codeforge/src/agent/xmlToolCalls.selftest.ts @@ -54,4 +54,48 @@ assert(sh.length === 1 && sh[0].function.name === 'shell', 'json-in-xml'); assert(parseXmlToolCalls('just prose').length === 0, 'no false positive'); +const leakedClosers = ` + + +src/a.ts + + +foo + + +bar + + +`; +const leaked = parseXmlToolCalls(leakedClosers); +assert(leaked.length === 1 && leaked[0].function.name === 'edit', 'leaked closer edit'); +const larg = JSON.parse(leaked[0].function.arguments) as Record; +assert( + larg.path === 'src/a.ts' && larg.old_string === 'foo' && larg.new_string === 'bar', + `leaked args=${JSON.stringify(larg)}` +); + +const runaway = ` + +x.ts +a +b + + + + +x.ts +a +b + + + + +x.ts +a +b + +`; +assert(parseXmlToolCalls(runaway).length === 1, 'dedupe identical edits'); + console.log('xmlToolCalls.selftest: ok'); diff --git a/extensions/codeforge/src/agent/xmlToolCalls.ts b/extensions/codeforge/src/agent/xmlToolCalls.ts index 7d4a0d6..277a740 100644 --- a/extensions/codeforge/src/agent/xmlToolCalls.ts +++ b/extensions/codeforge/src/agent/xmlToolCalls.ts @@ -36,6 +36,28 @@ function toolCall(name: string, args: Record, index: number): L }; } +/** + * Parse `…` and Qwen leak closers `` + * (e.g. `` instead of ``). + */ +function parseParameterArgs(inner: string): Record { + const args: Record = {}; + // Prefer ; also accept any closer (Qwen often emits + // even for a different parameter name — that still ends the current value). + const paramRe = + /([\s\S]*?)(?:<\/parameter>|<\/[A-Za-z0-9_]+>|(?=([\s\S]*?)(?:<\/parameter>|<\/[A-Za-z0-9_]+>)/gi; + while ((pm = namedRe.exec(inner))) { + args[pm[1]] = coerceParamValue(pm[2] ?? ''); + } + return args; +} + function parseFunctionBlocks(body: string, startIndex: number): LiftedToolCall[] { const out: LiftedToolCall[] = []; const fnRe = @@ -45,20 +67,25 @@ function parseFunctionBlocks(body: string, startIndex: number): LiftedToolCall[] while ((m = fnRe.exec(body))) { const name = m[1]; const inner = m[2] ?? ''; - const args: Record = {}; - const paramRe = - /([\s\S]*?)(?:<\/parameter>|(?=... - const namedRe = - /([\s\S]*?)<\/parameter>/gi; - while ((pm = namedRe.exec(inner))) { - args[pm[1]] = coerceParamValue(pm[2] ?? ''); + out.push(toolCall(name, parseParameterArgs(inner), i++)); + } + return out; +} + +/** Collapse runaway identical tool_calls (same name + arguments) to one. */ +export function dedupeIdenticalToolCalls(calls: LiftedToolCall[]): LiftedToolCall[] { + if (calls.length < 2) return calls; + const out: LiftedToolCall[] = []; + for (const c of calls) { + const prev = out[out.length - 1]; + if ( + prev && + prev.function.name === c.function.name && + prev.function.arguments === c.function.arguments + ) { + continue; } - out.push(toolCall(name, args, i++)); + out.push(c); } return out; } @@ -136,5 +163,5 @@ export function parseXmlToolCalls(text: string): LiftedToolCall[] { } out.push(...parseFunctionBlocks(body, out.length)); } - return out.filter(c => !!c.function.name); + return dedupeIdenticalToolCalls(out.filter(c => !!c.function.name)); } diff --git a/extensions/codeforge/src/bridge/shellPrep.selftest.ts b/extensions/codeforge/src/bridge/shellPrep.selftest.ts new file mode 100644 index 0000000..6171d4d --- /dev/null +++ b/extensions/codeforge/src/bridge/shellPrep.selftest.ts @@ -0,0 +1,38 @@ +/** + * Shell cd-normalization checks (run via node after compile). + */ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { isPathInsideWorkspace, prepareWindowsCommand } from './shellPrep'; + +function assert(cond: boolean, msg: string): void { + if (!cond) throw new Error(msg); +} + +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cf-shell-')); +const sub = path.join(root, 'sub'); +fs.mkdirSync(sub); + +const ok = prepareWindowsCommand('cd sub && echo hi', root); +assert(!('notRun' in ok && ok.notRun), 'existing subdir runs'); +if (!('notRun' in ok && ok.notRun)) { + assert(ok.cwd === sub, 'cwd is subfolder'); +} + +const missing = prepareWindowsCommand('cd nowhere && echo hi', root); +assert('notRun' in missing && missing.notRun === true, 'missing cd target does not run'); +assert( + 'message' in missing && String(missing.message).includes('does not exist'), + 'missing cd message' +); + +const outside = prepareWindowsCommand('cd .. && echo hi', root); +assert('notRun' in outside && outside.notRun === true, 'outside workspace cd does not run'); + +assert(isPathInsideWorkspace(root, sub), 'sub inside workspace'); +assert(!isPathInsideWorkspace(root, path.dirname(root)), 'parent outside workspace'); + +fs.rmSync(root, { recursive: true, force: true }); + +console.log('shellPrep ok'); diff --git a/extensions/codeforge/src/bridge/shellPrep.ts b/extensions/codeforge/src/bridge/shellPrep.ts new file mode 100644 index 0000000..6756a02 --- /dev/null +++ b/extensions/codeforge/src/bridge/shellPrep.ts @@ -0,0 +1,71 @@ +/** + * Normalize Windows shell commands (cd X && rest) before sandbox execution. + */ + +import * as fs from 'fs'; +import * as path from 'path'; + +export function isPathInsideWorkspace(workspaceRoot: string, absPath: string): boolean { + const root = path.resolve(workspaceRoot); + const target = path.resolve(absPath); + const rel = path.relative(root, target); + if (!rel || rel === '') return true; + return !rel.startsWith('..') && !path.isAbsolute(rel); +} + +export type PreparedShellCommand = + | { + notRun: true; + message: string; + } + | { + notRun?: false; + command: string; + cwd: string; + display: string; + note?: string; + }; + +/** + * Prefer running under the right cwd instead of `cd path && …`. + * Keeps && for cmd.exe chaining on Windows. + */ +export function prepareWindowsCommand(command: string, workspaceRoot: string): PreparedShellCommand { + const pipeStripped = command + .replace(/\s*\|\s*(tail|head|grep)\b[^|&;]*/gi, '') + .trim(); + const pipeNote = + pipeStripped !== command.trim() + ? 'Removed Unix pipe (tail/head/grep) — shell is cmd.exe.' + : undefined; + const trimmed = pipeStripped; + const cdMatch = /^(?:cd(?:\s+\/d)?)\s+("([^"]+)"|([^\s&;]+))\s*(?:&&|;)\s*([\s\S]+)$/i.exec( + trimmed + ); + if (cdMatch) { + const target = (cdMatch[2] || cdMatch[3] || '').trim(); + const rest = (cdMatch[4] || '').trim(); + if (target && rest) { + const abs = path.isAbsolute(target) ? target : path.resolve(workspaceRoot, target); + if (!fs.existsSync(abs) || !isPathInsideWorkspace(workspaceRoot, abs)) { + return { + notRun: true, + message: `Not run: directory ${target} does not exist`, + }; + } + return { + command: rest, + cwd: abs, + display: rest, + note: [pipeNote, `normalized: cwd → ${abs}`].filter(Boolean).join(' | '), + }; + } + } + + return { + command: trimmed, + cwd: workspaceRoot, + display: trimmed, + note: pipeNote, + }; +} diff --git a/extensions/codeforge/src/bridge/vscodeBridge.ts b/extensions/codeforge/src/bridge/vscodeBridge.ts index 29416b0..c35e4f6 100644 --- a/extensions/codeforge/src/bridge/vscodeBridge.ts +++ b/extensions/codeforge/src/bridge/vscodeBridge.ts @@ -8,7 +8,10 @@ import * as vscode from 'vscode'; import * as path from 'path'; import * as fs from 'fs'; -import { resolveSandboxLevel, runSandboxed, type SandboxLevel } from '../sandbox/runtime'; +import { resolveSandboxLevel, runSandboxed } from '../sandbox/runtime'; +import { killProcessTree } from '../sandbox/processKill'; +import { prepareWindowsCommand } from './shellPrep'; +import { assertWorkspaceWriteAllowed } from './workspaceWriteGate'; import { clipBuildOutput, clipData, looksLikeBuildTestOutput } from '../agent/clip'; import { applyEdit, excerptAround } from '../agent/editTool'; import { applyPatchHunk, parsePatch } from '../agent/applyPatch'; @@ -129,11 +132,7 @@ export class VSCodeAIBridge { case 'diagnostics': return this.getDiagnostics(args.path ? String(args.path) : undefined); case 'shell': - return this.runInTerminal( - String(args.command), - toolCall.abortSignal, - args.sandbox ? (String(args.sandbox) as SandboxLevel) : undefined - ); + return this.runInTerminal(String(args.command), toolCall.abortSignal); default: throw new Error(`Unknown bridge tool: ${toolCall.name}`); } @@ -286,6 +285,7 @@ export class VSCodeAIBridge { } private async writeFile(filePath: string, content: string): Promise { + await assertWorkspaceWriteAllowed('write', [filePath], this.getWorkspaceRoot()); try { await getEditJournal().recordBeforeWrite(filePath, p => this.readRaw(p)); } catch { @@ -326,6 +326,7 @@ export class VSCodeAIBridge { newString: string, replaceAll: boolean ): Promise { + await assertWorkspaceWriteAllowed('edit', [filePath], this.getWorkspaceRoot()); try { await getEditJournal().recordBeforeWrite(filePath, p => this.readRaw(p)); } catch { @@ -365,6 +366,10 @@ export class VSCodeAIBridge { if (!parsed.ok) { throw new Error(`apply_patch: ${parsed.error}`); } + const patchPaths = parsed.hunks.flatMap(h => + h.type === 'update' ? [h.path, h.movePath].filter(Boolean) as string[] : [h.path] + ); + await assertWorkspaceWriteAllowed('apply_patch', patchPaths, this.getWorkspaceRoot()); const applied: string[] = []; const diagPaths: string[] = []; for (const hunk of parsed.hunks) { @@ -597,6 +602,7 @@ export class VSCodeAIBridge { } private async deleteFile(filePath: string): Promise { + await assertWorkspaceWriteAllowed('delete', [filePath], this.getWorkspaceRoot()); try { await getEditJournal().recordBeforeWrite(filePath, p => this.readRaw(p), { deleting: true }); } catch { @@ -608,6 +614,7 @@ export class VSCodeAIBridge { } private async renameFile(oldPath: string, newPath: string): Promise { + await assertWorkspaceWriteAllowed('rename', [oldPath, newPath], this.getWorkspaceRoot()); try { await getEditJournal().recordBeforeWrite(oldPath, p => this.readRaw(p), { deleting: true }); await getEditJournal().recordBeforeWrite(newPath, p => this.readRaw(p)); @@ -764,11 +771,7 @@ export class VSCodeAIBridge { close: () => { this.agentPty = undefined; for (const child of this.backgroundChildren) { - try { - child.kill(); - } catch { - /* ignore */ - } + killProcessTree(child); } this.backgroundChildren = []; }, @@ -799,19 +802,11 @@ export class VSCodeAIBridge { /** Kill foreground + background agent shell processes (Stop). */ abortRunningShells(): void { for (const child of this.backgroundChildren) { - try { - child.kill(); - } catch { - /* ignore */ - } + killProcessTree(child); } this.backgroundChildren = []; if (this.activeShellChild) { - try { - this.activeShellChild.kill(); - } catch { - /* ignore */ - } + killProcessTree(this.activeShellChild); this.activeShellChild = undefined; } try { @@ -821,11 +816,7 @@ export class VSCodeAIBridge { } } - private async runInTerminal( - command: string, - abortSignal?: AbortSignal, - sandboxHint?: SandboxLevel - ): Promise { + private async runInTerminal(command: string, abortSignal?: AbortSignal): Promise { const cwd = this.getWorkspaceRoot(); if (!cwd) { throw new Error('No workspace folder open — open a folder before running shell commands'); @@ -836,10 +827,12 @@ export class VSCodeAIBridge { } const prepared = prepareWindowsCommand(command, cwd); + if ('notRun' in prepared && prepared.notRun) { + this.output.appendLine(`[shell] ${prepared.message}`); + return prepared.message; + } const cfg = vscode.workspace.getConfiguration('codeforge.ai'); const level = resolveSandboxLevel(prepared.command, { - requested: sandboxHint, - risk: 'medium', configured: cfg.get('shellSandbox'), fullFreedom: cfg.get('fullAgentFreedom') === true, }); @@ -909,7 +902,7 @@ export class VSCodeAIBridge { } abortSignal?.removeEventListener('abort', onAbort); const raw = chunks.join(''); - const exit = opts?.aborted ? 130 : code ?? 0; + const exit = opts?.aborted ? 130 : code ?? 1; if (opts?.aborted) { mirror.write('\x1b[33m[aborted by user]\x1b[0m\r\n'); } else if (!opts?.background) { @@ -943,11 +936,7 @@ export class VSCodeAIBridge { this.activeShellChild = child; const onAbort = () => { - try { - child.kill(); - } catch { - /* ignore */ - } + killProcessTree(child); finish(130, { aborted: true }); }; if (abortSignal) { @@ -985,7 +974,7 @@ export class VSCodeAIBridge { if (!settled) { finish(code); } else { - mirror.write(`\x1b[90m[background exit ${code ?? 0}]\x1b[0m\r\n`); + mirror.write(`\x1b[90m[background exit ${code ?? 1}]\x1b[0m\r\n`); } }); @@ -1016,60 +1005,3 @@ export function summarizeShellOutput(raw: string, maxChars = 8000): string { return looksLikeBuildTestOutput(text) ? clipBuildOutput(text, maxChars) : clipData(text, maxChars); } -/** - * Prefer running under the right cwd instead of `cd path && …`. - * Keeps && for cmd.exe chaining on Windows. - */ -function prepareWindowsCommand( - command: string, - workspaceRoot: string -): { command: string; cwd: string; display: string; note?: string } { - // Strip Unix pipes that fail on cmd.exe before any normalization. - const pipeStripped = command - .replace(/\s*\|\s*(tail|head|grep)\b[^|&;]*/gi, '') - .trim(); - const pipeNote = - pipeStripped !== command.trim() - ? 'Removed Unix pipe (tail/head/grep) — shell is cmd.exe.' - : undefined; - const trimmed = pipeStripped; - // cd X && rest | cd /d X && rest | cd X; rest - const cdMatch = /^(?:cd(?:\s+\/d)?)\s+("([^"]+)"|([^\s&;]+))\s*(?:&&|;)\s*([\s\S]+)$/i.exec( - trimmed - ); - if (cdMatch) { - const target = (cdMatch[2] || cdMatch[3] || '').trim(); - const rest = (cdMatch[4] || '').trim(); - if (target && rest) { - const abs = path.isAbsolute(target) - ? target - : path.resolve(workspaceRoot, target); - if (fs.existsSync(abs)) { - return { - command: rest, - cwd: abs, - display: rest, - note: [pipeNote, `normalized: cwd → ${abs}`].filter(Boolean).join(' | '), - }; - } - return { - command: rest, - cwd: workspaceRoot, - display: rest, - note: [ - pipeNote, - `cwd "${target}" does not exist — using workspace root`, - ] - .filter(Boolean) - .join(' | '), - }; - } - } - - return { - command: trimmed, - cwd: workspaceRoot, - display: trimmed, - note: pipeNote, - }; -} diff --git a/extensions/codeforge/src/bridge/workspaceWriteGate.ts b/extensions/codeforge/src/bridge/workspaceWriteGate.ts new file mode 100644 index 0000000..84f9f7e --- /dev/null +++ b/extensions/codeforge/src/bridge/workspaceWriteGate.ts @@ -0,0 +1,83 @@ +/** + * Gate write/edit/delete/rename/apply_patch targets outside the workspace folder. + */ + +import * as path from 'path'; +import * as vscode from 'vscode'; +import { isPathInsideWorkspace } from './shellPrep'; + +export type OutsideWorkspaceMode = 'ask' | 'allow' | 'deny'; + +const WRITE_TOOLS = new Set(['write', 'edit', 'apply_patch', 'delete', 'rename']); + +export function isWriteTool(name: string): boolean { + return WRITE_TOOLS.has(name.trim().toLowerCase()); +} + +export function resolveWorkspaceAbsolute(workspaceRoot: string | undefined, filePath: string): string { + const p = String(filePath || '').trim(); + if (!p) return ''; + if (path.isAbsolute(p)) return path.resolve(p); + if (!workspaceRoot) return path.resolve(p); + return path.resolve(workspaceRoot, p); +} + +export function pathsOutsideWorkspace( + workspaceRoot: string | undefined, + paths: string[] +): string[] { + if (!workspaceRoot) return []; + const out: string[] = []; + for (const p of paths) { + const abs = resolveWorkspaceAbsolute(workspaceRoot, p); + if (!abs) continue; + if (!isPathInsideWorkspace(workspaceRoot, abs)) { + out.push(p); + } + } + return out; +} + +export function readOutsideWorkspaceMode(): OutsideWorkspaceMode { + const raw = vscode.workspace.getConfiguration('codeforge.ai').get('outsideWorkspaceWrites', 'ask'); + return raw === 'allow' || raw === 'deny' ? raw : 'ask'; +} + +export function allowAllApprovalsEnabled(): boolean { + const cfg = vscode.workspace.getConfiguration('codeforge.ai'); + if (cfg.get('allowAll') === true) return true; + if (cfg.get('fullAgentFreedom') === true) return true; + return false; +} + +export async function assertWorkspaceWriteAllowed( + tool: string, + paths: string[], + workspaceRoot: string | undefined +): Promise { + if (!isWriteTool(tool)) return; + const outside = pathsOutsideWorkspace(workspaceRoot, paths); + if (!outside.length) return; + + if (allowAllApprovalsEnabled()) { + return; + } + + const mode = readOutsideWorkspaceMode(); + if (mode === 'allow') return; + if (mode === 'deny') { + throw new Error( + `Outside workspace writes are denied (${outside.join(', ')}). Change codeforge.ai.outsideWorkspaceWrites or use paths under the workspace root.` + ); + } + + const picked = await vscode.window.showWarningMessage( + `Allow ${tool} outside the workspace?\n${outside.join('\n')}`, + { modal: true }, + 'Allow once', + 'Deny' + ); + if (picked !== 'Allow once') { + throw new Error(`Denied: ${tool} outside workspace (${outside.join(', ')})`); + } +} diff --git a/extensions/codeforge/src/extension.ts b/extensions/codeforge/src/extension.ts index 489f453..e49bcbb 100644 --- a/extensions/codeforge/src/extension.ts +++ b/extensions/codeforge/src/extension.ts @@ -17,6 +17,7 @@ import { TabEngine } from './tab/tabEngine'; import { AiSettingsStore } from './settings/aiSettingsStore'; import { AiSettingsPanel } from './settings/aiSettingsPanel'; import { showModelQuickPick } from './settings/modelQuickPick'; +import { promptAndRunCalibration, promptAndRunHarnessRevision } from './settings/modelProfileCalibrate'; import { initApprovalPolicy, getApprovalPolicy } from './policy/approvalPolicy'; import { initEditJournal, OriginalContentProvider } from './agent/editJournal'; import { SessionStore } from './sessions/sessionStore'; @@ -64,6 +65,7 @@ export async function activate(context: vscode.ExtensionContext): Promise outputChannel.appendLine(line)); sessionStore = new SessionStore(context); initApprovalPolicy(context); @@ -226,6 +228,15 @@ export async function activate(context: vscode.ExtensionContext): Promise { + await getApprovalPolicy().revokeAllPermissions(); + void vscode.window.showInformationMessage( + 'CodeForge: revoked Allow all, auto-approve tools, and auto-approve edits.' + ); + }) + ); + context.subscriptions.push( vscode.commands.registerCommand('codeforge.toggleAutoApproveEdits', async () => { const cfg = vscode.workspace.getConfiguration('codeforge.ai'); @@ -264,8 +275,10 @@ export async function activate(context: vscode.ExtensionContext): Promise { - const allowed = new Set(['models', 'mcp', 'guardrails', 'agent']); - const initial = allowed.has(String(tab)) ? (tab as 'models' | 'mcp' | 'guardrails' | 'agent') : 'models'; + const allowed = new Set(['models', 'mcp', 'guardrails', 'agent', 'benchmark']); + const initial = allowed.has(String(tab)) + ? (tab as 'models' | 'mcp' | 'guardrails' | 'agent' | 'benchmark') + : 'models'; AiSettingsPanel.show(context, settingsStore, governance, initial); }) ); @@ -283,6 +296,18 @@ export async function activate(context: vscode.ExtensionContext): Promise { + await promptAndRunCalibration(settingsStore, context.extensionPath, outputChannel); + }) + ); + + context.subscriptions.push( + vscode.commands.registerCommand('codeforge.reviseHarness', async () => { + await promptAndRunHarnessRevision(settingsStore, context.extensionPath, outputChannel); + }) + ); + context.subscriptions.push( vscode.commands.registerCommand('codeforge.resumeSession', async (sessionId: string) => { if (!sessionId) return; diff --git a/extensions/codeforge/src/hooks/hooksRunner.selftest.ts b/extensions/codeforge/src/hooks/hooksRunner.selftest.ts new file mode 100644 index 0000000..06aa436 --- /dev/null +++ b/extensions/codeforge/src/hooks/hooksRunner.selftest.ts @@ -0,0 +1,11 @@ +import { quoteShellArg } from './hooksShellQuote'; + +function assert(cond: boolean, msg: string): void { + if (!cond) throw new Error(msg); +} + +assert(quoteShellArg('plain') === 'plain', 'plain arg unchanged'); +assert(quoteShellArg('has space').includes('"'), 'spaces quoted on win'); +assert(quoteShellArg('a&b').includes('"'), 'shell metachar quoted'); + +console.log('hooksRunner.selftest: ok'); diff --git a/extensions/codeforge/src/hooks/hooksRunner.ts b/extensions/codeforge/src/hooks/hooksRunner.ts index 9c986c4..25e0ccd 100644 --- a/extensions/codeforge/src/hooks/hooksRunner.ts +++ b/extensions/codeforge/src/hooks/hooksRunner.ts @@ -11,6 +11,8 @@ import { spawn } from 'child_process'; import * as fs from 'fs/promises'; +import { killProcessTree } from '../sandbox/processKill'; +import { quoteShellArg } from './hooksShellQuote'; import * as os from 'os'; import * as path from 'path'; import * as vscode from 'vscode'; @@ -115,6 +117,18 @@ async function runCommandHook( a.replace(/^\.\//, path.join(cwd, '.CodeForge') + path.sep).replace(/\\/g, path.sep) ); + const scriptPath = resolveHookScriptPath(hook.command, args, cwd); + if (scriptPath) { + try { + await fs.access(scriptPath); + } catch { + log?.( + `[hooks] missing hook script (${payload.event}): ${scriptPath} — fail-open` + ); + return { permission: 'allow' }; + } + } + return new Promise(resolve => { let settled = false; const finish = (r: HookResult) => { @@ -125,7 +139,9 @@ async function runCommandHook( let child; try { - child = spawn(hook.command, args, { + const quotedArgs = args.map(quoteShellArg); + const cmdline = [hook.command, ...quotedArgs].join(' '); + child = spawn(cmdline, [], { cwd, shell: true, windowsHide: true, @@ -149,11 +165,7 @@ async function runCommandHook( }); const timer = setTimeout(() => { - try { - child.kill(); - } catch { - /* ignore */ - } + killProcessTree(child); log?.(`[hooks] timeout ${payload.event} (${timeoutMs}ms) — fail-open`); finish({ permission: 'allow' }); }, timeoutMs); @@ -204,6 +216,18 @@ async function runCommandHook( }); } +function resolveHookScriptPath(command: string, args: string[], cwd: string): string | undefined { + const scriptExt = /\.(js|mjs|cjs|ts|ps1|cmd|bat|sh)$/i; + if (args[0]) { + const p = path.isAbsolute(args[0]) ? args[0] : path.join(cwd, args[0]); + if (scriptExt.test(p) || args[0].includes(path.sep)) return p; + } + if (scriptExt.test(command) || command.includes(path.sep)) { + return path.isAbsolute(command) ? command : path.join(cwd, command); + } + return undefined; +} + function parseHookStdout(stdout: string): Record | undefined { const text = stdout.trim(); if (!text) return undefined; diff --git a/extensions/codeforge/src/hooks/hooksShellQuote.ts b/extensions/codeforge/src/hooks/hooksShellQuote.ts new file mode 100644 index 0000000..95f8af9 --- /dev/null +++ b/extensions/codeforge/src/hooks/hooksShellQuote.ts @@ -0,0 +1,8 @@ +/** Quote one argument for shell:true spawn (Windows-safe). */ +export function quoteShellArg(arg: string): string { + if (!/[\s"&|<>^()]/.test(arg)) return arg; + if (process.platform === 'win32') { + return `"${arg.replace(/"/g, '""')}"`; + } + return `'${arg.replace(/'/g, `'\\''`)}'`; +} diff --git a/extensions/codeforge/src/mcp/mcpClient.selftest.ts b/extensions/codeforge/src/mcp/mcpClient.selftest.ts new file mode 100644 index 0000000..a03570d --- /dev/null +++ b/extensions/codeforge/src/mcp/mcpClient.selftest.ts @@ -0,0 +1,17 @@ +import { formatMcpResult } from './mcpResultFormat'; + +function assert(cond: boolean, msg: string): void { + if (!cond) throw new Error(msg); +} + +assert(formatMcpResult({ isError: true, content: [{ type: 'text', text: 'bad input' }] }).startsWith('Error:'), 'isError prefixes Error:'); +assert( + formatMcpResult({ content: [{ type: 'error', text: 'denied' }] }).startsWith('Error:'), + 'error content type prefixes Error:' +); +const big = 'x'.repeat(30_000); +const clipped = formatMcpResult({ content: [{ text: big }] }, 1000); +assert(clipped.length <= 1100, 'size-limits MCP result'); +assert(clipped.includes('clipped') || clipped.length < big.length, 'clip marker or shorter'); + +console.log('mcpClient.selftest: ok'); diff --git a/extensions/codeforge/src/mcp/mcpClient.ts b/extensions/codeforge/src/mcp/mcpClient.ts index 4681064..a0356ea 100644 --- a/extensions/codeforge/src/mcp/mcpClient.ts +++ b/extensions/codeforge/src/mcp/mcpClient.ts @@ -11,6 +11,9 @@ import { spawn, execFileSync, ChildProcessWithoutNullStreams } from 'child_proce import { accessSync, constants } from 'fs'; import { McpServerConfig } from '../settings/aiSettingsStore'; import { parseEnvFile } from './mcpJsonFile'; +import { formatMcpResult } from './mcpResultFormat'; + +const MCP_HTTP_TIMEOUT_MS = 12_000; export interface McpToolDef { serverId: string; @@ -36,7 +39,11 @@ export class McpClientManager { private nextId = 1; private pending = new Map< number, - { resolve: (v: unknown) => void; reject: (e: Error) => void } + { + serverId: string; + resolve: (v: unknown) => void; + reject: (e: Error) => void; + } >(); private buffers = new Map(); private tools: McpToolDef[] = []; @@ -288,11 +295,24 @@ export class McpClientManager { method: 'tools/call', params: { name, arguments: args }, }; - const res = await fetch(url, { - method: 'POST', - headers: this.httpHeaders(server), - body: JSON.stringify(body), - }); + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), MCP_HTTP_TIMEOUT_MS); + let res: Response; + try { + res = await fetch(url, { + method: 'POST', + headers: this.httpHeaders(server), + body: JSON.stringify(body), + signal: ctrl.signal, + }); + } catch (err) { + if (ctrl.signal.aborted) { + throw new Error(`MCP HTTP timeout calling ${server.name}`); + } + throw err; + } finally { + clearTimeout(timer); + } if (!res.ok) { throw new Error(`MCP HTTP ${res.status}: ${await res.text()}`); } @@ -362,8 +382,13 @@ export class McpClientManager { }); proc.on('exit', code => { this.output.appendLine(`[mcp:${server.name}] exited ${code}`); + if (this.processes.get(server.id) !== proc) { + return; + } this.processes.delete(server.id); this.framing.delete(server.id); + this.buffers.delete(server.id); + this.rejectPendingForServer(server.id, `MCP server ${server.name} exited (${code ?? '?'})`); }); if (readyPromise) { @@ -413,7 +438,7 @@ export class McpClientManager { params: {}, }; const ctrl = new AbortController(); - const timer = setTimeout(() => ctrl.abort(), 12_000); + const timer = setTimeout(() => ctrl.abort(), MCP_HTTP_TIMEOUT_MS); let res: Response; try { res = await fetch(url, { @@ -519,6 +544,15 @@ export class McpClientManager { } } + private rejectPendingForServer(serverId: string, message: string): void { + const err = new Error(message); + for (const [id, p] of this.pending) { + if (p.serverId !== serverId) continue; + this.pending.delete(id); + p.reject(err); + } + } + private handleMessage(raw: string): void { try { const msg = JSON.parse(raw) as JsonRpcResponse; @@ -560,10 +594,11 @@ export class McpClientManager { resolve({}); return; } - this.pending.set(id, { resolve, reject }); + this.pending.set(id, { serverId, resolve, reject }); proc.stdin.write(frame); setTimeout(() => { - if (this.pending.has(id)) { + const p = this.pending.get(id); + if (p) { this.pending.delete(id); reject(new Error(`MCP timeout: ${method}`)); } @@ -675,16 +710,6 @@ function sanitizeName(name: string): string { return name.replace(/[^a-zA-Z0-9_-]/g, '_'); } -function formatMcpResult(result: unknown): string { - if (result == null) return '(empty)'; - if (typeof result === 'string') return result; - const r = result as { content?: Array<{ type?: string; text?: string }> }; - if (Array.isArray(r.content)) { - return r.content.map(c => c.text ?? JSON.stringify(c)).join('\n'); - } - return JSON.stringify(result, null, 2); -} - let sharedMcp: McpClientManager | undefined; export function initMcp(output: vscode.OutputChannel): McpClientManager { diff --git a/extensions/codeforge/src/mcp/mcpResultFormat.ts b/extensions/codeforge/src/mcp/mcpResultFormat.ts new file mode 100644 index 0000000..3e051e3 --- /dev/null +++ b/extensions/codeforge/src/mcp/mcpResultFormat.ts @@ -0,0 +1,25 @@ +import { clipData } from '../agent/clip'; + +export const MCP_RESULT_MAX_CHARS = 24_000; + +export function formatMcpResult(result: unknown, maxChars = MCP_RESULT_MAX_CHARS): string { + if (result == null) return '(empty)'; + const r = result as { + isError?: boolean; + content?: Array<{ type?: string; text?: string; isError?: boolean }>; + }; + let text: string; + if (typeof result === 'string') { + text = result; + } else if (Array.isArray(r.content)) { + text = r.content.map(c => c.text ?? JSON.stringify(c)).join('\n'); + } else { + text = JSON.stringify(result, null, 2); + } + const itemError = + r.isError === true || r.content?.some(c => c.isError === true || c.type === 'error'); + if (itemError && !/^error:/i.test(text.trim())) { + text = `Error: ${text}`; + } + return clipData(text, maxChars); +} diff --git a/extensions/codeforge/src/memory/extendedMemory.selftest.ts b/extensions/codeforge/src/memory/extendedMemory.selftest.ts new file mode 100644 index 0000000..54734cc --- /dev/null +++ b/extensions/codeforge/src/memory/extendedMemory.selftest.ts @@ -0,0 +1,17 @@ +import { isRedTestFailure, lessonFromFailure } from './extendedMemory'; + +function assert(cond: boolean, msg: string): void { + if (!cond) throw new Error(msg); +} + +assert( + isRedTestFailure('dotnet test', 'Failed: 2 tests\nTotal tests: 5'), + 'detects dotnet test failure' +); +assert(!isRedTestFailure('dotnet build', 'Build succeeded.'), 'build is not red test'); +assert( + lessonFromFailure('echo hi', 'not recognized').advice.includes('Do not run it again'), + 'non-test failures still produce do-not-repeat advice' +); + +console.log('extendedMemory.selftest: ok'); diff --git a/extensions/codeforge/src/memory/extendedMemory.ts b/extensions/codeforge/src/memory/extendedMemory.ts index 21e27bf..7be66e5 100644 --- a/extensions/codeforge/src/memory/extendedMemory.ts +++ b/extensions/codeforge/src/memory/extendedMemory.ts @@ -146,6 +146,23 @@ export interface ShellLessonResult { * Learn from a failed command. The lesson id comes from the error text, so the same * failure reinforces one record. Previously saved lessons match by their stored regex. */ +/** Failing test runs should not produce a "do not run again" lesson. */ +export function isRedTestFailure(command: string, output: string): boolean { + const cmd = command.trim().toLowerCase(); + if ( + !/\b(dotnet\s+test|npm\s+test|npm\s+run\s+test|yarn\s+test|pnpm\s+test|jest\b|pytest|cargo\s+test|go\s+test|vitest)\b/.test( + cmd + ) + ) { + return false; + } + const blob = `${command}\n${output}`; + return ( + /\bfailed\b|\bfailure\b|\berror\b|\bFAIL\b|×|✕|not passed/i.test(blob) || + /\b0 passed\b.*\bfailed\b/i.test(blob) + ); +} + export async function learnFromShellFailure( workspaceFolder: string | undefined, command: string, @@ -154,6 +171,7 @@ export async function learnFromShellFailure( const cmd = command.trim(); const out = output || ''; if (!cmd && !out.trim()) return null; + if (isRedTestFailure(cmd, out)) return null; const derived = lessonFromFailure(cmd, out); const known = (await load(workspaceFolder)).lessons.find( diff --git a/extensions/codeforge/src/policy/approvalPolicy.selftest.ts b/extensions/codeforge/src/policy/approvalPolicy.selftest.ts new file mode 100644 index 0000000..4e31420 --- /dev/null +++ b/extensions/codeforge/src/policy/approvalPolicy.selftest.ts @@ -0,0 +1,18 @@ +/** + * Allow-all policy helpers (no VS Code — run via node after compile). + */ +function assert(cond: boolean, msg: string): void { + if (!cond) throw new Error(msg); +} + +function globalAllowAll(cfg: { allowAll?: boolean; fullAgentFreedom?: boolean }): boolean { + return cfg.allowAll === true || cfg.fullAgentFreedom === true; +} + +assert(!globalAllowAll({}), 'default off'); +assert(globalAllowAll({ allowAll: true }), 'allowAll setting'); +assert(globalAllowAll({ fullAgentFreedom: true }), 'full freedom implies allowAll'); +const editTools = new Set(['edit', 'write', 'apply_patch', 'rename', 'delete']); +assert(editTools.has('write') && editTools.has('apply_patch'), 'edit tools set'); + +console.log('approvalPolicy ok'); diff --git a/extensions/codeforge/src/policy/approvalPolicy.ts b/extensions/codeforge/src/policy/approvalPolicy.ts index ca9a497..11145a7 100644 --- a/extensions/codeforge/src/policy/approvalPolicy.ts +++ b/extensions/codeforge/src/policy/approvalPolicy.ts @@ -7,6 +7,7 @@ import * as vscode from 'vscode'; const GLOBAL_KEY = 'codeforge.ai.approvals.always'; const EDITS_ALWAYS_KEY = 'codeforge.ai.approvals.editsAlways'; +const MIGRATED_KEY = 'codeforge.ai.approvals.migratedToSettings'; /** File-modification tools that share "Allow all edits". */ export const EDIT_TOOLS = new Set(['edit', 'write', 'apply_patch', 'rename', 'delete']); @@ -16,31 +17,37 @@ export type PermissionLevel = 'default' | 'assisted' | 'allowAll'; export class ApprovalPolicy { private readonly sessionAllowed = new Set(); private sessionAllowAllEdits = false; - private sessionAllowAll = false; constructor(private readonly context: vscode.ExtensionContext) {} + private cfg() { + return vscode.workspace.getConfiguration('codeforge.ai'); + } + + globalAllowAllEnabled(): boolean { + if (this.cfg().get('allowAll') === true) return true; + if (this.cfg().get('fullAgentFreedom') === true) return true; + return false; + } + private alwaysAllowed(): Set { - const fromState = this.context.globalState.get(GLOBAL_KEY) ?? []; - const fromConfig = - vscode.workspace.getConfiguration('codeforge.ai').get('autoApprove') ?? []; - return new Set([...fromState, ...fromConfig].map(normalizeTool)); + const fromConfig = this.cfg().get('autoApprove') ?? []; + return new Set(fromConfig.map(normalizeTool)); } /** Setting or persisted "always allow all file edits". */ autoApproveEditsEnabled(): boolean { - const cfg = vscode.workspace.getConfiguration('codeforge.ai'); - if (cfg.get('autoApproveEdits') === true) { + if (this.cfg().get('autoApproveEdits') === true) { return true; } - return this.context.globalState.get(EDITS_ALWAYS_KEY) === true; + return false; } isAutoApproved(tool: string): boolean { - const key = normalizeTool(tool); - if (this.sessionAllowAll) { + if (this.globalAllowAllEnabled()) { return true; } + const key = normalizeTool(tool); if (EDIT_TOOLS.has(key) && (this.sessionAllowAllEdits || this.autoApproveEditsEnabled())) { return true; } @@ -58,16 +65,22 @@ export class ApprovalPolicy { } this.sessionAllowed.add(key); - const next = Array.from(new Set([...this.alwaysAllowed(), key])); - await this.context.globalState.update(GLOBAL_KEY, next); - - const cfg = vscode.workspace.getConfiguration('codeforge.ai'); + const cfg = this.cfg(); const current = cfg.get('autoApprove') ?? []; if (!current.map(normalizeTool).includes(key)) { await cfg.update('autoApprove', [...current, key], vscode.ConfigurationTarget.Global); } } + async setGlobalAllowAll(enabled: boolean): Promise { + await this.cfg().update('allowAll', enabled, vscode.ConfigurationTarget.Global); + if (enabled) { + await this.allowAutoSession(); + } else { + this.clearSession(); + } + } + /** Cursor-like: allow all file edits for session or forever. */ async allowAllEdits(scope: 'session' | 'always'): Promise { this.sessionAllowAllEdits = true; @@ -75,15 +88,9 @@ export class ApprovalPolicy { this.sessionAllowed.add(t); } if (scope === 'always') { - await this.context.globalState.update(EDITS_ALWAYS_KEY, true); - await vscode.workspace - .getConfiguration('codeforge.ai') - .update('autoApproveEdits', true, vscode.ConfigurationTarget.Global); + await this.cfg().update('autoApproveEdits', true, vscode.ConfigurationTarget.Global); const next = Array.from(new Set([...this.alwaysAllowed(), ...EDIT_TOOLS])); - await this.context.globalState.update(GLOBAL_KEY, Array.from(next)); - await vscode.workspace - .getConfiguration('codeforge.ai') - .update('autoApprove', Array.from(next), vscode.ConfigurationTarget.Global); + await this.cfg().update('autoApprove', Array.from(next), vscode.ConfigurationTarget.Global); } } @@ -91,20 +98,17 @@ export class ApprovalPolicy { async allowAutoSession(): Promise { await this.allowAllEdits('session'); this.sessionAllowed.add('shell'); - this.sessionAllowAll = true; } clearSession(): void { this.sessionAllowed.clear(); this.sessionAllowAllEdits = false; - this.sessionAllowAll = false; } /** Composer permission chip level (does not replace Ask mode). */ getPermissionLevel(_mode?: string): PermissionLevel { - if (this.sessionAllowAll) return 'allowAll'; - const preview = - vscode.workspace.getConfiguration('codeforge.ai').get('previewEdits') === true; + if (this.globalAllowAllEnabled()) return 'allowAll'; + const preview = this.cfg().get('previewEdits') === true; if (preview) return 'assisted'; return 'default'; } @@ -114,12 +118,14 @@ export class ApprovalPolicy { * Default/Assisted/Allow all — Auto is no longer a mode). */ async applyPermissionLevel(level: PermissionLevel): Promise<'agent'> { - const cfg = vscode.workspace.getConfiguration('codeforge.ai'); + const cfg = this.cfg(); if (level === 'allowAll') { - await this.allowAutoSession(); + await cfg.update('allowAll', true, vscode.ConfigurationTarget.Global); await cfg.update('previewEdits', false, vscode.ConfigurationTarget.Global); + await this.allowAutoSession(); return 'agent'; } + await cfg.update('allowAll', false, vscode.ConfigurationTarget.Global); this.clearSession(); if (level === 'assisted') { await cfg.update('previewEdits', true, vscode.ConfigurationTarget.Global); @@ -139,22 +145,43 @@ export class ApprovalPolicy { const key = normalizeTool(tool); this.sessionAllowed.delete(key); const next = Array.from(this.alwaysAllowed()).filter(t => t !== key); - await this.context.globalState.update(GLOBAL_KEY, next); - await vscode.workspace - .getConfiguration('codeforge.ai') - .update('autoApprove', next, vscode.ConfigurationTarget.Global); + await this.cfg().update('autoApprove', next, vscode.ConfigurationTarget.Global); } async revokeAllEditsAlways(): Promise { this.sessionAllowAllEdits = false; - await this.context.globalState.update(EDITS_ALWAYS_KEY, false); - await vscode.workspace - .getConfiguration('codeforge.ai') - .update('autoApproveEdits', false, vscode.ConfigurationTarget.Global); + await this.cfg().update('autoApproveEdits', false, vscode.ConfigurationTarget.Global); for (const t of EDIT_TOOLS) { await this.revokeAlways(t); } } + + async revokeAllPermissions(): Promise { + await this.cfg().update('allowAll', false, vscode.ConfigurationTarget.Global); + await this.revokeAllEditsAlways(); + this.clearSession(); + await this.cfg().update('autoApprove', [], vscode.ConfigurationTarget.Global); + } + + /** Move legacy globalState approval lists into workspace settings once. */ + async migrateLegacyGlobalState(): Promise { + if (this.context.globalState.get(MIGRATED_KEY)) return; + + const fromState = this.context.globalState.get(GLOBAL_KEY) ?? []; + const editsAlways = this.context.globalState.get(EDITS_ALWAYS_KEY) === true; + const cfg = this.cfg(); + const current = cfg.get('autoApprove') ?? []; + const merged = Array.from(new Set([...current, ...fromState].map(normalizeTool))); + if (merged.length) { + await cfg.update('autoApprove', merged, vscode.ConfigurationTarget.Global); + } + if (editsAlways) { + await cfg.update('autoApproveEdits', true, vscode.ConfigurationTarget.Global); + } + await this.context.globalState.update(GLOBAL_KEY, undefined); + await this.context.globalState.update(EDITS_ALWAYS_KEY, undefined); + await this.context.globalState.update(MIGRATED_KEY, true); + } } function normalizeTool(tool: string): string { @@ -165,6 +192,7 @@ let shared: ApprovalPolicy | undefined; export function initApprovalPolicy(context: vscode.ExtensionContext): ApprovalPolicy { shared = new ApprovalPolicy(context); + void shared.migrateLegacyGlobalState(); return shared; } diff --git a/extensions/codeforge/src/sandbox/processKill.ts b/extensions/codeforge/src/sandbox/processKill.ts new file mode 100644 index 0000000..b32cda0 --- /dev/null +++ b/extensions/codeforge/src/sandbox/processKill.ts @@ -0,0 +1,37 @@ +/** + * Terminate a spawned process and its children (Windows job tree via taskkill). + */ + +import { execFile } from 'child_process'; +import type { ChildProcess } from 'child_process'; + +export function killProcessTree(child: ChildProcess): void { + if (!child.pid) { + try { + child.kill(); + } catch { + /* ignore */ + } + return; + } + if (process.platform === 'win32') { + execFile( + 'taskkill', + ['/pid', String(child.pid), '/T', '/F'], + { windowsHide: true }, + () => { + /* best-effort */ + } + ); + return; + } + try { + child.kill('SIGTERM'); + } catch { + try { + child.kill(); + } catch { + /* ignore */ + } + } +} diff --git a/extensions/codeforge/src/sandbox/runtime.selftest.ts b/extensions/codeforge/src/sandbox/runtime.selftest.ts new file mode 100644 index 0000000..ff5004c --- /dev/null +++ b/extensions/codeforge/src/sandbox/runtime.selftest.ts @@ -0,0 +1,39 @@ +/** + * Sandbox policy unit checks (run via node after compile). + */ +import { + hasShellChaining, + isBlockedCommand, + pickSandboxLevel, + resolveSandboxLevel, +} from './runtime'; + +function assert(cond: boolean, msg: string): void { + if (!cond) throw new Error(msg); +} + +assert( + resolveSandboxLevel('echo hi', { configured: 'restricted' }) === 'restricted', + 'restricted config stays restricted' +); +assert( + resolveSandboxLevel('echo hi', { configured: 'restricted', fullFreedom: true }) === + 'unrestricted', + 'full freedom overrides restricted' +); +assert( + pickSandboxLevel('rm -rf ~', 'unrestricted') === 'unrestricted', + 'pickSandboxLevel uses configured only' +); +assert( + resolveSandboxLevel('echo ok', { configured: 'restricted' }) === 'restricted', + 'tool sandbox hint cannot elevate — only configured level' +); + +assert(hasShellChaining('npm test && npm run build'), 'detects && chaining'); +assert(isBlockedCommand('rm -rf ~') !== undefined, 'blocks rm -rf ~'); +assert(isBlockedCommand('rd /s /q C:\\temp') !== undefined, 'blocks rd /s'); +assert(isBlockedCommand('Remove-Item -Recurse -Force x') !== undefined, 'blocks Remove-Item -Recurse'); +assert(isBlockedCommand('del /f /q /s *.log') !== undefined, 'blocks del with /f /q /s any order'); + +console.log('sandbox runtime ok'); diff --git a/extensions/codeforge/src/sandbox/runtime.ts b/extensions/codeforge/src/sandbox/runtime.ts index 1575087..cbcf67d 100644 --- a/extensions/codeforge/src/sandbox/runtime.ts +++ b/extensions/codeforge/src/sandbox/runtime.ts @@ -4,6 +4,7 @@ import { spawn, SpawnOptionsWithoutStdio } from 'child_process'; import * as path from 'path'; +import { killProcessTree } from './processKill'; export type SandboxLevel = | 'safe-local' @@ -81,15 +82,25 @@ const SAFE_PREFIXES = [ ]; const BLOCKED_PATTERNS = [ + /\brm\s+-rf\s+(?:~|\.\.|[\\/])/i, /\brm\s+-rf\s+[\\/]/i, + /\brd\s+\/s\b/i, + /\brmdir\s+\/s\b/i, + /\bremove-item\b[^\n\r;|&]*-recurse\b/i, + /\bdel(?:ete)?\b(?:[^\n\r;|&]*\/[fqs]){2,}/i, /\bformat\s+[a-z]:/i, - /\bdel\s+\/s\s+\/q\s+[a-z]:\\/i, /\bshutdown\b/i, /\breg\s+delete\b/i, /\bcurl\s+.*\|\s*(sh|bash|powershell)/i, /\bwget\s+.*\|\s*(sh|bash)/i, ]; +const SHELL_CHAIN_RE = /(?:&&|\|\||\||;)/; + +export function hasShellChaining(command: string): boolean { + return SHELL_CHAIN_RE.test(command); +} + const SECRET_ENV_KEYS = [ /api[_-]?key/i, /secret/i, @@ -129,35 +140,24 @@ export function isBlockedCommand(command: string): string | undefined { return undefined; } -export function pickSandboxLevel( - command: string, - requested?: SandboxLevel, - risk: 'low' | 'medium' | 'high' = 'medium' -): SandboxLevel { - if (requested) return requested; - if (isSafeLocalCommand(command) && risk === 'low') return 'safe-local'; - if (risk === 'high') return 'isolated'; - return 'restricted'; +export function pickSandboxLevel(_command: string, configured: SandboxLevel | undefined): SandboxLevel { + return configured ?? 'restricted'; } /** - * Resolve the effective sandbox level from settings + optional tool hint. + * Resolve the effective sandbox level from `codeforge.ai.shellSandbox` only. * `fullAgentFreedom` always wins as unrestricted. */ export function resolveSandboxLevel( command: string, opts?: { - requested?: SandboxLevel; - risk?: 'low' | 'medium' | 'high'; configured?: string | null; fullFreedom?: boolean; } ): SandboxLevel { if (opts?.fullFreedom) return 'unrestricted'; - const configured = isSandboxLevel(opts?.configured) ? opts.configured : undefined; - // Tool hint wins over configured default (except unrestricted setting forces floor). - if (configured === 'unrestricted') return 'unrestricted'; - return pickSandboxLevel(command, opts?.requested ?? configured, opts?.risk ?? 'medium'); + const configured = isSandboxLevel(opts?.configured) ? opts.configured : 'restricted'; + return pickSandboxLevel(command, configured); } export async function runSandboxed(opts: SandboxOptions): Promise { @@ -172,6 +172,15 @@ export async function runSandboxed(opts: SandboxOptions): Promise }; } + if (level === 'safe-local' && hasShellChaining(opts.command)) { + return { + exitCode: 126, + stdout: '', + stderr: 'safe-local refuses command chaining (&, &&, |, ;). Run one command at a time or use restricted+.', + level, + }; + } + if (level === 'safe-local' && !isSafeLocalCommand(opts.command)) { return { exitCode: 126, @@ -190,7 +199,7 @@ export async function runSandboxed(opts: SandboxOptions): Promise const cmdDefault = isWin && !opts.shell && !opts.args ? cmdExeSpawn(opts.command) : undefined; const args = opts.args ?? cmdDefault?.args ?? (isWin ? ['/d', '/s', '/c', opts.command] : ['-lc', opts.command]); const env = - level === 'safe-local' || level === 'unrestricted' + level === 'unrestricted' ? { ...process.env, ...(opts.env ?? {}) } : scrubEnv({ ...process.env, ...(opts.env ?? {}) }); @@ -279,21 +288,13 @@ function spawnOnce( }; const onAbort = () => { - try { - child.kill(); - } catch { - /* ignore */ - } + killProcessTree(child); finish(130, { aborted: true }); }; opts.abortSignal?.addEventListener('abort', onAbort); const timer = setTimeout(() => { - try { - child.kill(); - } catch { - /* ignore */ - } + killProcessTree(child); finish(124, { timedOut: true }); }, timeoutMs); @@ -311,6 +312,6 @@ function spawnOnce( chunksErr.push(err.message); finish(1); }); - child.on('close', code => finish(code ?? 0)); + child.on('close', code => finish(code ?? 1)); }); } diff --git a/extensions/codeforge/src/sessions/sessionStore.ts b/extensions/codeforge/src/sessions/sessionStore.ts index cc94833..1a0f2d3 100644 --- a/extensions/codeforge/src/sessions/sessionStore.ts +++ b/extensions/codeforge/src/sessions/sessionStore.ts @@ -7,7 +7,7 @@ import { randomUUID } from 'crypto'; import * as fs from 'fs/promises'; import * as path from 'path'; import { StableFacts, normalizeFacts } from '../agent/stableFacts'; -import { ensureDir, ensureHomeLayout, sessionDir, sessionsDir } from '../storage/paths'; +import { ensureDir, ensureHomeLayout, sessionDir, sessionsDir, workspaceHash } from '../storage/paths'; import { getSessionWikiStore } from '../memory/sessionWiki'; import { salvageSessionProgress } from '../agent/projectStatus'; @@ -76,6 +76,7 @@ export interface ChatSession { const STORAGE_KEY = 'codeforge.ai.sessions.v2'; const LEGACY_KEY = 'codeforge.ai.sessions.v1'; const ACTIVE_KEY = 'codeforge.ai.activeSessionId'; +const ACTIVE_BY_WORKSPACE_KEY = 'codeforge.ai.activeSessionByWorkspace'; const MAX_SESSIONS = 100; const INDEX_FILE = 'index.json'; @@ -84,6 +85,7 @@ export class SessionStore { readonly onDidChange = this._onDidChange.event; private cache: ChatSession[] | null = null; private ready: Promise; + private writeQueue: Promise = Promise.resolve(); constructor(private readonly context: vscode.ExtensionContext) { this.ready = this.bootstrap(); @@ -117,10 +119,31 @@ export class SessionStore { } getActiveId(): string | undefined { + const ws = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath; + if (ws) { + const map = + this.context.globalState.get>(ACTIVE_BY_WORKSPACE_KEY) ?? {}; + const id = map[workspaceHash(ws)]; + if (id) return id; + } return this.context.globalState.get(ACTIVE_KEY); } async setActiveId(id: string | null): Promise { + const ws = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath; + if (ws) { + const map = { + ...(this.context.globalState.get>(ACTIVE_BY_WORKSPACE_KEY) ?? + {}), + }; + const key = workspaceHash(ws); + if (id) { + map[key] = id; + } else { + delete map[key]; + } + await this.context.globalState.update(ACTIVE_BY_WORKSPACE_KEY, map); + } await this.context.globalState.update(ACTIVE_KEY, id ?? undefined); } @@ -151,7 +174,9 @@ export class SessionStore { }; const all = this.list(); all.unshift(session); - await this.persist(all.slice(0, MAX_SESSIONS)); + const trimmed = all.slice(0, MAX_SESSIONS); + this.cache = trimmed; + await this.persistChanged(session.id, { rewriteIndex: true }); await this.setActiveId(session.id); await getSessionWikiStore().ensure(session.id, partial.task); return session; @@ -183,7 +208,8 @@ export class SessionStore { } all.splice(idx, 1); all.unshift(session); - await this.persist(all); + this.cache = all; + await this.persistChanged(session.id, { rewriteIndex: true }); return session; } @@ -232,7 +258,8 @@ export class SessionStore { const all = this.list(); all.unshift(session); - await this.persist(all.slice(0, MAX_SESSIONS)); + this.cache = all.slice(0, MAX_SESSIONS); + await this.persistChanged(session.id, { rewriteIndex: true }); await this.setActiveId(session.id); await getSessionWikiStore().ensure(session.id, session.task); return session; @@ -252,7 +279,8 @@ export class SessionStore { timestamp: new Date().toISOString(), }); session.updatedAt = new Date().toISOString(); - await this.persist(all); + this.cache = all; + await this.persistChanged(id); } async createCheckpoint( @@ -281,7 +309,8 @@ export class SessionStore { session.state = 'paused'; } session.updatedAt = new Date().toISOString(); - await this.persist(all); + this.cache = all; + await this.persistChanged(id); return checkpoint; } @@ -292,7 +321,8 @@ export class SessionStore { if (!session) return; session.rollingSummary = summary; session.updatedAt = new Date().toISOString(); - await this.persist(all); + this.cache = all; + await this.persistChanged(id); } async setStableFacts(id: string, facts: StableFacts): Promise { @@ -302,7 +332,8 @@ export class SessionStore { if (!session) return; session.stableFacts = normalizeFacts(facts); session.updatedAt = new Date().toISOString(); - await this.persist(all); + this.cache = all; + await this.persistChanged(id); await getSessionWikiStore().setStableFacts(id, session.stableFacts); } @@ -313,7 +344,8 @@ export class SessionStore { if (!session) return; session.state = state; session.updatedAt = new Date().toISOString(); - await this.persist(all); + this.cache = all; + await this.persistChanged(id); } /** @@ -336,7 +368,8 @@ export class SessionStore { session.updatedAt = new Date().toISOString(); all.splice(idx, 1); all.unshift(session); - await this.persist(all); + this.cache = all; + await this.persistChanged(session.id, { rewriteIndex: true }); return session; } @@ -344,7 +377,8 @@ export class SessionStore { await this.ensureReady(); const session = this.list().find(s => s.id === id); await this.keepProjectProgress(session); - await this.persist(this.list().filter(s => s.id !== id)); + this.cache = this.list().filter(s => s.id !== id); + await this.persistChanged(undefined, { rewriteIndex: true }); try { await fs.rm(sessionDir(id), { recursive: true, force: true }); } catch { @@ -361,7 +395,8 @@ export class SessionStore { for (const session of sessions) { await this.keepProjectProgress(session); } - await this.persist([]); + this.cache = []; + await this.persistChanged(undefined, { rewriteIndex: true }); await this.setActiveId(null); for (const session of sessions) { try { @@ -417,7 +452,8 @@ export class SessionStore { parsed.workspaceFolder = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath; const all = this.list(); all.unshift(parsed); - await this.persist(all.slice(0, MAX_SESSIONS)); + this.cache = all.slice(0, MAX_SESSIONS); + await this.persistChanged(parsed.id, { rewriteIndex: true }); await this.setActiveId(parsed.id); return parsed; } @@ -439,7 +475,7 @@ export class SessionStore { } if (legacy?.length) { this.cache = legacy.map(normalizeSession); - await this.persist(this.cache); + await this.persistAllSessions(this.cache); await this.context.globalState.update(STORAGE_KEY, undefined); await this.context.globalState.update(LEGACY_KEY, undefined); } @@ -464,17 +500,62 @@ export class SessionStore { } } - private async persist(sessions: ChatSession[]): Promise { + private enqueueWrite(task: () => Promise): Promise { + const run = this.writeQueue.then(task, task); + this.writeQueue = run.catch(() => {}); + return run; + } + + private async atomicWrite(filePath: string, body: string): Promise { + const dir = path.dirname(filePath); + await ensureDir(dir); + const tmp = path.join(dir, `${path.basename(filePath)}.${randomUUID()}.tmp`); + await fs.writeFile(tmp, body, 'utf8'); + await fs.rename(tmp, filePath); + } + + private async writeSessionToDisk(session: ChatSession): Promise { + await ensureDir(sessionDir(session.id)); + const file = path.join(sessionDir(session.id), 'session.json'); + await this.atomicWrite(file, JSON.stringify(normalizeSession(session), null, 2)); + } + + private async writeIndexToDisk(): Promise { + const ids = (this.cache ?? []).slice(0, MAX_SESSIONS).map(s => s.id); + await this.atomicWrite(path.join(sessionsDir(), INDEX_FILE), JSON.stringify(ids, null, 2)); + } + + /** Write one session (and optionally refresh index.json). */ + private async persistChanged( + sessionId: string | undefined, + opts?: { rewriteIndex?: boolean } + ): Promise { + await this.enqueueWrite(async () => { + await ensureDir(sessionsDir()); + if (sessionId) { + const session = (this.cache ?? []).find(s => s.id === sessionId); + if (session) { + await this.writeSessionToDisk(session); + } + } + if (opts?.rewriteIndex) { + await this.writeIndexToDisk(); + } + this._onDidChange.fire(); + }); + } + + /** Migration / full rewrite — every session file + index. */ + private async persistAllSessions(sessions: ChatSession[]): Promise { this.cache = sessions; - await ensureDir(sessionsDir()); - const ids: string[] = []; - for (const session of sessions.slice(0, MAX_SESSIONS)) { - ids.push(session.id); - const dir = await ensureDir(sessionDir(session.id)); - await fs.writeFile(path.join(dir, 'session.json'), JSON.stringify(session, null, 2), 'utf8'); - } - await fs.writeFile(path.join(sessionsDir(), INDEX_FILE), JSON.stringify(ids, null, 2), 'utf8'); - this._onDidChange.fire(); + await this.enqueueWrite(async () => { + await ensureDir(sessionsDir()); + for (const session of sessions.slice(0, MAX_SESSIONS)) { + await this.writeSessionToDisk(session); + } + await this.writeIndexToDisk(); + this._onDidChange.fire(); + }); } } diff --git a/extensions/codeforge/src/settings/aiSecrets.ts b/extensions/codeforge/src/settings/aiSecrets.ts new file mode 100644 index 0000000..0f04583 --- /dev/null +++ b/extensions/codeforge/src/settings/aiSecrets.ts @@ -0,0 +1,153 @@ +/** + * SecretStorage for API keys and MCP sensitive fields (not in settings.json / webview). + */ + +import * as vscode from 'vscode'; +import type { AiServer, McpServerConfig } from './aiSettingsStore'; + +export const MASKED_SECRET = '••••••••'; + +function serverKey(id: string): string { + return `codeforge.ai.server.${id}.apiKey`; +} + +function mcpApiKeyKey(id: string): string { + return `codeforge.ai.mcp.${id}.apiKey`; +} + +function mcpEnvKey(id: string): string { + return `codeforge.ai.mcp.${id}.env`; +} + +function mcpHeadersKey(id: string): string { + return `codeforge.ai.mcp.${id}.headers`; +} + +export async function loadServerSecrets( + context: vscode.ExtensionContext, + servers: AiServer[] +): Promise { + const out: AiServer[] = []; + for (const s of servers) { + const stored = await context.secrets.get(serverKey(s.id)); + out.push({ + ...s, + apiKey: stored ?? s.apiKey, + }); + } + return out; +} + +export async function persistServerSecret( + context: vscode.ExtensionContext, + serverId: string, + apiKey: string | undefined +): Promise { + const key = serverKey(serverId); + const trimmed = (apiKey ?? '').trim(); + if (!trimmed || trimmed === MASKED_SECRET) return; + await context.secrets.store(key, trimmed); +} + +export async function loadMcpSecrets( + context: vscode.ExtensionContext, + servers: McpServerConfig[] +): Promise { + const out: McpServerConfig[] = []; + for (const m of servers) { + const [apiKey, envJson, headersJson] = await Promise.all([ + context.secrets.get(mcpApiKeyKey(m.id)), + context.secrets.get(mcpEnvKey(m.id)), + context.secrets.get(mcpHeadersKey(m.id)), + ]); + let env = m.env; + let headers = m.headers; + try { + if (envJson) env = JSON.parse(envJson) as Record; + } catch { + /* keep inline */ + } + try { + if (headersJson) headers = JSON.parse(headersJson) as Record; + } catch { + /* keep inline */ + } + out.push({ + ...m, + apiKey: apiKey ?? m.apiKey, + env, + headers, + }); + } + return out; +} + +export async function persistMcpSecrets( + context: vscode.ExtensionContext, + server: McpServerConfig +): Promise { + const api = (server.apiKey ?? '').trim(); + if (api && api !== MASKED_SECRET) { + await context.secrets.store(mcpApiKeyKey(server.id), api); + } + if (server.env && Object.keys(server.env).length) { + await context.secrets.store(mcpEnvKey(server.id), JSON.stringify(server.env)); + } + if (server.headers && Object.keys(server.headers).length) { + await context.secrets.store(mcpHeadersKey(server.id), JSON.stringify(server.headers)); + } +} + +export function maskStateForWebview; headers?: Record }>( + item: T +): T { + const next = { ...item }; + if (next.apiKey?.trim()) { + next.apiKey = MASKED_SECRET; + } + if (next.env) { + const env: Record = {}; + for (const [k, v] of Object.entries(next.env)) { + env[k] = secretLikeKey(k) && v?.trim() ? MASKED_SECRET : v; + } + next.env = env; + } + if (next.headers) { + const headers: Record = {}; + for (const [k, v] of Object.entries(next.headers)) { + headers[k] = /authorization|token|secret|api-key/i.test(k) && v?.trim() ? MASKED_SECRET : v; + } + next.headers = headers; + } + return next; +} + +function secretLikeKey(k: string): boolean { + return /secret|token|password|api[_-]?key|authorization/i.test(k); +} + +/** One-time migration from plaintext settings / globalState into SecretStorage. */ +export async function migrateLegacySecretsToStorage( + context: vscode.ExtensionContext, + state: { servers: AiServer[]; mcpServers: McpServerConfig[] } +): Promise { + const flag = 'codeforge.ai.secretsMigrated.v1'; + if (context.globalState.get(flag)) return; + + for (const s of state.servers) { + if (s.apiKey?.trim()) { + await context.secrets.store(serverKey(s.id), s.apiKey.trim()); + } + } + for (const m of state.mcpServers) { + await persistMcpSecrets(context, m); + } + + const cfg = vscode.workspace.getConfiguration('codeforge.ai'); + const legacyKey = (cfg.get('apiKey') ?? '').trim(); + if (legacyKey && state.servers[0]) { + await context.secrets.store(serverKey(state.servers[0].id), legacyKey); + } + + await context.globalState.update(flag, true); +} diff --git a/extensions/codeforge/src/settings/aiSettingsPanel.ts b/extensions/codeforge/src/settings/aiSettingsPanel.ts index 3dda54d..33e05a5 100644 --- a/extensions/codeforge/src/settings/aiSettingsPanel.ts +++ b/extensions/codeforge/src/settings/aiSettingsPanel.ts @@ -1,15 +1,17 @@ /** * CodeForge AI Settings — Cursor-like editor panel * - * Tabs: Models · MCP · Guardrails · Agent · Plugins + * Tabs: Models · MCP · Guardrails · Agent · Benchmark · Plugins * - Models / MCP / Guardrails: catalog UIs (same surface the Settings TOC opens) * - Agent: codeforge.ai.* prefs (shared with Settings TOC → AI → Agent) + * - Benchmark: calibrate weak vs strong model profiles (packaged harness suite) * - Plugins: marketplace (Discover / Featured) — Cursor Customize layout * * Quick model switch while coding stays on QuickPick (separate command). */ import * as vscode from 'vscode'; +import { randomBytes } from 'crypto'; import { AiSettingsStore, AiSettingsState, @@ -27,8 +29,15 @@ import { isPluginInstalled, } from './pluginsCatalog'; import { getApprovalPolicy } from '../policy/approvalPolicy'; +import { maskStateForWebview, MASKED_SECRET } from './aiSecrets'; +import { + calibrateUnavailableMessage, + findBenchScript, + runHarnessRevision, + runModelProfileCalibration, +} from './modelProfileCalibrate'; -export type AiSettingsTab = 'models' | 'mcp' | 'guardrails' | 'agent' | 'plugins'; +export type AiSettingsTab = 'models' | 'mcp' | 'guardrails' | 'agent' | 'benchmark' | 'plugins'; type AgentPrefs = { enabled: boolean; @@ -40,6 +49,9 @@ type AgentPrefs = { previewEdits: boolean; shellSandbox: string; fullAgentFreedom: boolean; + allowAll: boolean; + confirmRunPython: boolean; + outsideWorkspaceWrites: string; traceLevel: string; tabCompletion: boolean; agentCheckpointSteps: number; @@ -68,6 +80,9 @@ function readAgentPrefs(): AgentPrefs { ? shellSandbox : 'restricted', fullAgentFreedom: cfg.get('fullAgentFreedom', false), + allowAll: cfg.get('allowAll', false), + confirmRunPython: cfg.get('confirmRunPython', false), + outsideWorkspaceWrites: cfg.get('outsideWorkspaceWrites', 'ask'), traceLevel: cfg.get('traceLevel', 'basic'), tabCompletion: cfg.get('tabCompletion', true), agentCheckpointSteps: cfg.get('agentCheckpointSteps', 30), @@ -109,15 +124,28 @@ export class AiSettingsPanel { } ); - AiSettingsPanel.current = new AiSettingsPanel(panel, store, governance, tab); + AiSettingsPanel.current = new AiSettingsPanel( + panel, + store, + governance, + tab, + context.extensionPath, + context + ); return AiSettingsPanel.current; } + private calibrateBusy = false; + private readonly webviewNonce = randomBytes(16).toString('base64'); + private readonly freedomSnapshotKey = 'codeforge.ai.fullFreedom.snapshot'; + private constructor( panel: vscode.WebviewPanel, private readonly store: AiSettingsStore, private readonly governance: GovernanceStore, - initialTab: AiSettingsTab + initialTab: AiSettingsTab, + private readonly extensionPath: string, + private readonly extensionContext: vscode.ExtensionContext ) { this.panel = panel; this.panel.iconPath = undefined; @@ -150,6 +178,10 @@ export class AiSettingsPanel { state: sanitizeForWebview(state), governance: this.governance.getState(), agent: readAgentPrefs(), + calibrate: { + available: !!findBenchScript(this.extensionPath), + busy: this.calibrateBusy, + }, }); } @@ -161,13 +193,34 @@ export class AiSettingsPanel { this.postState(); return; + case 'revokePermissions': + await getApprovalPolicy().revokeAllPermissions(); + this.postState(); + return; + + case 'revokeAutoApproveTool': { + const tool = String(msg.tool ?? '').trim(); + if (tool) await getApprovalPolicy().revokeAlways(tool); + this.postState(); + return; + } + case 'updateAgentPrefs': { const prefs = msg.prefs as Partial; const cfg = vscode.workspace.getConfiguration('codeforge.ai'); + const wasFreedom = cfg.get('fullAgentFreedom') === true; let next = { ...prefs }; - if (next.fullAgentFreedom === true) { + if (next.fullAgentFreedom === true && !wasFreedom) { + await this.extensionContext.globalState.update(this.freedomSnapshotKey, { + allowAll: cfg.get('allowAll'), + shellSandbox: cfg.get('shellSandbox'), + autoApproveEdits: cfg.get('autoApproveEdits'), + previewEdits: cfg.get('previewEdits'), + autoApprove: cfg.get('autoApprove') ?? [], + }); next = { ...next, + allowAll: true, shellSandbox: 'unrestricted', autoApproveEdits: true, previewEdits: false, @@ -176,11 +229,43 @@ export class AiSettingsPanel { ), }; try { - await getApprovalPolicy().allowAutoSession(); + await getApprovalPolicy().setGlobalAllowAll(true); } catch { /* policy may not be ready in settings-only path */ } } + if (next.fullAgentFreedom === false && wasFreedom) { + const snap = this.extensionContext.globalState.get<{ + allowAll?: boolean; + shellSandbox?: string; + autoApproveEdits?: boolean; + previewEdits?: boolean; + autoApprove?: string[]; + }>(this.freedomSnapshotKey); + if (snap) { + if (snap.allowAll !== undefined) next.allowAll = snap.allowAll; + if (snap.shellSandbox) next.shellSandbox = snap.shellSandbox; + if (snap.autoApproveEdits !== undefined) next.autoApproveEdits = snap.autoApproveEdits; + if (snap.previewEdits !== undefined) next.previewEdits = snap.previewEdits; + if (snap.autoApprove) next.autoApprove = snap.autoApprove; + } + try { + getApprovalPolicy().clearSession(); + } catch { + /* ignore */ + } + await this.extensionContext.globalState.update(this.freedomSnapshotKey, undefined); + } + if (next.allowAll === true && next.fullAgentFreedom !== true) { + try { + await getApprovalPolicy().setGlobalAllowAll(true); + } catch { + /* ignore */ + } + } + if (next.allowAll === false) { + await cfg.update('allowAll', false, vscode.ConfigurationTarget.Global); + } const entries: Array<[keyof AgentPrefs, unknown]> = [ ['enabled', next.enabled], ['mode', next.mode], @@ -191,6 +276,9 @@ export class AiSettingsPanel { ['previewEdits', next.previewEdits], ['shellSandbox', next.shellSandbox], ['fullAgentFreedom', next.fullAgentFreedom], + ['allowAll', next.allowAll], + ['confirmRunPython', next.confirmRunPython], + ['outsideWorkspaceWrites', next.outsideWorkspaceWrites], ['traceLevel', next.traceLevel], ['tabCompletion', next.tabCompletion], ['agentCheckpointSteps', next.agentCheckpointSteps], @@ -224,7 +312,7 @@ export class AiSettingsPanel { if (idx < 0) return; // Preserve apiKey if webview sent masked placeholder const prev = state.servers[idx]; - if (server.apiKey === '••••••••' || server.apiKey === undefined) { + if (server.apiKey === MASKED_SECRET || server.apiKey === undefined) { server.apiKey = prev.apiKey; } state.servers[idx] = server; @@ -485,6 +573,166 @@ export class AiSettingsPanel { void vscode.window.showInformationMessage('Reset to built-in defaults'); return; } + case 'calibrateProfiles': { + await this.runCalibrateFromUi({ + model: String(msg.model || ''), + reps: Number(msg.reps) || 3, + }); + return; + } + case 'reviseHarness': { + await this.runReviseFromUi({ + model: String(msg.model || ''), + reps: Number(msg.reps) || 3, + }); + return; + } + case 'openCalibrateReport': { + const p = String(msg.path || ''); + if (p) { + try { + await vscode.window.showTextDocument(vscode.Uri.file(p)); + } catch (err) { + void vscode.window.showWarningMessage( + err instanceof Error ? err.message : String(err) + ); + } + } + return; + } + } + } + + private async runReviseFromUi(opts: { model: string; reps: number }): Promise { + if (this.calibrateBusy) { + void vscode.window.showWarningMessage('A benchmark is already running.'); + return; + } + if (!findBenchScript(this.extensionPath)) { + void vscode.window.showErrorMessage(calibrateUnavailableMessage()); + return; + } + const state = this.store.getState(); + this.calibrateBusy = true; + this.postState(); + this.panel.webview.postMessage({ + type: 'calibrateProgress', + message: 'Revising harness: proposer, then T2 and T4 × 3…', + }); + const result = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Revising harness', + cancellable: true, + }, + async (progress, token) => { + progress.report({ message: 'Starting…' }); + return runHarnessRevision({ + model: opts.model, + reps: Math.max(3, opts.reps || 3), + serverId: state.activeServerId ?? undefined, + extensionPath: this.extensionPath, + token, + onLine: line => { + const noteworthy = + line.startsWith('===') || + /^(PASS|FAIL|KEPT|REVERTED|Diagnosis|Harness file|Harness revision|Selection suite|Protocol )/.test( + line + ); + if (noteworthy) { + this.panel.webview.postMessage({ type: 'calibrateProgress', message: line }); + } + if (line.startsWith('=== revise') || line.startsWith('KEPT') || line.startsWith('REVERTED')) { + progress.report({ message: line.replace(/^===\s*/, '').replace(/\s*===$/, '') }); + } + }, + }); + } + ); + this.calibrateBusy = false; + this.postState(); + this.panel.webview.postMessage({ + type: 'calibrateResult', + ok: result.ok, + summary: result.summary, + error: result.error, + }); + } + + private async runCalibrateFromUi(opts: { + model: string; + reps: number; + }): Promise { + if (this.calibrateBusy) { + void vscode.window.showWarningMessage('Calibration is already running.'); + return; + } + if (!findBenchScript(this.extensionPath)) { + void vscode.window.showErrorMessage(calibrateUnavailableMessage()); + this.panel.webview.postMessage({ + type: 'calibrateResult', + ok: false, + error: calibrateUnavailableMessage(), + }); + return; + } + const state = this.store.getState(); + this.calibrateBusy = true; + this.postState(); + this.panel.webview.postMessage({ + type: 'calibrateProgress', + message: 'Starting suite (T2, T4, CS2, CS3)…', + }); + + const result = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Notification, + title: 'Calibrating model profiles', + cancellable: true, + }, + async (progress, token) => { + progress.report({ message: 'Starting…' }); + return runModelProfileCalibration({ + model: opts.model, + reps: opts.reps, + serverId: state.activeServerId ?? undefined, + extensionPath: this.extensionPath, + token, + onLine: line => { + // Skip bulky headless JSON dumps; keep suite progress lines for the UI. + const noteworthy = + line.startsWith('===') || + /^(PASS|FAIL|Merged|Wrote|# Calibration)/.test(line) || + line.startsWith('| '); + if (noteworthy) { + this.panel.webview.postMessage({ type: 'calibrateProgress', message: line }); + } + if (line.startsWith('=== calibrate')) { + progress.report({ + message: line.replace(/^===\s*/, '').replace(/\s*===$/, ''), + }); + } + }, + }); + } + ); + + this.calibrateBusy = false; + this.postState(); + this.panel.webview.postMessage({ + type: 'calibrateResult', + ok: result.ok, + summary: result.summary, + reportPath: result.reportPath, + reportMarkdown: result.reportMarkdown, + error: result.error, + }); + if (result.ok) { + void vscode.window.showInformationMessage( + 'Calibration finished. Profiles updated under ~/.CodeForge/ai/model-profiles/.' + ); + } else if (result.error && result.error !== 'Calibration cancelled.') { + void vscode.window.showErrorMessage(result.error); } } @@ -508,6 +756,7 @@ export class AiSettingsPanel { + CodeForge Settings