From ddbf96e2e70ccddfb0669c0ae7213c05febda5d2 Mon Sep 17 00:00:00 2001
From: Tal Zaccai <talzacc@microsoft.com>
Date: Thu, 25 Jun 2026 17:31:51 -0600
Subject: [PATCH 01/15] Replay: make Impact Report two-mode (nfa-grammar vs
 completionBased-cache)

Make the replay's deterministic dispatch model explicit instead of implicitly
mixing two paths that never coexist in a real dispatcher config.

Level A (core gating): add StudioReplayMode = "nfa-grammar" |
"completionBased-cache" and mode? on StudioReplayRequest (default
nfa-grammar); gate the live construction-cache consult behind
completionBased-cache. Default runs are now grammar-only and A/B-symmetric.

Level B (plumbing + UI + test):
- Thread mode through the webview run message; parseWebviewMessage validates
  it (unknown/missing -> nfa-grammar). The host forwards it into replayCorpus;
  the channel/RPC layers ride on StudioReplayRequest unchanged.
- Add a two-state Grammar/Cache toggle to the Impact Report action bar with
  explanatory tooltips, persisted/restored with the version selection.
- Add an injectable resolveConstructionCache seam to CreateStudioRuntimeOptions
  so the gating is testable without a live cache.
- Tests: 3 runtime gating tests over a scaffolded agent (cache skipped in
  nfa-grammar / when mode omitted, consulted in completionBased-cache); update
  webview protocol run-message expectations to include mode.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../src/runtime/studioRuntimeCore.ts          |  46 ++++-
 .../typeagent-studio/src/impactReportView.ts  |   4 +
 .../src/test/studioRuntimeCore.spec.ts        | 165 ++++++++++++++++++
 .../src/test/webviewProtocol.spec.ts          |  12 +-
 .../src/webviewKit/client/impactReport.ts     |  68 +++++++-
 .../src/webviewKit/protocol.ts                |  18 +-
 6 files changed, 303 insertions(+), 10 deletions(-)

diff --git a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
index e0529f34d6..2371d6b6b0 100644
--- a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
+++ b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
@@ -233,6 +233,21 @@ export interface PackagingHealthGateResult {
     checkedAgent?: string;
 }
 
+/**
+ * Which deterministic dispatch path replay models:
+ * - `nfa-grammar` (default) — grammar-only matching through the real NFA grammar
+ *   store, symmetric for both versions. The construction cache is NOT consulted
+ *   (faithful to the dispatcher's `grammarSystem: "nfa"` mode, where cache
+ *   validation is intentionally skipped). The cleanest signal for "what did my
+ *   `.agr` edit change", since A and B are equivalent environments.
+ * - `completionBased-cache` — the live working-tree side additionally consults
+ *   the agent's real per-session construction cache before the grammar match
+ *   (faithful to the dispatcher's default `completionBased` mode). Asymmetric:
+ *   only the working-tree side reads the cache, so a cache hit can mask or fake a
+ *   grammar regression. Opt-in for "what would my default dispatcher likely do".
+ */
+export type StudioReplayMode = "nfa-grammar" | "completionBased-cache";
+
 /** Request shape for {@link StudioRuntime.replayCorpus}. Versions and miss
  *  policy default to a deterministic working-tree self-compare. */
 export interface StudioReplayRequest {
@@ -241,6 +256,8 @@ export interface StudioReplayRequest {
     versionA?: VersionSpec;
     versionB?: VersionSpec;
     missPolicy?: ReplayMissPolicy;
+    /** Deterministic dispatch path to model. Defaults to `nfa-grammar`. */
+    mode?: StudioReplayMode;
 }
 
 /**
@@ -546,6 +563,15 @@ export interface CreateStudioRuntimeOptions {
      * deterministic identity replay over each entry's captured `expectedAction`.
      */
     replayResolver?: ReplayActionResolver;
+    /**
+     * Resolves the live construction-cache layer for the working-tree side of a
+     * replay. Injected so tests can exercise the mode gating (the default
+     * discovers the dispatcher's live cache from the instance dir, which a test
+     * can't fabricate). Only consulted in `completionBased-cache` mode.
+     */
+    resolveConstructionCache?: (
+        target: GrammarReplayTarget,
+    ) => Promise<ConstructionCacheLayer | undefined>;
     collisions?: CollisionService;
     /**
      * Scans agents' compiled grammars for collisions. Injected so tests can
@@ -995,6 +1021,7 @@ export function createStudioRuntimeCore(
             return feedback.list(filter);
         },
         async replayCorpus(request) {
+            const mode: StudioReplayMode = request.mode ?? "nfa-grammar";
             const replayOptions = {
                 agent: request.agent,
                 corpus: request.corpus ?? {},
@@ -1024,12 +1051,21 @@ export function createStudioRuntimeCore(
                 );
                 if (target !== undefined) {
                     // Best-effort consult of the agent's live per-session
-                    // construction cache for the working-tree side. Hash-gated to
-                    // the current schema exactly as the dispatcher gates it, so a
-                    // schema edit invalidates the cached constructions (→ stale →
-                    // grammar fallback) rather than reporting a phantom cache hit.
+                    // construction cache for the working-tree side — but ONLY in
+                    // `completionBased-cache` mode. In the default `nfa-grammar`
+                    // mode the dispatcher does not consult the cache (grammar
+                    // rules alone decide the match), so replay stays grammar-only
+                    // and A/B-symmetric. Hash-gated to the current schema exactly
+                    // as the dispatcher gates it, so a schema edit invalidates the
+                    // cached constructions (→ stale → grammar fallback) rather
+                    // than reporting a phantom cache hit.
                     const constructionCache =
-                        await resolveConstructionCacheLayer(target);
+                        mode === "completionBased-cache"
+                            ? await (
+                                  options.resolveConstructionCache ??
+                                  resolveConstructionCacheLayer
+                              )(target)
+                            : undefined;
 
                     const grammarResolver = createGrammarReplayResolver({
                         target,
diff --git a/ts/packages/typeagent-studio/src/impactReportView.ts b/ts/packages/typeagent-studio/src/impactReportView.ts
index 1c95d67a2a..5aaf6a38fd 100644
--- a/ts/packages/typeagent-studio/src/impactReportView.ts
+++ b/ts/packages/typeagent-studio/src/impactReportView.ts
@@ -360,6 +360,10 @@ export function openImpactReport(
                 // free of LLM calls.
                 versionA: msg.versionA,
                 versionB: msg.versionB,
+                // The webview's mode toggle selects which deterministic dispatch
+                // path to model (grammar-only vs construction-cache-first); the
+                // runtime defaults unknown/missing to the cache-free baseline.
+                mode: msg.mode,
                 missPolicy: "needs-explanation",
             });
             post({
diff --git a/ts/packages/typeagent-studio/src/test/studioRuntimeCore.spec.ts b/ts/packages/typeagent-studio/src/test/studioRuntimeCore.spec.ts
index d72ca3076f..57d2a104dc 100644
--- a/ts/packages/typeagent-studio/src/test/studioRuntimeCore.spec.ts
+++ b/ts/packages/typeagent-studio/src/test/studioRuntimeCore.spec.ts
@@ -1182,3 +1182,168 @@ test("scanGrammarCollisions defaults to agents loaded across sandboxes", async (
 
     assert.deepEqual(received, []);
 });
+
+// --- Replay mode gating (Level B) -----------------------------------------
+// `replayCorpus` consults the live construction cache only in
+// `completionBased-cache` mode; the default `nfa-grammar` mode matches against
+// the grammar alone. These tests scaffold a real single-schema agent so the
+// auto-engaged grammar path runs, and inject a spy cache resolver so the gating
+// is observable without a real on-disk construction cache.
+
+const GATING_GRAMMAR = [
+    "<Start> = <Pause> | <Resume>;",
+    '<Pause> = pause -> { actionName: "pause" };',
+    '<Resume> = resume -> { actionName: "resume" };',
+].join("\n");
+
+const GATING_SCHEMA_TS = [
+    "export type DemoActions = PauseAction | ResumeAction;",
+    "",
+    "// Pause playback.",
+    'export type PauseAction = { actionName: "pause" };',
+    "",
+    "// Resume playback.",
+    'export type ResumeAction = { actionName: "resume" };',
+    "",
+].join("\n");
+
+const GATING_MANIFEST = JSON.stringify({
+    schema: {
+        originalSchemaFile: "./demoSchema.ts",
+        schemaType: { action: "DemoActions" },
+    },
+});
+
+/**
+ * Scaffold a temp repo whose `packages/agents/demo` is a discoverable
+ * single-schema agent (grammar + schema + manifest), so `resolveRepoRoot` and
+ * `resolveGrammarReplayTarget` engage the real grammar replay path.
+ */
+async function scaffoldGatingRepo(): Promise<string> {
+    const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "studio-gating-"));
+    const agentDir = path.join(
+        repoRoot,
+        "packages",
+        "agents",
+        "demo",
+        "src",
+        "agent",
+    );
+    await fs.mkdir(agentDir, { recursive: true });
+    await fs.writeFile(
+        path.join(agentDir, "demoSchema.agr"),
+        GATING_GRAMMAR,
+        "utf8",
+    );
+    await fs.writeFile(
+        path.join(agentDir, "demoSchema.ts"),
+        GATING_SCHEMA_TS,
+        "utf8",
+    );
+    await fs.writeFile(
+        path.join(agentDir, "demoManifest.json"),
+        GATING_MANIFEST,
+        "utf8",
+    );
+    return repoRoot;
+}
+
+function gatingEntry(id: string, utterance: string): CorpusEntry {
+    return {
+        id,
+        utterance,
+        agent: "demo",
+        source: "in-repo",
+        provenance: { sourceUri: `mem://${id}` },
+    };
+}
+
+/** A spy `resolveConstructionCache` that records consults and returns a valid
+ *  layer resolving "pause" from the cache. */
+function spyCacheResolver(record: { consulted: boolean }) {
+    return async () => {
+        record.consulted = true;
+        return {
+            status: "valid" as const,
+            cacheFilePath: "/stub/constructions.json",
+            schemaName: "demo",
+            currentHash: "hash=",
+            cachedHash: "hash=",
+            match(utterance: string) {
+                return utterance === "pause"
+                    ? { schemaName: "demo", actionName: "pause" }
+                    : undefined;
+            },
+        };
+    };
+}
+
+test("replayCorpus skips the construction cache in the default nfa-grammar mode", async () => {
+    const repoRoot = await scaffoldGatingRepo();
+    try {
+        const corpus = new StubCorpusService([gatingEntry("p", "pause")]);
+        const record = { consulted: false };
+        const runtime = createStudioRuntimeCore(
+            createContext([repoRoot]).context,
+            { corpus, resolveConstructionCache: spyCacheResolver(record) },
+        );
+
+        const result = await runtime.replayCorpus({
+            agent: "demo",
+            missPolicy: "needs-explanation",
+            mode: "nfa-grammar",
+        });
+
+        assert.equal(record.consulted, false);
+        assert.notEqual(result.method, "construction-cache");
+    } finally {
+        await fs.rm(repoRoot, { recursive: true, force: true });
+    }
+});
+
+test("replayCorpus omitting the mode defaults to grammar-only (no cache consult)", async () => {
+    const repoRoot = await scaffoldGatingRepo();
+    try {
+        const corpus = new StubCorpusService([gatingEntry("p", "pause")]);
+        const record = { consulted: false };
+        const runtime = createStudioRuntimeCore(
+            createContext([repoRoot]).context,
+            { corpus, resolveConstructionCache: spyCacheResolver(record) },
+        );
+
+        const result = await runtime.replayCorpus({
+            agent: "demo",
+            missPolicy: "needs-explanation",
+        });
+
+        assert.equal(record.consulted, false);
+        assert.notEqual(result.method, "construction-cache");
+    } finally {
+        await fs.rm(repoRoot, { recursive: true, force: true });
+    }
+});
+
+test("replayCorpus consults the construction cache in completionBased-cache mode", async () => {
+    const repoRoot = await scaffoldGatingRepo();
+    try {
+        const corpus = new StubCorpusService([gatingEntry("p", "pause")]);
+        const record = { consulted: false };
+        const runtime = createStudioRuntimeCore(
+            createContext([repoRoot]).context,
+            { corpus, resolveConstructionCache: spyCacheResolver(record) },
+        );
+
+        const result = await runtime.replayCorpus({
+            agent: "demo",
+            missPolicy: "needs-explanation",
+            mode: "completionBased-cache",
+        });
+
+        assert.equal(record.consulted, true);
+        // A valid cache layer for the working-tree side surfaces as the
+        // construction-cache method.
+        assert.equal(result.method, "construction-cache");
+    } finally {
+        await fs.rm(repoRoot, { recursive: true, force: true });
+    }
+});
diff --git a/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts b/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts
index 946b89dd75..9adeefdede 100644
--- a/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts
+++ b/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts
@@ -17,7 +17,8 @@ test("parseWebviewMessage accepts well-formed messages", () => {
         type: "pickVersion",
         side: "b",
     });
-    // Missing/invalid version fields coerce to the working tree.
+    // Missing/invalid version fields coerce to the working tree; an absent mode
+    // defaults to the grammar-only baseline.
     assert.deepEqual(
         parseWebviewMessage({ type: "run", requestId: 3, agent: "player" }),
         {
@@ -26,6 +27,7 @@ test("parseWebviewMessage accepts well-formed messages", () => {
             agent: "player",
             versionA: { kind: "workingTree" },
             versionB: { kind: "workingTree" },
+            mode: "nfa-grammar",
         },
     );
     // String version fields are coerced (legacy text-field / test seam).
@@ -36,6 +38,7 @@ test("parseWebviewMessage accepts well-formed messages", () => {
             agent: "player",
             versionA: "HEAD",
             versionB: "working tree",
+            mode: "completionBased-cache",
         }),
         {
             type: "run",
@@ -43,9 +46,11 @@ test("parseWebviewMessage accepts well-formed messages", () => {
             agent: "player",
             versionA: { kind: "git", ref: "HEAD" },
             versionB: { kind: "workingTree" },
+            mode: "completionBased-cache",
         },
     );
-    // Typed specs from a picker selection are validated and taken as-is.
+    // Typed specs from a picker selection are validated and taken as-is; an
+    // unknown mode value falls back to the grammar-only baseline.
     assert.deepEqual(
         parseWebviewMessage({
             type: "run",
@@ -53,6 +58,7 @@ test("parseWebviewMessage accepts well-formed messages", () => {
             agent: "player",
             versionA: { kind: "git", ref: "v1.0" },
             versionB: { kind: "workingTree" },
+            mode: "bogus-mode",
         }),
         {
             type: "run",
@@ -60,6 +66,7 @@ test("parseWebviewMessage accepts well-formed messages", () => {
             agent: "player",
             versionA: { kind: "git", ref: "v1.0" },
             versionB: { kind: "workingTree" },
+            mode: "nfa-grammar",
         },
     );
     // Non-string / malformed version fields fall back to the working tree.
@@ -77,6 +84,7 @@ test("parseWebviewMessage accepts well-formed messages", () => {
             agent: "player",
             versionA: { kind: "workingTree" },
             versionB: { kind: "workingTree" },
+            mode: "nfa-grammar",
         },
     );
 });
diff --git a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
index 88e02b3183..c7436d19de 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
@@ -10,7 +10,10 @@
  * `vscode`, or node built-ins (so it bundles for the browser).
  */
 
-import type { StudioReplayResult } from "@typeagent/core/runtime";
+import type {
+    StudioReplayResult,
+    StudioReplayMode,
+} from "@typeagent/core/runtime";
 import type { ActionDelta } from "@typeagent/core/replay";
 import type {
     HostToWebviewMessage,
@@ -64,6 +67,7 @@ interface PanelState {
     selectedAgent?: string;
     versionA?: ResolvedVersion;
     versionB?: ResolvedVersion;
+    mode?: StudioReplayMode;
     lastResult?: PersistedResult;
 }
 interface VsCodeApi {
@@ -113,6 +117,31 @@ const activeFilters = defaultImpactFilters();
 let currentAgent: string | undefined;
 let versionA: ResolvedVersion = DEFAULT_VERSION_A;
 let versionB: ResolvedVersion = DEFAULT_VERSION_B;
+// Which deterministic dispatch path the next run models. `nfa-grammar` (default)
+// matches both sides against the compiled grammar only — symmetric, no cache.
+// `completionBased-cache` lets the working-tree side consult the live
+// construction cache first (the default-dispatcher path). Persisted with the
+// version selection so a reload keeps the chosen mode.
+let mode: StudioReplayMode = "nfa-grammar";
+
+/** Short toolbar label for each replay mode. */
+const MODE_LABEL: Record<StudioReplayMode, string> = {
+    "nfa-grammar": "Grammar",
+    "completionBased-cache": "Cache",
+};
+/** Hover copy explaining each mode and that the button toggles between them. */
+const MODE_TOOLTIP: Record<StudioReplayMode, string> = {
+    "nfa-grammar":
+        "Grammar mode: both versions match against the compiled grammar only — " +
+        "the construction cache is never consulted, so A and B stay symmetric. " +
+        "Cleanest signal for what a grammar or schema edit changed. " +
+        "Click to switch to Cache mode.",
+    "completionBased-cache":
+        "Cache mode: the working-tree side consults the live construction cache " +
+        "before the grammar (the default dispatcher path), so a cache hit reflects " +
+        "what the dispatcher would serve. Asymmetric — a B-side cache hit can mask " +
+        "a grammar regression. Click to switch to Grammar mode.",
+};
 
 const root = document.getElementById("root")!;
 
@@ -143,6 +172,16 @@ const swapButton = toolButton("arrow-swap", "Swap A and B", () =>
     swapVersions(),
 );
 swapButton.title = "Swap the base (A) and compare (B) versions.";
+
+// A two-state toggle for the replay mode. Only two modes exist, so a click
+// flips between them (no dropdown). Built by hand (not `toolButton`) so we keep
+// a handle on the label node and update it in place from `renderModeButton`.
+const modeButton = document.createElement("button");
+modeButton.type = "button";
+modeButton.className = "tool-button";
+const modeButtonText = text("");
+modeButton.append(codicon("settings"), modeButtonText);
+modeButton.addEventListener("click", () => cycleMode());
 const runButton = toolButton("play", "Run", () => runReplay(), {
     primary: true,
     text: "Run",
@@ -159,6 +198,8 @@ actionBar.append(
     group(codicon("library"), agentNameEl),
     separator(),
     group(versionAButton.button, swapButton, versionBButton.button),
+    separator(),
+    group(modeButton),
     spacer(),
     runButton,
     connectionPill,
@@ -188,6 +229,7 @@ setControlsEnabled(false);
 renderConnection("connecting");
 restoreSelection();
 renderVersionButtons();
+renderModeButton();
 
 // --- Messaging ------------------------------------------------------------
 window.addEventListener("message", (event: MessageEvent) => {
@@ -259,7 +301,7 @@ function runReplay(): void {
     }
     requestId += 1;
     latestRequestId = requestId;
-    persistState({ selectedAgent: agent, versionA, versionB });
+    persistState({ selectedAgent: agent, versionA, versionB, mode });
     setControlsEnabled(false);
     setStatus(`Replaying ${agent}…`);
     clearNotification();
@@ -277,9 +319,27 @@ function runReplay(): void {
         agent,
         versionA: versionA.spec,
         versionB: versionB.spec,
+        mode,
     });
 }
 
+/** Flip the replay mode between its two states and persist the choice. */
+function cycleMode(): void {
+    mode = mode === "nfa-grammar" ? "completionBased-cache" : "nfa-grammar";
+    renderModeButton();
+    persistState({ mode });
+}
+
+/** Paint the mode toggle's label and tooltip from the current `mode`. */
+function renderModeButton(): void {
+    modeButtonText.textContent = MODE_LABEL[mode];
+    modeButton.title = MODE_TOOLTIP[mode];
+    modeButton.setAttribute(
+        "aria-label",
+        `Replay mode: ${MODE_LABEL[mode]}. ${MODE_TOOLTIP[mode]}`,
+    );
+}
+
 /** Apply a version selection from the host picker to one side. */
 function applyVersionPick(side: ReplaySide, resolved: ResolvedVersion): void {
     if (side === "a") {
@@ -700,6 +760,9 @@ function restoreSelection(): void {
     if (state?.versionB) {
         versionB = state.versionB;
     }
+    if (state?.mode) {
+        mode = state.mode;
+    }
     // The agent is authoritative from the host `init`; until it arrives, show a
     // neutral placeholder rather than a stale persisted value.
     renderAgentName();
@@ -754,6 +817,7 @@ function setControlsEnabled(enabled: boolean): void {
     swapButton.disabled = !enabled;
     versionAButton.button.disabled = !enabled;
     versionBButton.button.disabled = !enabled;
+    modeButton.disabled = !enabled;
 }
 
 function setStatus(text: string): void {
diff --git a/ts/packages/typeagent-studio/src/webviewKit/protocol.ts b/ts/packages/typeagent-studio/src/webviewKit/protocol.ts
index 0e598ae848..493ed81ba7 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/protocol.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/protocol.ts
@@ -11,7 +11,10 @@
  * touches a socket (the security boundary).
  */
 
-import type { StudioReplayResult } from "@typeagent/core/runtime";
+import type {
+    StudioReplayResult,
+    StudioReplayMode,
+} from "@typeagent/core/runtime";
 import type { VersionSpec } from "@typeagent/core/replay";
 import {
     coerceVersionSpec,
@@ -71,6 +74,8 @@ export type WebviewToHostMessage =
           versionA: VersionSpec;
           /** Validated compare (B) version spec. */
           versionB: VersionSpec;
+          /** Which deterministic dispatch path to model. */
+          mode: StudioReplayMode;
       }
     /** Ask the host to open a native version QuickPick for one side. */
     | { type: "pickVersion"; side: ReplaySide };
@@ -79,6 +84,15 @@ function narrowSide(value: unknown): ReplaySide | undefined {
     return value === "a" || value === "b" ? value : undefined;
 }
 
+/**
+ * Narrow an untrusted mode into a {@link StudioReplayMode}, defaulting unknown
+ * or missing values to the grammar-only `nfa-grammar` baseline (the safer,
+ * cache-free default the runtime also falls back to).
+ */
+function narrowMode(value: unknown): StudioReplayMode {
+    return value === "completionBased-cache" ? value : "nfa-grammar";
+}
+
 /** Narrow an untrusted value into a {@link WebviewToHostMessage}. */
 export function parseWebviewMessage(
     value: unknown,
@@ -100,6 +114,7 @@ export function parseWebviewMessage(
                 agent?: unknown;
                 versionA?: unknown;
                 versionB?: unknown;
+                mode?: unknown;
             };
             if (
                 typeof m.requestId === "number" &&
@@ -116,6 +131,7 @@ export function parseWebviewMessage(
                     agent: m.agent,
                     versionA: coerceVersionSpec(m.versionA),
                     versionB: coerceVersionSpec(m.versionB),
+                    mode: narrowMode(m.mode),
                 };
             }
             return undefined;

From 3ebcf128fef722edbc0f9955d5857a7dea28e849 Mon Sep 17 00:00:00 2001
From: Tal Zaccai <talzacc@microsoft.com>
Date: Thu, 25 Jun 2026 18:55:59 -0600
Subject: [PATCH 02/15] Add replay wildcard validation (fidelity rung L4a)

Replay now optionally runs each agent's real validateWildcardMatch over the
working-tree side's wildcard grammar matches -- the dispatcher's only
beyond-grammar determinism (getValidatedMatches) -- dropping a match the agent
rejects, exactly as the dispatcher does. Working-tree side only; git-ref side
stays grammar-only. Opt-in (default off) and fail-open: only an explicit false
rejects; a missing/throwing validator or unloadable agent accepts and records a
diagnostic, so replay never fabricates a lost match from infrastructure noise.

Core (@typeagent/core, dependency-light):
- replay/wildcardValidator.ts: createWildcardMatchValidator with an INJECTED
  ReplayAppAgentLoader (no dispatcher dep), empty-object stub SessionContext,
  allowlist (timer/list/player), fail-open diagnostics, dispose->unloadAppAgent.
- grammarResolver.ts: selectValidatedMatchAction walks the ranked MatchResult[]
  (wildcardCharCount===0 auto-accepts, first accepted wins, all-rejected =>
  needs-explanation); exposes wildcardValidationApplied. Working-tree only.
- studioRuntimeCore.ts: StudioReplayRequest.validateWildcards opt-in,
  resolveWildcardValidator injectable seam, StudioReplayResult.wildcardValidation
  summary, validator build/dispose lifecycle. Re-exported the API from
  @typeagent/core/runtime so the host can wire a real loader.

Host (studio-service):
- wildcardValidation.ts: a default loader that lazily imports
  default-agent-provider (marked external in the service bundle), so it resolves
  on the in-repo dev path and cleanly no-ops in the packaged .vsix. Gated on the
  allowlist so the import only fires for an allowlisted wildcard match.

Webview (typeagent-studio):
- A lit Validate toggle (mirrors the mode toggle), threaded through the run
  protocol message, with an honest sub-bar indicator: wildcard-validated, or a
  warning-tinted unavailable/skipped/no-validator/degraded from the diagnostics.

Tests: core +40 (wildcardValidator + grammarReplayResolver L4a), studio protocol
expectations. core 189, studio 184, studio-service 21 green.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 ts/packages/studio-service/package.json       |   1 +
 ts/packages/studio-service/src/runtime.ts     |  17 +-
 .../studio-service/src/wildcardValidation.ts  | 123 +++++++++
 .../src/replay/grammarResolver.ts             | 103 ++++++-
 .../src/replay/wildcardValidator.ts           | 254 ++++++++++++++++++
 .../typeagent-core/src/runtime/index.ts       |  18 ++
 .../src/runtime/studioRuntimeCore.ts          |  88 +++++-
 .../test/grammarReplayResolver.spec.ts        | 216 +++++++++++++++
 .../test/wildcardValidator.spec.ts            | 217 +++++++++++++++
 ts/packages/typeagent-studio/esbuild.mjs      |   7 +-
 .../typeagent-studio/media/impactReport.css   |  25 ++
 .../typeagent-studio/src/impactReportView.ts  |   4 +
 .../src/test/webviewProtocol.spec.ts          |   9 +-
 .../src/webviewKit/client/impactReport.ts     | 152 ++++++++++-
 .../src/webviewKit/protocol.ts                |   9 +
 ts/pnpm-lock.yaml                             |   7 +-
 16 files changed, 1231 insertions(+), 19 deletions(-)
 create mode 100644 ts/packages/studio-service/src/wildcardValidation.ts
 create mode 100644 ts/packages/typeagent-core/src/replay/wildcardValidator.ts
 create mode 100644 ts/packages/typeagent-core/test/wildcardValidator.spec.ts

diff --git a/ts/packages/studio-service/package.json b/ts/packages/studio-service/package.json
index 923470525e..78dc02309c 100644
--- a/ts/packages/studio-service/package.json
+++ b/ts/packages/studio-service/package.json
@@ -38,6 +38,7 @@
     "@typeagent/core": "workspace:*",
     "@typeagent/websocket-utils": "workspace:*",
     "debug": "^4.4.0",
+    "default-agent-provider": "workspace:*",
     "ws": "^8.21.0"
   },
   "devDependencies": {
diff --git a/ts/packages/studio-service/src/runtime.ts b/ts/packages/studio-service/src/runtime.ts
index 0dcc7965b4..4ae42835ef 100644
--- a/ts/packages/studio-service/src/runtime.ts
+++ b/ts/packages/studio-service/src/runtime.ts
@@ -14,6 +14,7 @@ import {
     FileWorkspaceState,
     studioWorkspaceStateFile,
 } from "./fileWorkspaceState.js";
+import { createDefaultWildcardValidatorResolver } from "./wildcardValidation.js";
 
 /**
  * Candidate repository roots for Studio to inspect, most-specific first. The
@@ -130,11 +131,17 @@ export function getStudioRuntime(repoRoot?: string): StudioRuntime {
         // Recover a pre-canonical snapshot for this workspace, if any, before
         // the runtime reads its persisted state.
         migrateLegacyWorkspaceState(stateFile, resolved);
-        runtime = createStudioRuntimeCore({
-            workspaceState: new FileWorkspaceState(stateFile),
-            globalStorageFsPath: studioProfileDir(),
-            workspaceFolderFsPaths: candidates,
-        });
+        runtime = createStudioRuntimeCore(
+            {
+                workspaceState: new FileWorkspaceState(stateFile),
+                globalStorageFsPath: studioProfileDir(),
+                workspaceFolderFsPaths: candidates,
+            },
+            {
+                resolveWildcardValidator:
+                    createDefaultWildcardValidatorResolver(),
+            },
+        );
         runtimeCache.set(key, runtime);
     }
     return runtime;
diff --git a/ts/packages/studio-service/src/wildcardValidation.ts b/ts/packages/studio-service/src/wildcardValidation.ts
new file mode 100644
index 0000000000..b8a6b924f4
--- /dev/null
+++ b/ts/packages/studio-service/src/wildcardValidation.ts
@@ -0,0 +1,123 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+/**
+ * Production wiring for the replay wildcard validator (fidelity rung L4a).
+ *
+ * The core `@typeagent/core` runtime exposes the validation algorithm but is
+ * dependency-light and does NOT know how to load a real agent module. The real
+ * loader lives in `default-agent-provider` (the dispatcher's agent providers),
+ * which drags in the whole dispatcher + every agent — far too heavy to bundle
+ * into the standalone, esbuild-bundled Studio service.
+ *
+ * So we load it **lazily, by dynamic import, marked external in the service
+ * bundle**: it resolves from `node_modules` on the in-repo `typeagent-studio
+ * serve` dev path, and cleanly throws → no-ops in the packaged `.vsix` (which
+ * ships without `node_modules`). Wildcard validation is opt-in and fail-open, so
+ * when the provider is unavailable replay simply stays grammar-only.
+ */
+
+import {
+    createWildcardMatchValidator,
+    DEFAULT_WILDCARD_VALIDATION_ALLOWLIST,
+    type ReplayAppAgentLoader,
+    type ReplayValidatableAgent,
+    type WildcardMatchValidator,
+} from "@typeagent/core/runtime";
+
+/** Minimal structural view of an `AppAgentProvider` (avoids a type-only dep). */
+interface AppAgentProviderLike {
+    getAppAgentNames(): string[];
+    loadAppAgent(agentName: string): Promise<ReplayValidatableAgent>;
+    unloadAppAgent(agentName: string): Promise<void>;
+}
+
+interface DefaultAgentProviderModule {
+    getDefaultAppAgentProviders(
+        instanceDir: string | undefined,
+    ): AppAgentProviderLike[];
+}
+
+let providersPromise: Promise<AppAgentProviderLike[] | undefined> | undefined;
+
+/**
+ * Lazily import `default-agent-provider` and build the default providers once.
+ * Returns `undefined` (cached) when the module can't be resolved — e.g. inside
+ * the packaged extension where `node_modules` was stripped — so validation
+ * silently degrades to grammar-only instead of erroring.
+ */
+async function loadProviders(): Promise<AppAgentProviderLike[] | undefined> {
+    if (providersPromise === undefined) {
+        providersPromise = (async () => {
+            try {
+                const mod = (await import(
+                    "default-agent-provider"
+                )) as DefaultAgentProviderModule;
+                return mod.getDefaultAppAgentProviders(undefined);
+            } catch {
+                return undefined;
+            }
+        })();
+    }
+    return providersPromise;
+}
+
+function findProvider(
+    providers: AppAgentProviderLike[],
+    agentName: string,
+): AppAgentProviderLike | undefined {
+    return providers.find((p) => {
+        try {
+            return p.getAppAgentNames().includes(agentName);
+        } catch {
+            return false;
+        }
+    });
+}
+
+/** A {@link ReplayAppAgentLoader} backed by the dispatcher's agent providers. */
+function createDefaultLoader(): ReplayAppAgentLoader {
+    return {
+        async loadAppAgent(agentName) {
+            const providers = await loadProviders();
+            if (providers === undefined) {
+                throw new Error(
+                    "default-agent-provider unavailable (packaged build?)",
+                );
+            }
+            const provider = findProvider(providers, agentName);
+            if (provider === undefined) {
+                throw new Error(`no provider owns agent "${agentName}"`);
+            }
+            return provider.loadAppAgent(agentName);
+        },
+        async unloadAppAgent(agentName) {
+            const providers = await loadProviders();
+            if (providers === undefined) {
+                return;
+            }
+            const provider = findProvider(providers, agentName);
+            await provider?.unloadAppAgent(agentName);
+        },
+    };
+}
+
+/**
+ * Build the runtime's `resolveWildcardValidator` option: returns a validator
+ * only for allowlisted agents (so a non-allowlisted agent stays grammar-only
+ * and the run doesn't claim a validation it can't perform), backed by the
+ * lazy default loader. The dynamic import is deferred until a wildcard match
+ * actually triggers `loadAppAgent`, so a run that never hits a wildcard never
+ * pays for the provider.
+ */
+export function createDefaultWildcardValidatorResolver(): (
+    agentName: string,
+) => WildcardMatchValidator | undefined {
+    const loader = createDefaultLoader();
+    return (agentName) => {
+        if (!DEFAULT_WILDCARD_VALIDATION_ALLOWLIST.includes(agentName)) {
+            return undefined;
+        }
+        return createWildcardMatchValidator(agentName, { loader });
+    };
+}
diff --git a/ts/packages/typeagent-core/src/replay/grammarResolver.ts b/ts/packages/typeagent-core/src/replay/grammarResolver.ts
index 82ab4481c3..a02d7d45d6 100644
--- a/ts/packages/typeagent-core/src/replay/grammarResolver.ts
+++ b/ts/packages/typeagent-core/src/replay/grammarResolver.ts
@@ -62,6 +62,7 @@ import type {
     ConstructionCacheLayer,
     ConstructionCacheStatus,
 } from "./constructionCacheResolver.js";
+import type { WildcardMatchValidator } from "./wildcardValidator.js";
 
 const execFileAsync = promisify(execFile);
 
@@ -177,6 +178,17 @@ export interface CreateGrammarReplayResolverOptions {
      * a git ref. A `stale`/`absent` layer leaves behavior at the grammar match.
      */
     constructionCache?: ConstructionCacheLayer;
+    /**
+     * Runs the agent's real `validateWildcardMatch` over each candidate grammar
+     * match that captured a wildcard, mirroring the dispatcher's post-match
+     * `getValidatedMatches`. When provided, the **working-tree** side validates
+     * the ranked match list and resolves the first match the agent accepts (a
+     * rejected wildcard match falls through, exactly as the dispatcher falls back
+     * to a lower match / the LLM). Working-tree only; never run at a git ref
+     * (loading arbitrary-ref agent code is out of scope). Fail-open: only an
+     * explicit `false` drops a match.
+     */
+    wildcardValidator?: WildcardMatchValidator;
 }
 
 /**
@@ -204,6 +216,13 @@ export interface GrammarReplayResolver extends ReplayActionResolver {
      * label; `"stale"`/`"absent"` fall back to the grammar method label.
      */
     readonly constructionCacheStatus: ConstructionCacheStatus | undefined;
+    /**
+     * Whether the wildcard validator actually ran on at least one match this run
+     * (i.e. a wildcard match occurred on the working-tree side and the validator
+     * was consulted). Drives the run's "+ wildcard validation" reporting. Always
+     * `false` when no validator was supplied.
+     */
+    readonly wildcardValidationApplied: boolean;
 }
 
 let entitiesRegistered = false;
@@ -387,6 +406,52 @@ function topMatchAction(results: MatchResult[]): unknown {
     return actions.length > 0 ? actions[0].action : undefined;
 }
 
+/**
+ * Mirror the dispatcher's `getValidatedMatches` for the wildcard step: walk the
+ * heuristically-ranked matches and return the action of the first one the agent
+ * accepts. A match with no wildcard capture (`wildcardCharCount === 0`) is
+ * accepted without consulting the validator (exactly as the dispatcher
+ * short-circuits); a wildcard match is dropped only on an explicit `false`
+ * verdict, and the walk continues to the next candidate (the dispatcher's
+ * fall-back-to-a-lower-match behavior). Returns `undefined` when every candidate
+ * was rejected — the row then becomes `needs-explanation`, the deterministic
+ * stand-in for the dispatcher falling back to the LLM.
+ */
+/**
+ * Mirror the dispatcher's `getValidatedMatches` for the wildcard step: walk the
+ * heuristically-ranked matches and return the action of the first one the agent
+ * accepts. A match with no wildcard capture (`wildcardCharCount === 0`) is
+ * accepted without consulting the validator (exactly as the dispatcher
+ * short-circuits); a wildcard match is dropped only on an explicit `false`
+ * verdict, and the walk continues to the next candidate (the dispatcher's
+ * fall-back-to-a-lower-match behavior). Returns `action: undefined` when every
+ * candidate was rejected — the row then becomes `needs-explanation`, the
+ * deterministic stand-in for the dispatcher falling back to the LLM.
+ *
+ * Exported for unit testing the ranked-selection contract.
+ */
+export async function selectValidatedMatchAction(
+    results: MatchResult[],
+    validator: WildcardMatchValidator,
+): Promise<{ action: unknown; consulted: boolean }> {
+    let consulted = false;
+    for (const result of results) {
+        const actions = result.match.actions;
+        if (actions.length === 0) {
+            continue;
+        }
+        if (result.wildcardCharCount === 0) {
+            return { action: actions[0].action, consulted };
+        }
+        consulted = true;
+        const outcome = await validator.validateMatch(actions);
+        if (!outcome.rejected) {
+            return { action: actions[0].action, consulted };
+        }
+    }
+    return { action: undefined, consulted };
+}
+
 /**
  * Build a {@link GrammarReplayResolver}. Construct ONE per replay run so the
  * per-side grammar store never outlives a single `(versionA, versionB)` pair.
@@ -397,6 +462,8 @@ export function createGrammarReplayResolver(
     const { target } = opts;
     const now = opts.now ?? (() => Date.now());
     const constructionCache = opts.constructionCache;
+    const wildcardValidator = opts.wildcardValidator;
+    let wildcardValidationApplied = false;
     // Construction-cache consult is faithful to the dispatcher only for the live
     // working tree, and only when the cache's namespace hash still matches.
     const constructionActive =
@@ -431,6 +498,10 @@ export function createGrammarReplayResolver(
             return constructionCache?.status;
         },
 
+        get wildcardValidationApplied(): boolean {
+            return wildcardValidationApplied;
+        },
+
         async prepare(versionA, versionB): Promise<void> {
             // Build A then B so the first failure (by side) is the one reported.
             const a = await build(versionA, "A");
@@ -483,13 +554,35 @@ export function createGrammarReplayResolver(
             }
 
             const results = g.store.match(entry.utterance);
+
+            // Wildcard validation (L4a): on the working-tree side, run the
+            // agent's real `validateWildcardMatch` over the ranked candidates and
+            // resolve the first accepted match — mirroring the dispatcher's
+            // post-match `getValidatedMatches`. Never on a git ref (we can't load
+            // arbitrary-ref agent code). When no validator is supplied this is a
+            // plain top-match resolution (unchanged behavior).
+            const validateHere =
+                wildcardValidator !== undefined &&
+                version.kind === "workingTree";
+            let rawAction: unknown;
+            if (validateHere) {
+                const validated = await selectValidatedMatchAction(
+                    results,
+                    wildcardValidator!,
+                );
+                rawAction = validated.action;
+                if (validated.consulted) {
+                    wildcardValidationApplied = true;
+                }
+            } else {
+                rawAction =
+                    results.length > 0 ? topMatchAction(results) : undefined;
+            }
+
             const latencyMs = now() - t0;
             const action =
-                results.length > 0
-                    ? normalizeAction(
-                          target.schemaName,
-                          topMatchAction(results),
-                      )
+                rawAction !== undefined
+                    ? normalizeAction(target.schemaName, rawAction)
                     : undefined;
 
             if (action === undefined) {
diff --git a/ts/packages/typeagent-core/src/replay/wildcardValidator.ts b/ts/packages/typeagent-core/src/replay/wildcardValidator.ts
new file mode 100644
index 0000000000..9740b8f247
--- /dev/null
+++ b/ts/packages/typeagent-core/src/replay/wildcardValidator.ts
@@ -0,0 +1,254 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+/**
+ * Wildcard-match validation for the replay path (fidelity rung "L4a").
+ *
+ * The dispatcher's only beyond-grammar determinism is a post-match step: for
+ * each candidate match that captured a wildcard, it calls the agent's
+ * `validateWildcardMatch(action, sessionContext)` and DROPS the match if the
+ * agent returns `false` (then falls back to a lower-ranked match or the LLM).
+ * See `dispatcher/.../translation/matchRequest.ts` `getValidatedMatches`.
+ *
+ * Grammar matching alone cannot reproduce this, so a `.agr` that still matches
+ * an utterance can look unchanged in replay even though the real dispatcher
+ * would reject the wildcard value. This module runs the agent's REAL validator
+ * so that fidelity axis shows up in the Impact Report.
+ *
+ * ## Scope & honesty (deliberate, see `files/replay-l4a-design.md`)
+ * - **Working-tree side only.** Loading arbitrary git-ref agent code is L4b
+ *   build machinery; the caller never runs this for a git ref.
+ * - **Built module.** {@link ReplayAppAgentLoader.loadAppAgent} returns the
+ *   agent's BUILT module, so uncommitted validator `.ts` edits are reflected
+ *   only after a rebuild (grammar edits — the usual regression experiment — are
+ *   already read from source by the grammar resolver).
+ * - **Fail-OPEN.** Only an explicit `=== false` from the agent rejects a match.
+ *   A missing method / load failure / thrown validator / non-allowlisted agent
+ *   all ACCEPT the match and record a diagnostic — replay must never fabricate a
+ *   lost match from infrastructure noise.
+ * - **Allowlist.** Only validators known to be deterministic + side-effect-free
+ *   run by default ({@link DEFAULT_WILDCARD_VALIDATION_ALLOWLIST}); e.g.
+ *   `weather`'s validator does live network geocoding and is intentionally
+ *   excluded.
+ * - **Entity-wildcard validation is out of scope** (it needs conversation
+ *   memory replay does not have).
+ */
+
+/**
+ * Minimal structural view of an app agent for wildcard validation. Typed
+ * structurally (rather than importing `@typeagent/agent-sdk`) so the
+ * dependency-light core package does not take on the agent SDK; the real
+ * `AppAgent.validateWildcardMatch` satisfies this shape.
+ */
+export interface ReplayValidatableAgent {
+    validateWildcardMatch?(
+        action: unknown,
+        context: unknown,
+    ): Promise<boolean> | boolean;
+}
+
+/**
+ * Loads (and optionally unloads) a built app-agent module by name. The default
+ * production implementation wraps the dispatcher's `getDefaultAppAgentProviders`
+ * (which lives outside this dependency-light package), so the loader is injected
+ * by the host. Tests inject a fake.
+ */
+export interface ReplayAppAgentLoader {
+    loadAppAgent(agentName: string): Promise<ReplayValidatableAgent>;
+    /** Best-effort cleanup; many agents default to `execMode: "separate"`. */
+    unloadAppAgent?(agentName: string): Promise<void>;
+}
+
+/**
+ * Why a match was accepted without a genuine `validateWildcardMatch === false`
+ * verdict — recorded so the run can honestly report that validation degraded.
+ */
+export type WildcardValidationDiagnostic =
+    | "agent-not-in-allowlist"
+    | "no-validator"
+    | "load-failed"
+    | "errored";
+
+export interface WildcardValidationOutcome {
+    /** True ONLY when the agent's validator returned an explicit `false`. */
+    rejected: boolean;
+    /** Set when the verdict was fail-open rather than a real validator result. */
+    diagnostic?: WildcardValidationDiagnostic;
+}
+
+export interface WildcardMatchValidator {
+    /**
+     * Run the agent's validator over every action in one candidate match,
+     * mirroring the dispatcher's `validateWildcardMatch`: reject as soon as any
+     * action's validator returns `false`; otherwise accept. Fail-open on any
+     * error (records a diagnostic, returns `rejected: false`).
+     */
+    validateMatch(
+        actions: readonly unknown[],
+    ): Promise<WildcardValidationOutcome>;
+    /** Whether the agent module was actually loaded (drives unload + reporting). */
+    readonly loaded: boolean;
+    /** All fail-open reasons seen this run (for the run-level summary/tooltip). */
+    readonly diagnostics: ReadonlySet<WildcardValidationDiagnostic>;
+    /** Best-effort `unloadAppAgent`; safe to call when nothing was loaded. */
+    dispose(): Promise<void>;
+}
+
+/**
+ * Agents whose `validateWildcardMatch` is a pure/deterministic function of the
+ * action (no network, no external process, no live session state):
+ * - `timer` — `tryParseWhen` on the action params; context ignored.
+ * - `list` — `simpleNoun` heuristic over the action; context ignored.
+ * - `player` — reads `context.agentContext.spotify`, but returns `true` when it
+ *   is absent (our stub context has none), faithfully self-degrading like a
+ *   dispatcher session with no Spotify login.
+ *
+ * `weather`/`markdown`/`photo`/`androidMobile`/`taskflow`/`powershell` are NOT
+ * here: weather geocodes over the network, and the others are unverified.
+ */
+export const DEFAULT_WILDCARD_VALIDATION_ALLOWLIST: readonly string[] = [
+    "timer",
+    "list",
+    "player",
+];
+
+export interface CreateWildcardMatchValidatorOptions {
+    loader: ReplayAppAgentLoader;
+    /** Defaults to {@link DEFAULT_WILDCARD_VALIDATION_ALLOWLIST}. */
+    allowlist?: readonly string[];
+}
+
+/**
+ * A no-op `SessionContext`-shaped stub for replay validation. The few validators
+ * we allowlist either ignore the context (timer/list) or read only
+ * `agentContext` (player, which self-degrades when its client is absent). The
+ * object exposes `agentContext: {}` — NOT `undefined`, which would make
+ * `player`'s `context.agentContext.spotify` throw before its fallback — and the
+ * interactive methods throw, since a validator that calls them is out of the
+ * supported set and should fail-open via the wrapper's try/catch.
+ */
+export function createReplaySessionContextStub(): Record<string, unknown> {
+    const unsupported = (name: string) => () => {
+        throw new Error(
+            `SessionContext.${name} is not available during replay wildcard validation`,
+        );
+    };
+    return {
+        agentContext: {},
+        sessionStorage: undefined,
+        instanceStorage: undefined,
+        sessionContextId: "studio-replay-validation",
+        notify: () => {},
+        beginAgentThread: unsupported("beginAgentThread"),
+        popupQuestion: unsupported("popupQuestion"),
+        toggleTransientAgent: unsupported("toggleTransientAgent"),
+        addDynamicAgent: unsupported("addDynamicAgent"),
+        removeDynamicAgent: unsupported("removeDynamicAgent"),
+        forceCleanupDynamicAgent: unsupported("forceCleanupDynamicAgent"),
+        reloadAgentSchema: unsupported("reloadAgentSchema"),
+    };
+}
+
+/** Pull the `action` payloads out of a grammar match's `actions` array. */
+export function matchActionPayloads(actions: readonly unknown[]): unknown[] {
+    return actions.map((a) =>
+        a !== null && typeof a === "object" && "action" in a
+            ? (a as { action: unknown }).action
+            : a,
+    );
+}
+
+/**
+ * Build a {@link WildcardMatchValidator} for one agent. The agent module is
+ * loaded lazily on the first {@link WildcardMatchValidator.validateMatch} that
+ * actually needs it (so a run that never hits a wildcard match never loads the
+ * module), then cached for the rest of the run.
+ */
+export function createWildcardMatchValidator(
+    agentName: string,
+    options: CreateWildcardMatchValidatorOptions,
+): WildcardMatchValidator {
+    const { loader } = options;
+    const allowlist =
+        options.allowlist ?? DEFAULT_WILDCARD_VALIDATION_ALLOWLIST;
+    const allowed = allowlist.includes(agentName);
+    const diagnostics = new Set<WildcardValidationDiagnostic>();
+    const stubContext = createReplaySessionContextStub();
+
+    // null = not loaded yet; { agent } once a load attempt resolved (agent may
+    // be undefined when the load failed, so we never retry a failed load).
+    let loadState: { agent: ReplayValidatableAgent | undefined } | undefined;
+    let didLoad = false;
+
+    async function ensureAgent(): Promise<ReplayValidatableAgent | undefined> {
+        if (loadState !== undefined) {
+            return loadState.agent;
+        }
+        try {
+            const agent = await loader.loadAppAgent(agentName);
+            didLoad = true;
+            loadState = { agent };
+            return agent;
+        } catch {
+            diagnostics.add("load-failed");
+            loadState = { agent: undefined };
+            return undefined;
+        }
+    }
+
+    function note(d: WildcardValidationDiagnostic): WildcardValidationOutcome {
+        diagnostics.add(d);
+        return { rejected: false, diagnostic: d };
+    }
+
+    return {
+        get loaded(): boolean {
+            return didLoad;
+        },
+        get diagnostics(): ReadonlySet<WildcardValidationDiagnostic> {
+            return diagnostics;
+        },
+
+        async validateMatch(actions): Promise<WildcardValidationOutcome> {
+            if (!allowed) {
+                return note("agent-not-in-allowlist");
+            }
+            const agent = await ensureAgent();
+            if (agent === undefined) {
+                return { rejected: false, diagnostic: "load-failed" };
+            }
+            if (typeof agent.validateWildcardMatch !== "function") {
+                return note("no-validator");
+            }
+            for (const action of matchActionPayloads(actions)) {
+                let verdict: boolean;
+                try {
+                    verdict = await agent.validateWildcardMatch(
+                        action,
+                        stubContext,
+                    );
+                } catch {
+                    // Fail-open: a throwing validator must never fabricate a
+                    // lost match. Record it and accept.
+                    diagnostics.add("errored");
+                    continue;
+                }
+                if (verdict === false) {
+                    return { rejected: true };
+                }
+            }
+            return { rejected: false };
+        },
+
+        async dispose(): Promise<void> {
+            if (!didLoad || loader.unloadAppAgent === undefined) {
+                return;
+            }
+            try {
+                await loader.unloadAppAgent(agentName);
+            } catch {
+                // Best-effort cleanup.
+            }
+        },
+    };
+}
diff --git a/ts/packages/typeagent-core/src/runtime/index.ts b/ts/packages/typeagent-core/src/runtime/index.ts
index 05cd0eb224..221fa9f385 100644
--- a/ts/packages/typeagent-core/src/runtime/index.ts
+++ b/ts/packages/typeagent-core/src/runtime/index.ts
@@ -16,3 +16,21 @@ export * from "./studioRegistryProtocol.js";
 export * from "./studioServiceAuth.js";
 export * from "./studioWorkspaceId.js";
 export { getDefaultPhaseInputs } from "./onboardingPhaseInputs.js";
+
+// Wildcard-match validation (replay fidelity rung L4a). Re-exported here so the
+// service host can wire a real `ReplayAppAgentLoader` (the production loader
+// lives outside this dependency-light package) without reaching into `replay/`.
+export {
+    createWildcardMatchValidator,
+    createReplaySessionContextStub,
+    matchActionPayloads,
+    DEFAULT_WILDCARD_VALIDATION_ALLOWLIST,
+} from "../replay/wildcardValidator.js";
+export type {
+    ReplayAppAgentLoader,
+    ReplayValidatableAgent,
+    WildcardMatchValidator,
+    WildcardValidationOutcome,
+    WildcardValidationDiagnostic,
+    CreateWildcardMatchValidatorOptions,
+} from "../replay/wildcardValidator.js";
diff --git a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
index 2371d6b6b0..16f521f058 100644
--- a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
+++ b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
@@ -63,12 +63,17 @@ import {
     ReplayVersionBuildError,
     type ReplayRunError,
     type GrammarReplayTarget,
+    type GrammarReplayResolver,
 } from "../replay/grammarResolver.js";
 import {
     computeWorkingTreeSchemaHash,
     loadConstructionCacheLayer,
     type ConstructionCacheLayer,
 } from "../replay/constructionCacheResolver.js";
+import type {
+    WildcardMatchValidator,
+    WildcardValidationDiagnostic,
+} from "../replay/wildcardValidator.js";
 import type { CollisionDetectedEvent } from "../events/index.js";
 import { getDefaultPhaseInputs } from "./onboardingPhaseInputs.js";
 import {
@@ -258,6 +263,15 @@ export interface StudioReplayRequest {
     missPolicy?: ReplayMissPolicy;
     /** Deterministic dispatch path to model. Defaults to `nfa-grammar`. */
     mode?: StudioReplayMode;
+    /**
+     * Opt-in: additionally run the agent's real `validateWildcardMatch` over the
+     * working-tree side's wildcard grammar matches (the dispatcher's post-match
+     * validation step). Default off. Only takes effect when the runtime was given
+     * a `resolveWildcardValidator` and the agent is in the validation allowlist;
+     * otherwise it is a no-op. Working-tree side only; fail-open. See
+     * {@link StudioReplayResult.wildcardValidation}.
+     */
+    validateWildcards?: boolean;
 }
 
 /**
@@ -302,6 +316,12 @@ export interface StudioReplayResult {
      */
     methodA: StudioReplayMethod;
     methodB: StudioReplayMethod;
+    /**
+     * Set when the opt-in wildcard validation actually consulted the agent's
+     * `validateWildcardMatch` (a wildcard match occurred on the working-tree
+     * side). Omitted when validation was off or never reached a wildcard match.
+     */
+    wildcardValidation?: StudioWildcardValidationInfo;
     /**
      * Set when a version failed to materialize or compile. The run is aborted
      * with an empty summary rather than emitting fabricated regression rows.
@@ -309,6 +329,18 @@ export interface StudioReplayResult {
     error?: ReplayRunError;
 }
 
+/**
+ * Reports the opt-in wildcard-validation pass (replay fidelity rung L4a). Only
+ * present when validation was enabled AND consulted on at least one wildcard
+ * match. `diagnostics` lists fail-open reasons (e.g. the agent's module
+ * couldn't load, or its validator threw), so the UI can show that validation
+ * degraded rather than silently claiming full fidelity.
+ */
+export interface StudioWildcardValidationInfo {
+    applied: boolean;
+    diagnostics: WildcardValidationDiagnostic[];
+}
+
 export interface StudioCollisionScanRequest {
     /**
      * Agent package names to scan. Defaults to every agent currently loaded
@@ -572,6 +604,18 @@ export interface CreateStudioRuntimeOptions {
     resolveConstructionCache?: (
         target: GrammarReplayTarget,
     ) => Promise<ConstructionCacheLayer | undefined>;
+    /**
+     * Builds the agent's wildcard-match validator for an opt-in
+     * (`validateWildcards`) replay run. Returns `undefined` when the host can't
+     * validate this agent (e.g. not in the allowlist, or no agent loader
+     * available), in which case replay stays grammar-only. Injected because the
+     * production loader pulls the dispatcher's agent providers, which live
+     * outside this dependency-light package; tests inject a fake-loader-backed
+     * validator. The runtime disposes the returned validator at run end.
+     */
+    resolveWildcardValidator?: (
+        agentName: string,
+    ) => WildcardMatchValidator | undefined;
     collisions?: CollisionService;
     /**
      * Scans agents' compiled grammars for collisions. Injected so tests can
@@ -1039,6 +1083,10 @@ export function createStudioRuntimeCore(
             let method: StudioReplayMethod = "identity";
             let methodA: StudioReplayMethod = "identity";
             let methodB: StudioReplayMethod = "identity";
+            // Opt-in wildcard validator (L4a); kept in scope so its diagnostics
+            // can be read after the run and it can be disposed (unload the agent).
+            let wildcardValidator: WildcardMatchValidator | undefined;
+            let activeGrammarResolver: GrammarReplayResolver | undefined;
             if (
                 resolver === undefined &&
                 replayOptions.missPolicy === "needs-explanation" &&
@@ -1067,13 +1115,29 @@ export function createStudioRuntimeCore(
                               )(target)
                             : undefined;
 
+                    // Opt-in: build the agent's wildcard validator so the
+                    // working-tree side runs the dispatcher's real post-match
+                    // `validateWildcardMatch`. `undefined` when the host has no
+                    // loader or the agent isn't validatable — replay then stays
+                    // grammar-only (a silent no-op, not an error).
+                    wildcardValidator =
+                        request.validateWildcards === true
+                            ? options.resolveWildcardValidator?.(
+                                  replayOptions.agent,
+                              )
+                            : undefined;
+
                     const grammarResolver = createGrammarReplayResolver({
                         target,
                         repoRoot,
                         ...(constructionCache !== undefined
                             ? { constructionCache }
                             : {}),
+                        ...(wildcardValidator !== undefined
+                            ? { wildcardValidator }
+                            : {}),
                     });
+                    activeGrammarResolver = grammarResolver;
                     const labelFor = (): StudioReplayMethod =>
                         grammarResolver.constructionCacheStatus === "valid"
                             ? "construction-cache"
@@ -1116,6 +1180,7 @@ export function createStudioRuntimeCore(
                                   ? "schema-grammar"
                                   : "static-grammar";
                     } catch (err) {
+                        await wildcardValidator?.dispose();
                         if (err instanceof ReplayVersionBuildError) {
                             return abortedReplayResult(
                                 replayOptions,
@@ -1143,10 +1208,26 @@ export function createStudioRuntimeCore(
             });
 
             const rows: ActionDelta[] = [];
-            for await (const row of handle.rows) {
-                rows.push(row);
+            try {
+                for await (const row of handle.rows) {
+                    rows.push(row);
+                }
+            } finally {
+                await wildcardValidator?.dispose();
             }
             const summary = await handle.summary;
+            // Surface the wildcard-validation pass only when it actually ran on a
+            // wildcard match (so a run that never hit a wildcard doesn't claim a
+            // validation it never performed).
+            const wildcardValidation: StudioWildcardValidationInfo | undefined =
+                activeGrammarResolver?.wildcardValidationApplied === true
+                    ? {
+                          applied: true,
+                          diagnostics: [
+                              ...(wildcardValidator?.diagnostics ?? []),
+                          ],
+                      }
+                    : undefined;
             return {
                 runId: handle.runId,
                 summary,
@@ -1154,6 +1235,9 @@ export function createStudioRuntimeCore(
                 method,
                 methodA,
                 methodB,
+                ...(wildcardValidation !== undefined
+                    ? { wildcardValidation }
+                    : {}),
             };
         },
         reportCollision(event) {
diff --git a/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts b/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts
index f2c47d9b9c..4e01abe32b 100644
--- a/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts
+++ b/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts
@@ -17,9 +17,17 @@ import {
     normalizeGrammarAction,
     resolveGrammarReplayTarget,
     ReplayVersionBuildError,
+    selectValidatedMatchAction,
     type GrammarReplayTarget,
 } from "../src/replay/grammarResolver.js";
 import type { ConstructionCacheLayer } from "../src/replay/constructionCacheResolver.js";
+import {
+    createWildcardMatchValidator,
+    type ReplayAppAgentLoader,
+    type ReplayValidatableAgent,
+    type WildcardMatchValidator,
+} from "../src/replay/wildcardValidator.js";
+import type { MatchResult } from "agent-cache";
 
 const GRAMMAR_V1 = [
     "<Start> = <Pause> | <Resume>;",
@@ -490,3 +498,211 @@ describe("resolveGrammarReplayTarget + schema enrichment (L1)", () => {
         });
     });
 });
+
+// ---------------------------------------------------------------------------
+// L4a — working-tree wildcard validation
+// ---------------------------------------------------------------------------
+
+// A grammar whose <PlayTrack> rule captures a greedy wildcard (so a match has
+// wildcardCharCount > 0 and is subject to validateWildcardMatch), plus a
+// non-wildcard <Pause> rule (validator never consulted).
+const WILDCARD_GRAMMAR = [
+    "<Start> = <PlayTrack> | <Pause>;",
+    '<PlayTrack> = play $(trackName:wildcard) -> { actionName: "playTrack", parameters: { trackName } };',
+    '<Pause> = pause -> { actionName: "pause" };',
+].join("\n");
+
+function wildcardLoader(agent: ReplayValidatableAgent): ReplayAppAgentLoader {
+    return {
+        async loadAppAgent() {
+            return agent;
+        },
+        async unloadAppAgent() {},
+    };
+}
+
+/** A validator (allowlisted for "demo") that rejects a given track name. */
+function rejectTrackValidator(reject: string): WildcardMatchValidator {
+    const agent: ReplayValidatableAgent = {
+        async validateWildcardMatch(a) {
+            const params = (a as { parameters?: { trackName?: string } })
+                .parameters;
+            return params?.trackName !== reject;
+        },
+    };
+    return createWildcardMatchValidator("demo", {
+        loader: wildcardLoader(agent),
+        allowlist: ["demo"],
+    });
+}
+
+describe("grammar resolver wildcard validation (L4a)", () => {
+    let dir: string;
+
+    beforeEach(() => {
+        dir = tempDir();
+    });
+
+    afterEach(() => {
+        rmSync(dir, { recursive: true, force: true });
+    });
+
+    function wildcardTarget(): GrammarReplayTarget {
+        const grammarPath = path.join(dir, "demo.agr");
+        writeFileSync(grammarPath, WILDCARD_GRAMMAR, "utf8");
+        return {
+            agent: "demo",
+            schemaName: "demo",
+            grammarFilePath: grammarPath,
+        };
+    }
+
+    it("drops a wildcard match the agent rejects (→ needs-explanation)", async () => {
+        const validator = rejectTrackValidator("despacito");
+        const resolver = createGrammarReplayResolver({
+            target: wildcardTarget(),
+            wildcardValidator: validator,
+        });
+        await resolver.prepare(
+            { kind: "workingTree" },
+            { kind: "workingTree" },
+        );
+        const res = await resolver.resolve(
+            entry("e1", "play despacito"),
+            { kind: "workingTree" },
+            "A",
+        );
+        expect(res.cacheState).toBe("needs-explanation");
+        expect(res.action).toBeUndefined();
+        expect(resolver.wildcardValidationApplied).toBe(true);
+        await validator.dispose();
+    });
+
+    it("keeps a wildcard match the agent accepts", async () => {
+        const validator = rejectTrackValidator("nope");
+        const resolver = createGrammarReplayResolver({
+            target: wildcardTarget(),
+            wildcardValidator: validator,
+        });
+        await resolver.prepare(
+            { kind: "workingTree" },
+            { kind: "workingTree" },
+        );
+        const res = await resolver.resolve(
+            entry("e1", "play despacito"),
+            { kind: "workingTree" },
+            "A",
+        );
+        expect(res.cacheState).toBe("hit");
+        expect(res.action).toMatchObject({
+            schemaName: "demo",
+            actionName: "playTrack",
+            parameters: { trackName: "despacito" },
+        });
+        await validator.dispose();
+    });
+
+    it("does not consult the validator for a non-wildcard match", async () => {
+        const validator = rejectTrackValidator("despacito");
+        const resolver = createGrammarReplayResolver({
+            target: wildcardTarget(),
+            wildcardValidator: validator,
+        });
+        await resolver.prepare(
+            { kind: "workingTree" },
+            { kind: "workingTree" },
+        );
+        const res = await resolver.resolve(
+            entry("e1", "pause"),
+            { kind: "workingTree" },
+            "A",
+        );
+        expect(res.cacheState).toBe("hit");
+        expect(res.action).toEqual({ schemaName: "demo", actionName: "pause" });
+        // No wildcard match occurred, so validation never "applied".
+        expect(resolver.wildcardValidationApplied).toBe(false);
+        await validator.dispose();
+    });
+
+    it("behaves as plain top-match resolution when no validator is supplied", async () => {
+        const resolver = createGrammarReplayResolver({
+            target: wildcardTarget(),
+        });
+        await resolver.prepare(
+            { kind: "workingTree" },
+            { kind: "workingTree" },
+        );
+        const res = await resolver.resolve(
+            entry("e1", "play despacito"),
+            { kind: "workingTree" },
+            "A",
+        );
+        expect(res.cacheState).toBe("hit");
+        expect(res.action).toMatchObject({ actionName: "playTrack" });
+        expect(resolver.wildcardValidationApplied).toBe(false);
+    });
+});
+
+describe("selectValidatedMatchAction (ranked-list contract)", () => {
+    function match(actionName: string, wildcardCharCount: number): MatchResult {
+        return {
+            type: "grammar",
+            match: {
+                actions: [{ action: { schemaName: "demo", actionName } }],
+            },
+            matchedCount: 1,
+            wildcardCharCount,
+            nonOptionalCount: 0,
+            implicitParameterCount: 0,
+            entityWildcardPropertyNames: [],
+        } as unknown as MatchResult;
+    }
+
+    function rejectingValidator(
+        rejectActionNames: string[],
+    ): WildcardMatchValidator {
+        const agent: ReplayValidatableAgent = {
+            async validateWildcardMatch(a) {
+                return !rejectActionNames.includes(
+                    (a as { actionName: string }).actionName,
+                );
+            },
+        };
+        return createWildcardMatchValidator("demo", {
+            loader: wildcardLoader(agent),
+            allowlist: ["demo"],
+        });
+    }
+
+    it("falls through a rejected top wildcard match to a surviving candidate", async () => {
+        // Top match (wildcard) is rejected; the 2nd wildcard match passes — so
+        // this is NOT a lost match, mirroring the dispatcher's fall-back.
+        const validator = rejectingValidator(["top"]);
+        const result = await selectValidatedMatchAction(
+            [match("top", 5), match("second", 3)],
+            validator,
+        );
+        expect(result.consulted).toBe(true);
+        expect(result.action).toMatchObject({ actionName: "second" });
+    });
+
+    it("returns no action when every wildcard candidate is rejected", async () => {
+        const validator = rejectingValidator(["a", "b"]);
+        const result = await selectValidatedMatchAction(
+            [match("a", 2), match("b", 4)],
+            validator,
+        );
+        expect(result.action).toBeUndefined();
+        expect(result.consulted).toBe(true);
+    });
+
+    it("accepts a non-wildcard match without consulting the validator", async () => {
+        const validator = rejectingValidator(["top"]);
+        const result = await selectValidatedMatchAction(
+            [match("top", 0)],
+            validator,
+        );
+        expect(result.action).toMatchObject({ actionName: "top" });
+        expect(result.consulted).toBe(false);
+    });
+});
diff --git a/ts/packages/typeagent-core/test/wildcardValidator.spec.ts b/ts/packages/typeagent-core/test/wildcardValidator.spec.ts
new file mode 100644
index 0000000000..c3b9a2437d
--- /dev/null
+++ b/ts/packages/typeagent-core/test/wildcardValidator.spec.ts
@@ -0,0 +1,217 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import {
+    createWildcardMatchValidator,
+    createReplaySessionContextStub,
+    matchActionPayloads,
+    DEFAULT_WILDCARD_VALIDATION_ALLOWLIST,
+    type ReplayAppAgentLoader,
+    type ReplayValidatableAgent,
+} from "../src/replay/wildcardValidator.js";
+
+/** A loader over a fixed map of fake agents, recording load/unload calls. */
+function fakeLoader(
+    agents: Record<string, ReplayValidatableAgent>,
+    log?: { loaded: string[]; unloaded: string[] },
+): ReplayAppAgentLoader {
+    return {
+        async loadAppAgent(name) {
+            log?.loaded.push(name);
+            const agent = agents[name];
+            if (agent === undefined) {
+                throw new Error(`no such agent: ${name}`);
+            }
+            return agent;
+        },
+        async unloadAppAgent(name) {
+            log?.unloaded.push(name);
+        },
+    };
+}
+
+/** One grammar-match action wrapper `{ action }`, as `match.actions` carries. */
+function action(actionName: string, parameters?: Record<string, unknown>) {
+    return {
+        action: {
+            schemaName: "demo",
+            actionName,
+            ...(parameters !== undefined ? { parameters } : {}),
+        },
+    };
+}
+
+describe("createWildcardMatchValidator", () => {
+    it("accepts (no-op) and diagnoses when the agent is not in the allowlist", async () => {
+        const log = { loaded: [] as string[], unloaded: [] as string[] };
+        const validator = createWildcardMatchValidator("notallowed", {
+            loader: fakeLoader({ notallowed: {} }, log),
+            allowlist: ["timer"],
+        });
+        const outcome = await validator.validateMatch([action("doThing")]);
+        expect(outcome.rejected).toBe(false);
+        expect(outcome.diagnostic).toBe("agent-not-in-allowlist");
+        // Never even loaded the module.
+        expect(log.loaded).toEqual([]);
+        expect(validator.loaded).toBe(false);
+        expect([...validator.diagnostics]).toContain("agent-not-in-allowlist");
+        await validator.dispose();
+        expect(log.unloaded).toEqual([]);
+    });
+
+    it("rejects when the agent validator returns false", async () => {
+        const agent: ReplayValidatableAgent = {
+            async validateWildcardMatch(a) {
+                return (a as { actionName: string }).actionName !== "bad";
+            },
+        };
+        const validator = createWildcardMatchValidator("timer", {
+            loader: fakeLoader({ timer: agent }),
+            allowlist: ["timer"],
+        });
+        expect((await validator.validateMatch([action("good")])).rejected).toBe(
+            false,
+        );
+        expect((await validator.validateMatch([action("bad")])).rejected).toBe(
+            true,
+        );
+    });
+
+    it("rejects the whole match if ANY action's validator returns false", async () => {
+        const agent: ReplayValidatableAgent = {
+            async validateWildcardMatch(a) {
+                return (a as { actionName: string }).actionName !== "bad";
+            },
+        };
+        const validator = createWildcardMatchValidator("timer", {
+            loader: fakeLoader({ timer: agent }),
+            allowlist: ["timer"],
+        });
+        const outcome = await validator.validateMatch([
+            action("good"),
+            action("bad"),
+        ]);
+        expect(outcome.rejected).toBe(true);
+    });
+
+    it("fails open with a diagnostic when the validator throws", async () => {
+        const agent: ReplayValidatableAgent = {
+            async validateWildcardMatch() {
+                throw new Error("boom");
+            },
+        };
+        const validator = createWildcardMatchValidator("timer", {
+            loader: fakeLoader({ timer: agent }),
+            allowlist: ["timer"],
+        });
+        const outcome = await validator.validateMatch([action("x")]);
+        expect(outcome.rejected).toBe(false);
+        expect([...validator.diagnostics]).toContain("errored");
+    });
+
+    it("fails open with a diagnostic when the module fails to load", async () => {
+        const validator = createWildcardMatchValidator("timer", {
+            // Loader has no "timer" → loadAppAgent throws.
+            loader: fakeLoader({}),
+            allowlist: ["timer"],
+        });
+        const outcome = await validator.validateMatch([action("x")]);
+        expect(outcome.rejected).toBe(false);
+        expect(outcome.diagnostic).toBe("load-failed");
+        expect([...validator.diagnostics]).toContain("load-failed");
+        // A failed load is not retried.
+        await validator.validateMatch([action("y")]);
+    });
+
+    it("accepts (no-op) when the agent has no validateWildcardMatch", async () => {
+        const validator = createWildcardMatchValidator("timer", {
+            loader: fakeLoader({ timer: {} }),
+            allowlist: ["timer"],
+        });
+        const outcome = await validator.validateMatch([action("x")]);
+        expect(outcome.rejected).toBe(false);
+        expect(outcome.diagnostic).toBe("no-validator");
+    });
+
+    it("does not throw for a player-style validator reading agentContext (empty stub)", async () => {
+        // Mirrors the real player validator: reads context.agentContext.spotify
+        // and returns true when the client is absent. Our stub uses
+        // `agentContext: {}` (NOT undefined), so this must not throw.
+        const playerLike: ReplayValidatableAgent = {
+            async validateWildcardMatch(_a, context) {
+                const spotify = (
+                    context as { agentContext?: { spotify?: unknown } }
+                ).agentContext?.spotify;
+                if (spotify === undefined) {
+                    return true;
+                }
+                return false;
+            },
+        };
+        const validator = createWildcardMatchValidator("player", {
+            loader: fakeLoader({ player: playerLike }),
+        });
+        const outcome = await validator.validateMatch([action("playTrack")]);
+        expect(outcome.rejected).toBe(false);
+        expect(outcome.diagnostic).toBeUndefined();
+    });
+
+    it("loads the module lazily and only once, then unloads on dispose", async () => {
+        const log = { loaded: [] as string[], unloaded: [] as string[] };
+        const agent: ReplayValidatableAgent = {
+            async validateWildcardMatch() {
+                return true;
+            },
+        };
+        const validator = createWildcardMatchValidator("timer", {
+            loader: fakeLoader({ timer: agent }, log),
+            allowlist: ["timer"],
+        });
+        // Not loaded until first validateMatch.
+        expect(log.loaded).toEqual([]);
+        await validator.validateMatch([action("a")]);
+        await validator.validateMatch([action("b")]);
+        expect(log.loaded).toEqual(["timer"]); // exactly once
+        expect(validator.loaded).toBe(true);
+        await validator.dispose();
+        expect(log.unloaded).toEqual(["timer"]);
+    });
+
+    it("dispose is a no-op when nothing was loaded", async () => {
+        const log = { loaded: [] as string[], unloaded: [] as string[] };
+        const validator = createWildcardMatchValidator("notallowed", {
+            loader: fakeLoader({}, log),
+            allowlist: ["timer"],
+        });
+        await validator.dispose();
+        expect(log.unloaded).toEqual([]);
+    });
+
+    it("ships timer/list/player as the default allowlist", () => {
+        expect([...DEFAULT_WILDCARD_VALIDATION_ALLOWLIST].sort()).toEqual([
+            "list",
+            "player",
+            "timer",
+        ]);
+    });
+});
+
+describe("createReplaySessionContextStub", () => {
+    it("exposes an empty (non-undefined) agentContext", () => {
+        const stub = createReplaySessionContextStub();
+        expect(stub.agentContext).toEqual({});
+    });
+
+    it("throws on interactive methods (validators must not call them)", () => {
+        const stub = createReplaySessionContextStub();
+        expect(() => (stub.popupQuestion as () => unknown)()).toThrow();
+    });
+});
+
+describe("matchActionPayloads", () => {
+    it("unwraps `{ action }` entries and passes through bare values", () => {
+        expect(
+            matchActionPayloads([{ action: { actionName: "x" } }, 42]),
+        ).toEqual([{ actionName: "x" }, 42]);
+    });
+});
diff --git a/ts/packages/typeagent-studio/esbuild.mjs b/ts/packages/typeagent-studio/esbuild.mjs
index 75af51206f..599c836d0a 100644
--- a/ts/packages/typeagent-studio/esbuild.mjs
+++ b/ts/packages/typeagent-studio/esbuild.mjs
@@ -100,7 +100,12 @@ const serviceConfig = {
     entryPoints: ["../studio-service/src/main.ts"],
     bundle: true,
     outfile: "dist/studio-service.js",
-    external: ["vscode"],
+    // `default-agent-provider` is dynamically imported by the replay wildcard
+    // validator (fidelity rung L4a). It drags in the whole dispatcher + every
+    // agent, so it must NOT be inlined here. Marked external, it resolves from
+    // `node_modules` on the in-repo `typeagent-studio serve` dev path and simply
+    // fails the dynamic import (→ grammar-only replay) in the packaged `.vsix`.
+    external: ["vscode", "default-agent-provider"],
     alias: nodeBundleAlias,
     format: "cjs",
     platform: "node",
diff --git a/ts/packages/typeagent-studio/media/impactReport.css b/ts/packages/typeagent-studio/media/impactReport.css
index c23c19e484..74bf893c45 100644
--- a/ts/packages/typeagent-studio/media/impactReport.css
+++ b/ts/packages/typeagent-studio/media/impactReport.css
@@ -307,6 +307,13 @@ body {
   background: var(--vscode-button-hoverBackground);
 }
 
+/* A lit two-state toggle (e.g. the wildcard-validation switch) when active. */
+.tool-button.is-active {
+  color: var(--vscode-inputOption-activeForeground, var(--vscode-foreground));
+  background: var(--vscode-inputOption-activeBackground);
+  border-color: var(--vscode-inputOption-activeBorder, transparent);
+}
+
 /* --- Sub-bar: provenance + run status ------------------------------------ */
 .sub-bar {
   display: flex;
@@ -348,6 +355,24 @@ body {
   text-overflow: ellipsis;
 }
 
+/* The wildcard-validation outcome of the last run; sits next to provenance.
+   Tinted by severity: a clean pass is neutral, a degraded/unavailable pass
+   warns so the report never silently claims a fidelity it didn't reach. */
+.validation-note {
+  display: inline-flex;
+  align-items: center;
+  gap: 5px;
+  white-space: nowrap;
+  color: var(--vscode-descriptionForeground);
+}
+
+.validation-note.is-degraded {
+  color: var(
+    --vscode-editorWarning-foreground,
+    var(--vscode-descriptionForeground)
+  );
+}
+
 /* --- Inline notification (errors) ---------------------------------------- */
 .notification {
   display: flex;
diff --git a/ts/packages/typeagent-studio/src/impactReportView.ts b/ts/packages/typeagent-studio/src/impactReportView.ts
index 5aaf6a38fd..8b64855330 100644
--- a/ts/packages/typeagent-studio/src/impactReportView.ts
+++ b/ts/packages/typeagent-studio/src/impactReportView.ts
@@ -364,6 +364,10 @@ export function openImpactReport(
                 // path to model (grammar-only vs construction-cache-first); the
                 // runtime defaults unknown/missing to the cache-free baseline.
                 mode: msg.mode,
+                // Opt-in wildcard validation (L4a): the runtime only acts on it
+                // when a `resolveWildcardValidator` is wired and the agent is in
+                // the validation allowlist; otherwise it is a no-op.
+                validateWildcards: msg.validateWildcards,
                 missPolicy: "needs-explanation",
             });
             post({
diff --git a/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts b/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts
index 9adeefdede..313163e4f8 100644
--- a/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts
+++ b/ts/packages/typeagent-studio/src/test/webviewProtocol.spec.ts
@@ -28,6 +28,7 @@ test("parseWebviewMessage accepts well-formed messages", () => {
             versionA: { kind: "workingTree" },
             versionB: { kind: "workingTree" },
             mode: "nfa-grammar",
+            validateWildcards: false,
         },
     );
     // String version fields are coerced (legacy text-field / test seam).
@@ -39,6 +40,7 @@ test("parseWebviewMessage accepts well-formed messages", () => {
             versionA: "HEAD",
             versionB: "working tree",
             mode: "completionBased-cache",
+            validateWildcards: true,
         }),
         {
             type: "run",
@@ -47,10 +49,12 @@ test("parseWebviewMessage accepts well-formed messages", () => {
             versionA: { kind: "git", ref: "HEAD" },
             versionB: { kind: "workingTree" },
             mode: "completionBased-cache",
+            validateWildcards: true,
         },
     );
     // Typed specs from a picker selection are validated and taken as-is; an
-    // unknown mode value falls back to the grammar-only baseline.
+    // unknown mode value falls back to the grammar-only baseline, and a
+    // non-boolean validateWildcards falls back to off.
     assert.deepEqual(
         parseWebviewMessage({
             type: "run",
@@ -59,6 +63,7 @@ test("parseWebviewMessage accepts well-formed messages", () => {
             versionA: { kind: "git", ref: "v1.0" },
             versionB: { kind: "workingTree" },
             mode: "bogus-mode",
+            validateWildcards: "yes",
         }),
         {
             type: "run",
@@ -67,6 +72,7 @@ test("parseWebviewMessage accepts well-formed messages", () => {
             versionA: { kind: "git", ref: "v1.0" },
             versionB: { kind: "workingTree" },
             mode: "nfa-grammar",
+            validateWildcards: false,
         },
     );
     // Non-string / malformed version fields fall back to the working tree.
@@ -85,6 +91,7 @@ test("parseWebviewMessage accepts well-formed messages", () => {
             versionA: { kind: "workingTree" },
             versionB: { kind: "workingTree" },
             mode: "nfa-grammar",
+            validateWildcards: false,
         },
     );
 });
diff --git a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
index c7436d19de..226bc39b41 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
@@ -13,6 +13,8 @@
 import type {
     StudioReplayResult,
     StudioReplayMode,
+    StudioWildcardValidationInfo,
+    WildcardValidationDiagnostic,
 } from "@typeagent/core/runtime";
 import type { ActionDelta } from "@typeagent/core/replay";
 import type {
@@ -68,6 +70,7 @@ interface PanelState {
     versionA?: ResolvedVersion;
     versionB?: ResolvedVersion;
     mode?: StudioReplayMode;
+    validateWildcards?: boolean;
     lastResult?: PersistedResult;
 }
 interface VsCodeApi {
@@ -124,6 +127,14 @@ let versionB: ResolvedVersion = DEFAULT_VERSION_B;
 // version selection so a reload keeps the chosen mode.
 let mode: StudioReplayMode = "nfa-grammar";
 
+// Opt-in wildcard validation (replay fidelity rung L4a). When lit, the run asks
+// the host to additionally run the agent's real `validateWildcardMatch` over the
+// working-tree side's wildcard matches. Off by default; persisted with the rest
+// of the selection so a reload keeps the choice. The host/runtime no-op it when
+// no validator is wired (e.g. the packaged build) and report that back via
+// `wildcardValidation.diagnostics`, which `renderValidationNote` surfaces.
+let validateWildcards = false;
+
 /** Short toolbar label for each replay mode. */
 const MODE_LABEL: Record<StudioReplayMode, string> = {
     "nfa-grammar": "Grammar",
@@ -182,6 +193,14 @@ modeButton.className = "tool-button";
 const modeButtonText = text("");
 modeButton.append(codicon("settings"), modeButtonText);
 modeButton.addEventListener("click", () => cycleMode());
+// A two-state toggle for opt-in wildcard validation (L4a). Built by hand like
+// `modeButton` so we keep a handle on its label and lit state.
+const validateButton = document.createElement("button");
+validateButton.type = "button";
+validateButton.className = "tool-button";
+const validateButtonText = text("");
+validateButton.append(codicon("verified"), validateButtonText);
+validateButton.addEventListener("click", () => toggleValidateWildcards());
 const runButton = toolButton("play", "Run", () => runReplay(), {
     primary: true,
     text: "Run",
@@ -199,7 +218,7 @@ actionBar.append(
     separator(),
     group(versionAButton.button, swapButton, versionBButton.button),
     separator(),
-    group(modeButton),
+    group(modeButton, validateButton),
     spacer(),
     runButton,
     connectionPill,
@@ -209,8 +228,9 @@ actionBar.append(
 // commits each side ran against) and the live status text.
 const subBar = el("div", "sub-bar");
 const provenanceEl = el("span", "provenance");
+const validationEl = el("span", "validation-note");
 const statusEl = el("span", "status");
-subBar.append(provenanceEl, statusEl);
+subBar.append(provenanceEl, validationEl, statusEl);
 
 // The scrolling content region: an inline error notification, the status filter,
 // first-run guidance, the results list, and the drill-in detail pane.
@@ -230,6 +250,7 @@ renderConnection("connecting");
 restoreSelection();
 renderVersionButtons();
 renderModeButton();
+renderValidateButton();
 
 // --- Messaging ------------------------------------------------------------
 window.addEventListener("message", (event: MessageEvent) => {
@@ -301,11 +322,18 @@ function runReplay(): void {
     }
     requestId += 1;
     latestRequestId = requestId;
-    persistState({ selectedAgent: agent, versionA, versionB, mode });
+    persistState({
+        selectedAgent: agent,
+        versionA,
+        versionB,
+        mode,
+        validateWildcards,
+    });
     setControlsEnabled(false);
     setStatus(`Replaying ${agent}…`);
     clearNotification();
     provenanceEl.textContent = "";
+    validationEl.textContent = "";
     filtersEl.textContent = "";
     filtersEl.hidden = true;
     emptyStateEl.hidden = true;
@@ -320,6 +348,7 @@ function runReplay(): void {
         versionA: versionA.spec,
         versionB: versionB.spec,
         mode,
+        validateWildcards,
     });
 }
 
@@ -330,6 +359,34 @@ function cycleMode(): void {
     persistState({ mode });
 }
 
+/** Toggle opt-in wildcard validation and persist the choice. */
+function toggleValidateWildcards(): void {
+    validateWildcards = !validateWildcards;
+    renderValidateButton();
+    persistState({ validateWildcards });
+}
+
+/** Paint the wildcard-validation toggle's label, tooltip, and lit state. */
+function renderValidateButton(): void {
+    validateButtonText.textContent = "Validate";
+    validateButton.classList.toggle("is-active", validateWildcards);
+    validateButton.setAttribute("aria-pressed", String(validateWildcards));
+    const tip = validateWildcards
+        ? "Wildcard validation ON: the working-tree side additionally runs the " +
+          "agent's real validateWildcardMatch over its wildcard matches (the " +
+          "dispatcher's post-match step), dropping a match the agent rejects. " +
+          "Only for allowlisted agents (timer, list, player); a no-op otherwise. " +
+          "Click to turn off."
+        : "Wildcard validation OFF: matches are taken from the grammar alone. " +
+          "Click to additionally run the agent's real validateWildcardMatch on " +
+          "wildcard matches (allowlisted agents only).";
+    validateButton.title = tip;
+    validateButton.setAttribute(
+        "aria-label",
+        `Wildcard validation ${validateWildcards ? "on" : "off"}. ${tip}`,
+    );
+}
+
 /** Paint the mode toggle's label and tooltip from the current `mode`. */
 function renderModeButton(): void {
     modeButtonText.textContent = MODE_LABEL[mode];
@@ -374,6 +431,8 @@ function renderResult(
     // second table instead of upgrading the snapshot in place.
     clearNotification();
     provenanceEl.textContent = "";
+    validationEl.textContent = "";
+    validationEl.classList.remove("is-degraded");
     filtersEl.textContent = "";
     filtersEl.hidden = true;
     emptyStateEl.hidden = true;
@@ -407,6 +466,10 @@ function renderResult(
         renderProvenance(provenance);
     }
 
+    // Surface the opt-in wildcard-validation outcome (L4a), including an honest
+    // "unavailable" state when the validator couldn't load (e.g. packaged build).
+    renderValidationNote(result.wildcardValidation);
+
     currentRows = toImpactRows(result.rows, result.methodA, result.methodB);
     currentTotal = result.summary.rowCount;
 
@@ -422,6 +485,85 @@ function renderResult(
     );
 }
 
+/** Paint the wildcard-validation outcome (L4a) into the sub-bar, or clear it.
+ *  Shown only when validation actually ran on a wildcard match; a clean pass is
+ *  neutral, a degraded/unavailable pass warns so the report stays honest. */
+function renderValidationNote(
+    info: StudioWildcardValidationInfo | undefined,
+): void {
+    validationEl.textContent = "";
+    validationEl.classList.remove("is-degraded");
+    if (!info || !info.applied) {
+        return;
+    }
+    const { icon, label, detail, degraded } = describeValidation(
+        info.diagnostics,
+    );
+    validationEl.classList.toggle("is-degraded", degraded);
+    validationEl.title = detail;
+    validationEl.append(codicon(icon), text(label));
+}
+
+/** Map the validation diagnostics to a compact label + hover detail. An empty
+ *  diagnostics list means the validator ran cleanly; otherwise we surface the
+ *  most significant fail-open reason so the run never overclaims fidelity. */
+function describeValidation(diagnostics: WildcardValidationDiagnostic[]): {
+    icon: string;
+    label: string;
+    detail: string;
+    degraded: boolean;
+} {
+    if (diagnostics.length === 0) {
+        return {
+            icon: "verified",
+            label: "wildcard-validated",
+            detail:
+                "The working-tree side ran the agent's real validateWildcardMatch " +
+                "over its wildcard matches; any match the agent rejected was dropped.",
+            degraded: false,
+        };
+    }
+    if (diagnostics.includes("load-failed")) {
+        return {
+            icon: "warning",
+            label: "validation unavailable",
+            detail:
+                "Wildcard validation was requested but the agent module couldn't be " +
+                "loaded (e.g. the packaged build ships no agent code), so matches " +
+                "fell back to the grammar alone.",
+            degraded: true,
+        };
+    }
+    if (diagnostics.includes("agent-not-in-allowlist")) {
+        return {
+            icon: "info",
+            label: "validation skipped",
+            detail:
+                "This agent isn't in the validation allowlist, so its validator " +
+                "wasn't run and matches came from the grammar alone.",
+            degraded: true,
+        };
+    }
+    if (diagnostics.includes("no-validator")) {
+        return {
+            icon: "info",
+            label: "no validator",
+            detail:
+                "This agent exposes no validateWildcardMatch, so validation made " +
+                "no change to the grammar matches.",
+            degraded: true,
+        };
+    }
+    return {
+        icon: "warning",
+        label: "validation degraded",
+        detail:
+            "The agent's validator threw while checking a wildcard match; the match " +
+            "was kept (fail-open) so the run stayed grammar-faithful.",
+        degraded: true,
+    };
+}
+
 /** Paint the status filter chips for the rows of the current result. */
 function renderFilters(): void {
     filtersEl.textContent = "";
@@ -763,6 +905,9 @@ function restoreSelection(): void {
     if (state?.mode) {
         mode = state.mode;
     }
+    if (state?.validateWildcards !== undefined) {
+        validateWildcards = state.validateWildcards === true;
+    }
     // The agent is authoritative from the host `init`; until it arrives, show a
     // neutral placeholder rather than a stale persisted value.
     renderAgentName();
@@ -818,6 +963,7 @@ function setControlsEnabled(enabled: boolean): void {
     versionAButton.button.disabled = !enabled;
     versionBButton.button.disabled = !enabled;
     modeButton.disabled = !enabled;
+    validateButton.disabled = !enabled;
 }
 
 function setStatus(text: string): void {
diff --git a/ts/packages/typeagent-studio/src/webviewKit/protocol.ts b/ts/packages/typeagent-studio/src/webviewKit/protocol.ts
index 493ed81ba7..6a5f2f7de6 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/protocol.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/protocol.ts
@@ -76,6 +76,12 @@ export type WebviewToHostMessage =
           versionB: VersionSpec;
           /** Which deterministic dispatch path to model. */
           mode: StudioReplayMode;
+          /**
+           * Opt-in: additionally run the agent's real `validateWildcardMatch`
+           * over the working-tree side's wildcard matches (replay fidelity rung
+           * L4a). Off unless the user lit the validation toggle.
+           */
+          validateWildcards: boolean;
       }
     /** Ask the host to open a native version QuickPick for one side. */
     | { type: "pickVersion"; side: ReplaySide };
@@ -115,6 +121,7 @@ export function parseWebviewMessage(
                 versionA?: unknown;
                 versionB?: unknown;
                 mode?: unknown;
+                validateWildcards?: unknown;
             };
             if (
                 typeof m.requestId === "number" &&
@@ -132,6 +139,8 @@ export function parseWebviewMessage(
                     versionA: coerceVersionSpec(m.versionA),
                     versionB: coerceVersionSpec(m.versionB),
                     mode: narrowMode(m.mode),
+                    // Opt-in and conservative: only an explicit `true` enables it.
+                    validateWildcards: m.validateWildcards === true,
                 };
             }
             return undefined;
diff --git a/ts/pnpm-lock.yaml b/ts/pnpm-lock.yaml
index dd1667979b..05fe69ef4b 100644
--- a/ts/pnpm-lock.yaml
+++ b/ts/pnpm-lock.yaml
@@ -155,7 +155,7 @@ importers:
         version: 8.18.1
       jest:
         specifier: ^29.7.0
-        version: 29.7.0(@types/node@25.9.3)(ts-node@10.9.2(@types/node@25.9.3)(typescript@5.4.5))
+        version: 29.7.0(@types/node@22.19.19)(ts-node@10.9.2(@types/node@22.19.19)(typescript@5.4.5))
       prettier:
         specifier: ^3.5.3
         version: 3.5.3
@@ -919,7 +919,7 @@ importers:
         version: 12.0.2(webpack@5.105.0)
       jest:
         specifier: ^29.7.0
-        version: 29.7.0(@types/node@22.19.19)(ts-node@10.9.2(@types/node@22.19.19)(typescript@5.4.5))
+        version: 29.7.0(@types/node@25.9.3)(ts-node@10.9.2(@types/node@25.9.3)(typescript@5.4.5))
       prettier:
         specifier: ^3.5.3
         version: 3.5.3
@@ -5632,6 +5632,9 @@ importers:
       debug:
         specifier: ^4.4.0
         version: 4.4.3(supports-color@8.1.1)
+      default-agent-provider:
+        specifier: workspace:*
+        version: link:../defaultAgentProvider
       ws:
         specifier: ^8.21.0
         version: 8.21.0

From 4ee1ebb20c32b6abc21d7fd51b8a3e27cceb041c Mon Sep 17 00:00:00 2001
From: Tal Zaccai <talzacc@microsoft.com>
Date: Thu, 25 Jun 2026 19:07:54 -0600
Subject: [PATCH 03/15] Drop player from L4a wildcard-validation allowlist

A live integration smoke against the real built agent modules (via
default-agent-provider) surfaced that player's validateWildcardMatch throws
during replay: player runs execMode "separate", so the loader returns an RPC
proxy whose child process reconstructs its own SessionContext where agentContext
is undefined (our agentContext:{} stub is never serialized across the wire).
player then throws "Cannot read properties of undefined (reading 'spotify')"
before reaching its no-client self-degrade guard. Even without the throw it can
only ever return true without a live Spotify client, so it adds no fidelity --
it would just fail open with an `errored` diagnostic and a misleading "degraded"
indicator.

timer and list, by contrast, ignore the context and validate correctly over RPC
(timer even produces real rejects), so the allowlist is now timer/list only.
Updated the stub-context rationale, the webview toggle tooltip, and the tests.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../src/replay/wildcardValidator.ts           | 36 ++++++++++++-------
 .../test/wildcardValidator.spec.ts            | 15 ++++----
 .../src/webviewKit/client/impactReport.ts     |  2 +-
 3 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/ts/packages/typeagent-core/src/replay/wildcardValidator.ts b/ts/packages/typeagent-core/src/replay/wildcardValidator.ts
index 9740b8f247..c1f839ffd2 100644
--- a/ts/packages/typeagent-core/src/replay/wildcardValidator.ts
+++ b/ts/packages/typeagent-core/src/replay/wildcardValidator.ts
@@ -95,13 +95,22 @@ export interface WildcardMatchValidator {
 }
 
 /**
- * Agents whose `validateWildcardMatch` is a pure/deterministic function of the
- * action (no network, no external process, no live session state):
- * - `timer` — `tryParseWhen` on the action params; context ignored.
+ * Agents whose `validateWildcardMatch` validates cleanly during replay — a
+ * deterministic function of the action that does NOT depend on live session
+ * state (verified end-to-end against the real built modules):
+ * - `timer` — `tryParseWhen` on the action params; context ignored. Rejects a
+ *   wildcard that isn't a real duration/timestamp.
  * - `list` — `simpleNoun` heuristic over the action; context ignored.
- * - `player` — reads `context.agentContext.spotify`, but returns `true` when it
- *   is absent (our stub context has none), faithfully self-degrading like a
- *   dispatcher session with no Spotify login.
+ *
+ * `player` is deliberately EXCLUDED even though its source returns `true` when
+ * `context.agentContext.spotify` is absent. It runs `execMode: "separate"`, so
+ * the loader returns an RPC proxy to a child process that reconstructs its OWN
+ * `SessionContext` — where `agentContext` is `undefined`, not our stub's `{}`.
+ * The child then throws `Cannot read properties of undefined (reading 'spotify')`
+ * before reaching its self-degrade guard. And even if it didn't throw, it can
+ * only ever return `true` without a live Spotify client, so it adds no fidelity.
+ * It would just fail-open with an `errored` diagnostic and a misleading
+ * "degraded" indicator — so we keep it grammar-only.
  *
  * `weather`/`markdown`/`photo`/`androidMobile`/`taskflow`/`powershell` are NOT
  * here: weather geocodes over the network, and the others are unverified.
@@ -109,7 +118,6 @@ export interface WildcardMatchValidator {
 export const DEFAULT_WILDCARD_VALIDATION_ALLOWLIST: readonly string[] = [
     "timer",
     "list",
-    "player",
 ];
 
 export interface CreateWildcardMatchValidatorOptions {
@@ -119,12 +127,14 @@ export interface CreateWildcardMatchValidatorOptions {
 }
 
 /**
- * A no-op `SessionContext`-shaped stub for replay validation. The few validators
- * we allowlist either ignore the context (timer/list) or read only
- * `agentContext` (player, which self-degrades when its client is absent). The
- * object exposes `agentContext: {}` — NOT `undefined`, which would make
- * `player`'s `context.agentContext.spotify` throw before its fallback — and the
- * interactive methods throw, since a validator that calls them is out of the
+ * A no-op `SessionContext`-shaped stub for replay validation. The validators we
+ * allowlist (timer/list) ignore the context entirely; the stub still exposes
+ * `agentContext: {}` — NOT `undefined` — so that any in-process validator that
+ * defensively reads `context.agentContext.<x>` degrades gracefully instead of
+ * throwing. (Out-of-process agents like `player` reconstruct their own context
+ * in the child and never see this stub, which is why `player` is excluded from
+ * the default allowlist — see {@link DEFAULT_WILDCARD_VALIDATION_ALLOWLIST}.)
+ * The interactive methods throw, since a validator that calls them is out of the
  * supported set and should fail-open via the wrapper's try/catch.
  */
 export function createReplaySessionContextStub(): Record<string, unknown> {
diff --git a/ts/packages/typeagent-core/test/wildcardValidator.spec.ts b/ts/packages/typeagent-core/test/wildcardValidator.spec.ts
index c3b9a2437d..3f5fcbbbd8 100644
--- a/ts/packages/typeagent-core/test/wildcardValidator.spec.ts
+++ b/ts/packages/typeagent-core/test/wildcardValidator.spec.ts
@@ -133,10 +133,13 @@ describe("createWildcardMatchValidator", () => {
         expect(outcome.diagnostic).toBe("no-validator");
     });
 
-    it("does not throw for a player-style validator reading agentContext (empty stub)", async () => {
-        // Mirrors the real player validator: reads context.agentContext.spotify
-        // and returns true when the client is absent. Our stub uses
-        // `agentContext: {}` (NOT undefined), so this must not throw.
+    it("does not throw for an in-process validator reading agentContext (empty stub)", async () => {
+        // A hypothetical in-process validator that reads context.agentContext
+        // and returns true when its client is absent. Our stub uses
+        // `agentContext: {}` (NOT undefined), so this must not throw. (The real
+        // player validator has this shape but runs out-of-process, so it's
+        // excluded from the default allowlist; here we override the allowlist to
+        // exercise the stub's defensive behaviour directly.)
         const playerLike: ReplayValidatableAgent = {
             async validateWildcardMatch(_a, context) {
                 const spotify = (
@@ -150,6 +153,7 @@ describe("createWildcardMatchValidator", () => {
         };
         const validator = createWildcardMatchValidator("player", {
             loader: fakeLoader({ player: playerLike }),
+            allowlist: ["player"],
         });
         const outcome = await validator.validateMatch([action("playTrack")]);
         expect(outcome.rejected).toBe(false);
@@ -187,10 +191,9 @@ describe("createWildcardMatchValidator", () => {
         expect(log.unloaded).toEqual([]);
     });
 
-    it("ships timer/list/player as the default allowlist", () => {
+    it("ships timer/list as the default allowlist", () => {
         expect([...DEFAULT_WILDCARD_VALIDATION_ALLOWLIST].sort()).toEqual([
             "list",
-            "player",
             "timer",
         ]);
     });
diff --git a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
index 226bc39b41..635054ff89 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
@@ -375,7 +375,7 @@ function renderValidateButton(): void {
         ? "Wildcard validation ON: the working-tree side additionally runs the " +
           "agent's real validateWildcardMatch over its wildcard matches (the " +
           "dispatcher's post-match step), dropping a match the agent rejects. " +
-          "Only for allowlisted agents (timer, list, player); a no-op otherwise. " +
+          "Only for allowlisted agents (timer, list); a no-op otherwise. " +
           "Click to turn off."
         : "Wildcard validation OFF: matches are taken from the grammar alone. " +
           "Click to additionally run the agent's real validateWildcardMatch on " +

From c1d893bb209e8b506b446861c182f7d67e79afe5 Mon Sep 17 00:00:00 2001
From: Tal Zaccai <talzacc@microsoft.com>
Date: Thu, 25 Jun 2026 20:16:24 -0600
Subject: [PATCH 04/15] Refresh Studio STATUS.md: replay at L1-L2 + two-mode +
 L4a

Update the capability matrix, long-pole narrative, and next-slice list to reflect shipped two-mode replay (grammar/cache) and L4a live wildcard validation, with L4b (build-from-ref) deferred to P-7 (post-Gate-C). The live priority is now player corpus capture -> Gate C measurement.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 ts/docs/plans/vscode-devx/STATUS.md | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/ts/docs/plans/vscode-devx/STATUS.md b/ts/docs/plans/vscode-devx/STATUS.md
index b6203e3fd4..3695087529 100644
--- a/ts/docs/plans/vscode-devx/STATUS.md
+++ b/ts/docs/plans/vscode-devx/STATUS.md
@@ -22,11 +22,11 @@ than an in-memory stand-in.
 | Agent health (status bar + findings)                           | 🟡 heuristic/filesystem checks                                              | ✅ status bar                                                      | ❌ no real schema parse / grammar compile                                  | ✅     |
 | Collisions (cross-schema grammar overlap)                      | ✅ real NFA scanner over compiled `.ag.json`                                | ✅ tree view + Skipped group + auto-scan (+ channel-backed source) | n/a (reads compiled grammars)                                              | ✅     |
 | Feedback (thumbs up/down → corpus)                             | ✅                                                                          | ✅ command                                                         | n/a                                                                        | ✅     |
-| Replay / compare engine                                        | ✅ static-grammar, schema-enriched (L1) & construction-cache (L2) resolvers | ✅ Impact Report webview (`ActionDelta[]`)                         | 🟡 grammar + live construction cache (L1–L2); no two-version build (L3–L4) | ✅     |
+| Replay / compare engine                                        | ✅ schema-enriched grammar (L1), construction-cache (L2), selectable two-mode (grammar/cache) + opt-in live wildcard validation (L4a) | ✅ Impact Report webview (`ActionDelta[]`)                         | 🟡 grammar + live construction cache + working-tree wildcard validation; no two-version build-from-ref (L4b, deferred to P-7) | ✅     |
 | Onboarding bridge (snapshot/restore, stale detection)          | ✅                                                                          | ✅ commands                                                        | ❌ in-memory bridge                                                        | ✅     |
 | Repo-root detection (find `packages/agents`)                   | ✅                                                                          | ✅ warn toast + status bar                                         | n/a                                                                        | ✅     |
 | Webview infrastructure (`webviewKit`)                          | ✅ CSP/nonce host + typed protocol                                          | ✅ singleton-panel host                                            | —                                                                          | ✅     |
-| Impact Report webview                                          | ✅ `replayCorpus` over channel                                              | ✅ context header, A/B controls, durable state                     | 🟡 grammar + construction-cache replay (L1–L2)                             | ✅     |
+| Impact Report webview                                          | ✅ `replayCorpus` over channel                                              | ✅ context header, A/B controls, Grammar/Cache + Validate toggles, durable state | 🟡 grammar + construction-cache + working-tree wildcard validation (L1–L2, L4a) | ✅     |
 | Player corpus capture                                          | ❌                                                                          | ❌                                                                 | ❌                                                                         | ❌     |
 | Schema Studio                                                  | ❌                                                                          | ❌                                                                 | ❌                                                                         | ❌     |
 | Live Trace                                                     | ❌                                                                          | ❌                                                                 | ❌                                                                         | ❌     |
@@ -41,7 +41,7 @@ Impact Report) is **progressing but not yet closed**:
   builder, singleton-panel host, typed host↔webview protocol, browser-neutral
   replay view model) and the Impact Report webview are built and tested — the
   foundation Schema Studio, Wizard, Trace, and Live Trace will reuse.
-- **Replay has climbed the fidelity ladder to L2.** Beyond the original identity
+- **Replay has climbed the fidelity ladder to L4a.** Beyond the original identity
   resolver, replay now runs **static-grammar** matching, **schema-enriched
   grammar** matching (L1: the agent's grammar is enriched with checked-variable
   metadata from its action schema and matched through the real `GrammarStore`),
@@ -52,8 +52,12 @@ Impact Report) is **progressing but not yet closed**:
   a phantom hit). The construction cache is consulted for the **working tree
   only** (caches are runtime artifacts, never committed / never read at a git
   ref) and degrades cleanly to L1 when no live cache is found or it has gone
-  stale. Results are still indicative for grammar-resolved rows — **L3–L4**
-  (deterministic dispatch, build-from-git-ref two-version sandboxes) remain, and
+  stale. Two replay **modes** are now selectable in the Impact Report (grammar-only
+  vs. construction-cache), and an opt-in **live wildcard validation** pass (L4a)
+  runs the agent's real `validateWildcardMatch` over working-tree wildcard matches
+  (`timer`/`list`; fail-open + diagnostics). Results are still indicative for
+  grammar-resolved rows — **L4b** (build-from-git-ref two-version sandboxes)
+  remains and is **deferred to P-7** (post-Gate-C, per the implementation plan);
   there is no git-worktree build of two versions yet.
 - **No capture-to-corpus path.** `vscode-shell` depends on core but doesn't use
   it; without capture the Impact Report would have no real labelled corpus.
@@ -171,7 +175,12 @@ Ready to start (smallest → larger):
    core corpus.
 4. **One real replay path** — one agent, one utterance, working tree vs. HEAD,
    real dispatch; validate the Impact Report `ActionDelta[]` contract (which the
-   agent's `ValidateChange` and the webview both consume).
+   agent's `ValidateChange` and the webview both consume). **Largely done:** the
+   grammar replay path (L1 schema-enriched, L2 construction-cache, two selectable
+   modes, L4a opt-in wildcard validation) is live and validated against the
+   contract. The remaining fidelity rung **L4b** (build-from-ref two-version
+   sandboxes) is **deferred to P-7** (post-Gate-C). **Live priority is now #3
+   (player corpus capture) → Gate C measurement** — the headline acceptance bar.
 5. **Active-sandbox selector + per-sandbox scoping** (plan phase **P-7a**;
    sequenced **after** the single-sandbox E2E closes) — collisions and corpora are
    intrinsically per-sandbox (a collision is a function of the co-loaded agent

From fb3dd1a08defee4617cab5681ca9bf7fbc8241f4 Mon Sep 17 00:00:00 2001
From: Tal Zaccai <talzacc@microsoft.com>
Date: Fri, 26 Jun 2026 20:34:26 -0600
Subject: [PATCH 05/15] studio: add replay fidelity transparency matrix and
 break build cycle

Surface a per-side fidelity matrix in the Impact Report so a replay is honest about which deterministic layers actually ran (grammar, schema enrichment, construction cache, wildcard validation) and what building from a git ref would add, instead of over-claiming fidelity.

Core: add FidelityLayer/SideFidelity types + sideFidelity on StudioReplayResult, populated by a pure, unit-tested deriveSideFidelity() in both the success and aborted paths. View model: pure toFidelityMatrix() with a build-from-ref preflight hint. Client+CSS: a collapsible fidelity panel with per-layer status icons and hover reasons.

Also break a build cycle introduced by L4a: studio-service no longer declares default-agent-provider (it is aggregated by studio-agent, which depends on studio-service). The optional, externally-bundled dynamic import now resolves from the bundling extension (typeagent-studio), which owns the dependency; the specifier is indirected so tsc does not statically resolve it.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 ts/packages/studio-service/package.json       |   1 -
 .../studio-service/src/wildcardValidation.ts  |  18 +-
 .../src/runtime/studioRuntimeCore.ts          | 258 +++++++++++++++++-
 .../typeagent-core/test/sideFidelity.spec.ts  | 170 ++++++++++++
 ts/packages/typeagent-studio/esbuild.mjs      |   4 +
 .../typeagent-studio/media/impactReport.css   |  94 +++++++
 ts/packages/typeagent-studio/package.json     |   1 +
 .../src/test/replayViewModel.spec.ts          |  84 ++++++
 .../src/webviewKit/client/impactReport.ts     | 103 ++++++-
 .../src/webviewKit/replayViewModel.ts         |  93 ++++++-
 ts/pnpm-lock.yaml                             |   6 +-
 11 files changed, 822 insertions(+), 10 deletions(-)
 create mode 100644 ts/packages/typeagent-core/test/sideFidelity.spec.ts

diff --git a/ts/packages/studio-service/package.json b/ts/packages/studio-service/package.json
index 78dc02309c..923470525e 100644
--- a/ts/packages/studio-service/package.json
+++ b/ts/packages/studio-service/package.json
@@ -38,7 +38,6 @@
     "@typeagent/core": "workspace:*",
     "@typeagent/websocket-utils": "workspace:*",
     "debug": "^4.4.0",
-    "default-agent-provider": "workspace:*",
     "ws": "^8.21.0"
   },
   "devDependencies": {
diff --git a/ts/packages/studio-service/src/wildcardValidation.ts b/ts/packages/studio-service/src/wildcardValidation.ts
index b8a6b924f4..341796a2f9 100644
--- a/ts/packages/studio-service/src/wildcardValidation.ts
+++ b/ts/packages/studio-service/src/wildcardValidation.ts
@@ -15,6 +15,14 @@
  * serve` dev path, and cleanly throws → no-ops in the packaged `.vsix` (which
  * ships without `node_modules`). Wildcard validation is opt-in and fail-open, so
  * when the provider is unavailable replay simply stays grammar-only.
+ *
+ * NOTE: `default-agent-provider` is intentionally NOT a package.json dependency
+ * of this service. `default-agent-provider` aggregates every agent including
+ * `studio-agent`, which depends on this service — declaring the dep here closes
+ * a build cycle (studio-service → default-agent-provider → studio-agent →
+ * studio-service). Because the import is external + dynamic, the dependency is
+ * owned by the bundling host (`typeagent-studio`, a leaf nothing depends on),
+ * whose `node_modules` the external import resolves against at runtime.
  */
 
 import {
@@ -50,8 +58,16 @@ async function loadProviders(): Promise<AppAgentProviderLike[] | undefined> {
     if (providersPromise === undefined) {
         providersPromise = (async () => {
             try {
+                // Indirect the specifier through a variable so TypeScript does
+                // not statically resolve the module: `default-agent-provider` is
+                // deliberately not a build-time dependency of this service (it
+                // would close a studio-service → default-agent-provider →
+                // studio-agent → studio-service build cycle). It is resolved at
+                // runtime from the bundling extension's `node_modules`, and the
+                // result is cast to the local structural type below.
+                const specifier = "default-agent-provider";
                 const mod = (await import(
-                    "default-agent-provider"
+                    specifier
                 )) as DefaultAgentProviderModule;
                 return mod.getDefaultAppAgentProviders(undefined);
             } catch {
diff --git a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
index 16f521f058..c84ed12f02 100644
--- a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
+++ b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
@@ -301,6 +301,45 @@ export type StudioReplayMethod =
     | "schema-grammar"
     | "construction-cache";
 
+/**
+ * The deterministic fidelity layers a replay side can exercise, mirroring the
+ * dispatcher's path: grammar match → schema enrichment → construction cache →
+ * wildcard validation → full dispatch. The Impact Report surfaces a per-side
+ * matrix of these so the run is honest about exactly what it ran (and what
+ * building from a ref would add) instead of over-claiming fidelity.
+ */
+export type FidelityLayer =
+    | "grammar"
+    | "schemaEnrichment"
+    | "constructionCache"
+    | "wildcardValidation"
+    | "dispatch";
+
+export type FidelityLayerStatus = "ran" | "skipped" | "unavailable";
+
+export interface FidelityLayerReport {
+    status: FidelityLayerStatus;
+    /** Short human reason (shown as hover detail). */
+    reason: string;
+}
+
+/**
+ * How fully a side's version was realized: `built-live` (the working tree — real
+ * compiled agent code) or `source` (only grammar/schema text materialized at a
+ * git ref via `git show`; no build, so live-only layers can't run).
+ */
+export type FidelityRealization = "built-live" | "source";
+
+export interface FidelityReport {
+    realization: FidelityRealization;
+    layers: Record<FidelityLayer, FidelityLayerReport>;
+}
+
+export interface SideFidelity {
+    A: FidelityReport;
+    B: FidelityReport;
+}
+
 export interface StudioReplayResult {
     runId: string;
     summary: ReplaySummary;
@@ -323,7 +362,13 @@ export interface StudioReplayResult {
      */
     wildcardValidation?: StudioWildcardValidationInfo;
     /**
-     * Set when a version failed to materialize or compile. The run is aborted
+     * Per-side fidelity descriptor: how each version was realized and which
+     * deterministic layers actually ran. Derived purely from signals the replay
+     * already computed (per-side method, mode, version kind, the L4a validation
+     * pass) so the Impact Report can show an honest "what ran" matrix.
+     */
+    sideFidelity: SideFidelity;
+    /**
      * with an empty summary rather than emitting fabricated regression rows.
      */
     error?: ReplayRunError;
@@ -341,6 +386,194 @@ export interface StudioWildcardValidationInfo {
     diagnostics: WildcardValidationDiagnostic[];
 }
 
+const FIDELITY_DISPATCH_REASON =
+    "Full dispatcher path (LLM translation + action handlers) is not run — replay is deterministic grammar/cache matching only.";
+
+export interface DeriveSideFidelityInput {
+    versionA: VersionSpec;
+    versionB: VersionSpec;
+    methodA: StudioReplayMethod;
+    methodB: StudioReplayMethod;
+    mode: StudioReplayMode;
+    validateWildcards: boolean;
+    wildcardValidation?: StudioWildcardValidationInfo;
+}
+
+/**
+ * Derive the per-side {@link SideFidelity} from signals the replay already
+ * produced. Pure and deterministic so it is unit-testable and reusable by any
+ * client. Wildcard validation only ever runs on the working-tree side, so the
+ * single `wildcardValidation` info (when present) is interpreted per side.
+ */
+export function deriveSideFidelity(
+    input: DeriveSideFidelityInput,
+): SideFidelity {
+    return {
+        A: deriveFidelityReport(
+            input.versionA,
+            input.methodA,
+            input.mode,
+            input.validateWildcards,
+            input.wildcardValidation,
+        ),
+        B: deriveFidelityReport(
+            input.versionB,
+            input.methodB,
+            input.mode,
+            input.validateWildcards,
+            input.wildcardValidation,
+        ),
+    };
+}
+
+function deriveFidelityReport(
+    version: VersionSpec,
+    method: StudioReplayMethod,
+    mode: StudioReplayMode,
+    validateWildcards: boolean,
+    wildcardValidation: StudioWildcardValidationInfo | undefined,
+): FidelityReport {
+    const isWorkingTree = version.kind === "workingTree";
+    return {
+        realization: isWorkingTree ? "built-live" : "source",
+        layers: {
+            grammar: fidelityGrammarLayer(method),
+            schemaEnrichment: fidelitySchemaEnrichmentLayer(method),
+            constructionCache: fidelityConstructionCacheLayer(
+                method,
+                isWorkingTree,
+                mode,
+            ),
+            wildcardValidation: fidelityWildcardValidationLayer(
+                isWorkingTree,
+                validateWildcards,
+                wildcardValidation,
+            ),
+            dispatch: {
+                status: "unavailable",
+                reason: FIDELITY_DISPATCH_REASON,
+            },
+        },
+    };
+}
+
+function fidelityGrammarLayer(method: StudioReplayMethod): FidelityLayerReport {
+    if (method === "identity") {
+        return {
+            status: "skipped",
+            reason: "Identity baseline — surfaces each entry's captured action without evaluating a grammar.",
+        };
+    }
+    return {
+        status: "ran",
+        reason: "Utterances matched through the real grammar store (NFA + sortMatches), the same engine the dispatcher uses.",
+    };
+}
+
+function fidelitySchemaEnrichmentLayer(
+    method: StudioReplayMethod,
+): FidelityLayerReport {
+    if (method === "schema-grammar" || method === "construction-cache") {
+        return {
+            status: "ran",
+            reason: "Grammar enriched with checked_wildcard metadata from the agent's action schema before compilation.",
+        };
+    }
+    if (method === "static-grammar") {
+        return {
+            status: "skipped",
+            reason: "Agent action-schema not discoverable at this version — matched the bare grammar (still indicative).",
+        };
+    }
+    return {
+        status: "skipped",
+        reason: "Identity baseline — no grammar evaluated.",
+    };
+}
+
+function fidelityConstructionCacheLayer(
+    method: StudioReplayMethod,
+    isWorkingTree: boolean,
+    mode: StudioReplayMode,
+): FidelityLayerReport {
+    if (method === "construction-cache") {
+        return {
+            status: "ran",
+            reason: "Consulted the agent's live construction cache (the dispatcher's first check), hash-gated to the current schema.",
+        };
+    }
+    if (!isWorkingTree) {
+        return {
+            status: "unavailable",
+            reason: "Construction caches are runtime artifacts, never committed — they cannot be read at a git ref.",
+        };
+    }
+    if (mode === "completionBased-cache") {
+        return {
+            status: "skipped",
+            reason: "No live cache was discovered, or it was stale versus the current schema → grammar fallback.",
+        };
+    }
+    return {
+        status: "skipped",
+        reason: "Grammar mode — the construction cache is intentionally not consulted (keeps matches A/B-symmetric).",
+    };
+}
+
+function fidelityWildcardValidationLayer(
+    isWorkingTree: boolean,
+    validateWildcards: boolean,
+    wildcardValidation: StudioWildcardValidationInfo | undefined,
+): FidelityLayerReport {
+    if (!isWorkingTree) {
+        return {
+            status: "unavailable",
+            reason: "The agent's validator code isn't built at a git ref — validation runs on the working-tree side only.",
+        };
+    }
+    if (!validateWildcards) {
+        return {
+            status: "skipped",
+            reason: "Validate toggle is off.",
+        };
+    }
+    if (wildcardValidation?.applied === true) {
+        const diagnostics = wildcardValidation.diagnostics;
+        if (diagnostics.includes("load-failed")) {
+            return {
+                status: "unavailable",
+                reason: "Agent module couldn't be loaded (e.g. a packaged build ships no agent code) — matches fell back to the grammar.",
+            };
+        }
+        if (diagnostics.includes("agent-not-in-allowlist")) {
+            return {
+                status: "skipped",
+                reason: "Agent isn't in the validation allowlist — its validator wasn't run.",
+            };
+        }
+        if (diagnostics.includes("no-validator")) {
+            return {
+                status: "skipped",
+                reason: "Agent exposes no validateWildcardMatch — validation made no change.",
+            };
+        }
+        if (diagnostics.includes("errored")) {
+            return {
+                status: "ran",
+                reason: "Validator threw on a match; the match was kept fail-open so the run stayed grammar-faithful.",
+            };
+        }
+        return {
+            status: "ran",
+            reason: "Ran the agent's real validateWildcardMatch over its wildcard matches; rejected matches were dropped.",
+        };
+    }
+    return {
+        status: "skipped",
+        reason: "Validation enabled but did not run — no wildcard match occurred, or the agent isn't validatable.",
+    };
+}
+
 export interface StudioCollisionScanRequest {
     /**
      * Agent package names to scan. Defaults to every agent currently loaded
@@ -656,6 +889,8 @@ function abortedReplayResult(
     options: Parameters<typeof replayCorpus>[0],
     error: ReplayRunError,
     method: StudioReplayMethod = "static-grammar",
+    mode: StudioReplayMode = "nfa-grammar",
+    validateWildcards = false,
 ): StudioReplayResult {
     const runId = `replay-error-${Date.now().toString(36)}`;
     return {
@@ -667,6 +902,14 @@ function abortedReplayResult(
         method,
         methodA: method,
         methodB: method,
+        sideFidelity: deriveSideFidelity({
+            versionA: options.versionA,
+            versionB: options.versionB,
+            methodA: method,
+            methodB: method,
+            mode,
+            validateWildcards,
+        }),
         summary: {
             runId,
             agent: options.agent,
@@ -1194,6 +1437,8 @@ export function createStudioRuntimeCore(
                                     message: err.message,
                                 },
                                 labelFor(),
+                                mode,
+                                request.validateWildcards === true,
                             );
                         }
                         throw err;
@@ -1235,6 +1480,17 @@ export function createStudioRuntimeCore(
                 method,
                 methodA,
                 methodB,
+                sideFidelity: deriveSideFidelity({
+                    versionA: replayOptions.versionA,
+                    versionB: replayOptions.versionB,
+                    methodA,
+                    methodB,
+                    mode,
+                    validateWildcards: request.validateWildcards === true,
+                    ...(wildcardValidation !== undefined
+                        ? { wildcardValidation }
+                        : {}),
+                }),
                 ...(wildcardValidation !== undefined
                     ? { wildcardValidation }
                     : {}),
diff --git a/ts/packages/typeagent-core/test/sideFidelity.spec.ts b/ts/packages/typeagent-core/test/sideFidelity.spec.ts
new file mode 100644
index 0000000000..3288539ee7
--- /dev/null
+++ b/ts/packages/typeagent-core/test/sideFidelity.spec.ts
@@ -0,0 +1,170 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import {
+    deriveSideFidelity,
+    type DeriveSideFidelityInput,
+    type FidelityLayer,
+    type FidelityLayerStatus,
+} from "../src/runtime/studioRuntimeCore.js";
+
+const workingTree = { kind: "workingTree" } as const;
+const gitRef = { kind: "git", ref: "HEAD" } as const;
+
+function baseInput(
+    overrides: Partial<DeriveSideFidelityInput> = {},
+): DeriveSideFidelityInput {
+    return {
+        versionA: workingTree,
+        versionB: gitRef,
+        methodA: "schema-grammar",
+        methodB: "schema-grammar",
+        mode: "nfa-grammar",
+        validateWildcards: false,
+        ...overrides,
+    };
+}
+
+function statusOf(
+    input: DeriveSideFidelityInput,
+    side: "A" | "B",
+    layer: FidelityLayer,
+): FidelityLayerStatus {
+    return deriveSideFidelity(input)[side].layers[layer].status;
+}
+
+describe("deriveSideFidelity", () => {
+    it("marks realization by version kind", () => {
+        const fidelity = deriveSideFidelity(baseInput());
+        expect(fidelity.A.realization).toBe("built-live");
+        expect(fidelity.B.realization).toBe("source");
+    });
+
+    it("every layer carries a non-empty reason", () => {
+        const fidelity = deriveSideFidelity(baseInput());
+        for (const side of [fidelity.A, fidelity.B]) {
+            for (const report of Object.values(side.layers)) {
+                expect(report.reason.length).toBeGreaterThan(0);
+            }
+        }
+    });
+
+    it("dispatch is always unavailable", () => {
+        expect(statusOf(baseInput(), "A", "dispatch")).toBe("unavailable");
+        expect(statusOf(baseInput(), "B", "dispatch")).toBe("unavailable");
+    });
+
+    it("grammar runs except for identity", () => {
+        expect(statusOf(baseInput(), "A", "grammar")).toBe("ran");
+        const identity = baseInput({ methodA: "identity" });
+        expect(statusOf(identity, "A", "grammar")).toBe("skipped");
+    });
+
+    it("schema enrichment runs only for schema/cache methods", () => {
+        expect(statusOf(baseInput(), "A", "schemaEnrichment")).toBe("ran");
+        const cache = baseInput({ methodA: "construction-cache" });
+        expect(statusOf(cache, "A", "schemaEnrichment")).toBe("ran");
+        const bare = baseInput({ methodA: "static-grammar" });
+        expect(statusOf(bare, "A", "schemaEnrichment")).toBe("skipped");
+    });
+
+    it("construction cache is unavailable at a git ref", () => {
+        expect(statusOf(baseInput(), "B", "constructionCache")).toBe(
+            "unavailable",
+        );
+    });
+
+    it("construction cache runs only when the method used it", () => {
+        const cache = baseInput({
+            versionA: workingTree,
+            methodA: "construction-cache",
+        });
+        expect(statusOf(cache, "A", "constructionCache")).toBe("ran");
+    });
+
+    it("construction cache is skipped in grammar mode on the working tree", () => {
+        const grammar = baseInput({
+            versionA: workingTree,
+            methodA: "schema-grammar",
+            mode: "nfa-grammar",
+        });
+        expect(statusOf(grammar, "A", "constructionCache")).toBe("skipped");
+    });
+
+    it("construction cache is skipped on a stale cache-mode working tree", () => {
+        const stale = baseInput({
+            versionA: workingTree,
+            methodA: "schema-grammar",
+            mode: "completionBased-cache",
+        });
+        expect(statusOf(stale, "A", "constructionCache")).toBe("skipped");
+    });
+
+    it("wildcard validation is unavailable at a git ref", () => {
+        expect(statusOf(baseInput(), "B", "wildcardValidation")).toBe(
+            "unavailable",
+        );
+    });
+
+    it("wildcard validation is skipped when the toggle is off", () => {
+        expect(
+            statusOf(
+                baseInput({ validateWildcards: false }),
+                "A",
+                "wildcardValidation",
+            ),
+        ).toBe("skipped");
+    });
+
+    it("wildcard validation runs when applied with no diagnostics", () => {
+        const applied = baseInput({
+            validateWildcards: true,
+            wildcardValidation: { applied: true, diagnostics: [] },
+        });
+        expect(statusOf(applied, "A", "wildcardValidation")).toBe("ran");
+    });
+
+    it("wildcard validation maps load-failed to unavailable", () => {
+        const degraded = baseInput({
+            validateWildcards: true,
+            wildcardValidation: {
+                applied: true,
+                diagnostics: ["load-failed"],
+            },
+        });
+        expect(statusOf(degraded, "A", "wildcardValidation")).toBe(
+            "unavailable",
+        );
+    });
+
+    it("wildcard validation maps allowlist/no-validator to skipped", () => {
+        for (const diagnostic of [
+            "agent-not-in-allowlist",
+            "no-validator",
+        ] as const) {
+            const skipped = baseInput({
+                validateWildcards: true,
+                wildcardValidation: {
+                    applied: true,
+                    diagnostics: [diagnostic],
+                },
+            });
+            expect(statusOf(skipped, "A", "wildcardValidation")).toBe(
+                "skipped",
+            );
+        }
+    });
+
+    it("wildcard validation maps errored to ran (fail-open)", () => {
+        const errored = baseInput({
+            validateWildcards: true,
+            wildcardValidation: { applied: true, diagnostics: ["errored"] },
+        });
+        expect(statusOf(errored, "A", "wildcardValidation")).toBe("ran");
+    });
+
+    it("wildcard validation is skipped when enabled but never applied", () => {
+        const enabled = baseInput({ validateWildcards: true });
+        expect(statusOf(enabled, "A", "wildcardValidation")).toBe("skipped");
+    });
+});
diff --git a/ts/packages/typeagent-studio/esbuild.mjs b/ts/packages/typeagent-studio/esbuild.mjs
index 599c836d0a..a2d4b05074 100644
--- a/ts/packages/typeagent-studio/esbuild.mjs
+++ b/ts/packages/typeagent-studio/esbuild.mjs
@@ -105,6 +105,10 @@ const serviceConfig = {
     // agent, so it must NOT be inlined here. Marked external, it resolves from
     // `node_modules` on the in-repo `typeagent-studio serve` dev path and simply
     // fails the dynamic import (→ grammar-only replay) in the packaged `.vsix`.
+    // The dependency is declared on THIS extension (not studio-service) so it
+    // resolves against the extension's `node_modules` here, and to avoid a build
+    // cycle: studio-service is aggregated by studio-agent, which default-agent
+    // -provider in turn depends on. See studio-service/src/wildcardValidation.ts.
     external: ["vscode", "default-agent-provider"],
     alias: nodeBundleAlias,
     format: "cjs",
diff --git a/ts/packages/typeagent-studio/media/impactReport.css b/ts/packages/typeagent-studio/media/impactReport.css
index 74bf893c45..8569c3367e 100644
--- a/ts/packages/typeagent-studio/media/impactReport.css
+++ b/ts/packages/typeagent-studio/media/impactReport.css
@@ -400,6 +400,100 @@ body {
   flex: 0 0 auto;
 }
 
+/* --- Per-side fidelity matrix -------------------------------------------- */
+.fidelity-panel {
+  margin: 8px 12px 0;
+}
+
+.fidelity-panel[hidden] {
+  display: none;
+}
+
+.fidelity-details {
+  border: 1px solid var(--vscode-panel-border);
+  border-radius: var(--ta-radius);
+  background: var(--vscode-sideBar-background, var(--vscode-editor-background));
+  padding: 0 10px;
+}
+
+.fidelity-summary {
+  display: inline-flex;
+  align-items: center;
+  gap: 6px;
+  padding: 7px 0;
+  cursor: pointer;
+  color: var(--vscode-descriptionForeground);
+  font-size: 12px;
+  user-select: none;
+}
+
+.fidelity-summary::-webkit-details-marker {
+  display: none;
+}
+
+.fidelity-grid {
+  display: grid;
+  grid-template-columns: 1fr auto auto;
+  gap: 4px 14px;
+  padding: 4px 0 10px;
+  font-size: 12px;
+}
+
+.fidelity-head {
+  color: var(--vscode-descriptionForeground);
+  font-weight: 600;
+  text-transform: uppercase;
+  font-size: 10px;
+  letter-spacing: 0.04em;
+}
+
+.fidelity-grid .fidelity-head:not(:first-child),
+.fidelity-grid .fidelity-cell {
+  justify-self: center;
+  text-align: center;
+}
+
+.fidelity-layer {
+  color: var(--vscode-foreground);
+}
+
+.fidelity-cell {
+  display: inline-flex;
+  align-items: center;
+  gap: 4px;
+  white-space: nowrap;
+}
+
+.fidelity-cell.is-ran {
+  color: var(--vscode-testing-iconPassed, var(--vscode-charts-green, #3fb950));
+}
+
+.fidelity-cell.is-skipped {
+  color: var(--vscode-descriptionForeground);
+}
+
+.fidelity-cell.is-unavailable {
+  color: var(
+    --vscode-editorWarning-foreground,
+    var(--vscode-descriptionForeground)
+  );
+}
+
+.fidelity-preflight {
+  display: flex;
+  align-items: flex-start;
+  gap: 6px;
+  padding: 0 0 10px;
+  color: var(--vscode-descriptionForeground);
+  font-size: 11px;
+  line-height: 1.4;
+}
+
+.fidelity-preflight .codicon {
+  flex: 0 0 auto;
+  color: var(--vscode-charts-yellow, var(--vscode-descriptionForeground));
+}
+
 /* --- Scrolling content area ---------------------------------------------- */
 .content {
   flex: 1 1 auto;
diff --git a/ts/packages/typeagent-studio/package.json b/ts/packages/typeagent-studio/package.json
index 688930af77..cba1c7600d 100644
--- a/ts/packages/typeagent-studio/package.json
+++ b/ts/packages/typeagent-studio/package.json
@@ -503,6 +503,7 @@
     "@typeagent/core": "workspace:*",
     "@typeagent/websocket-utils": "workspace:*",
     "debug": "^4.4.0",
+    "default-agent-provider": "workspace:*",
     "ws": "^8.21.0"
   },
   "devDependencies": {
diff --git a/ts/packages/typeagent-studio/src/test/replayViewModel.spec.ts b/ts/packages/typeagent-studio/src/test/replayViewModel.spec.ts
index c2d7024baf..0a545db7aa 100644
--- a/ts/packages/typeagent-studio/src/test/replayViewModel.spec.ts
+++ b/ts/packages/typeagent-studio/src/test/replayViewModel.spec.ts
@@ -24,8 +24,10 @@ import {
     impactEmptyState,
     allRowsEqual,
     IMPACT_FILTER_ORDER,
+    toFidelityMatrix,
     type ReplayRowStatus,
 } from "../webviewKit/replayViewModel.js";
+import type { SideFidelity } from "@typeagent/core/runtime";
 
 function row(overrides: Partial<ActionDelta>): ActionDelta {
     return {
@@ -422,3 +424,85 @@ test("impactEmptyState gives first-run guidance", () => {
     assert.ok(/base/i.test(state.hint));
     assert.ok(/compare/i.test(state.hint));
 });
+
+function fidelityReport(
+    realization: SideFidelity["A"]["realization"],
+    overrides: Partial<SideFidelity["A"]["layers"]> = {},
+): SideFidelity["A"] {
+    const ran = { status: "ran", reason: "ran reason" } as const;
+    return {
+        realization,
+        layers: {
+            grammar: { ...ran },
+            schemaEnrichment: { ...ran },
+            constructionCache: { ...ran },
+            wildcardValidation: { ...ran },
+            dispatch: { status: "unavailable", reason: "no dispatch" },
+            ...overrides,
+        },
+    };
+}
+
+test("toFidelityMatrix returns undefined when no descriptor", () => {
+    assert.equal(toFidelityMatrix(undefined), undefined);
+});
+
+test("toFidelityMatrix produces one row per layer in order", () => {
+    const view = toFidelityMatrix({
+        A: fidelityReport("built-live"),
+        B: fidelityReport("built-live"),
+    });
+    assert.ok(view);
+    assert.deepEqual(
+        view!.rows.map((r) => r.layer),
+        [
+            "Grammar match",
+            "Schema enrichment",
+            "Construction cache",
+            "Wildcard validation",
+            "Full dispatch",
+        ],
+    );
+    assert.equal(view!.realizationA, "built (live)");
+    assert.equal(view!.realizationB, "built (live)");
+    // No source side → no preflight hint.
+    assert.equal(view!.preflight, undefined);
+});
+
+test("toFidelityMatrix carries per-side status and reason", () => {
+    const view = toFidelityMatrix({
+        A: fidelityReport("built-live"),
+        B: fidelityReport("source", {
+            constructionCache: {
+                status: "unavailable",
+                reason: "no cache at a ref",
+            },
+        }),
+    });
+    assert.ok(view);
+    const cacheRow = view!.rows.find((r) => r.layer === "Construction cache");
+    assert.ok(cacheRow);
+    assert.equal(cacheRow!.a.status, "ran");
+    assert.equal(cacheRow!.b.status, "unavailable");
+    assert.equal(cacheRow!.b.reason, "no cache at a ref");
+});
+
+test("toFidelityMatrix adds a preflight hint for a single source side", () => {
+    const view = toFidelityMatrix({
+        A: fidelityReport("built-live"),
+        B: fidelityReport("source"),
+    });
+    assert.ok(view);
+    assert.ok(view!.preflight);
+    assert.ok(/Side B is /.test(view!.preflight!));
+});
+
+test("toFidelityMatrix pluralizes the preflight hint for two source sides", () => {
+    const view = toFidelityMatrix({
+        A: fidelityReport("source"),
+        B: fidelityReport("source"),
+    });
+    assert.ok(view);
+    assert.ok(view!.preflight);
+    assert.ok(/Side A & B are /.test(view!.preflight!));
+});
diff --git a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
index 635054ff89..8964da1ade 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
@@ -38,10 +38,12 @@ import {
     formatProvenanceLine,
     formatVersionProvenance,
     toActionDiff,
+    toFidelityMatrix,
     type ImpactRow,
     type ReplayRowStatus,
     type ResolvedVersion,
     type RunProvenance,
+    type FidelityCell,
 } from "../replayViewModel.js";
 
 /** Default base (A): the last commit — the baseline of the regression journey. */
@@ -232,16 +234,26 @@ const validationEl = el("span", "validation-note");
 const statusEl = el("span", "status");
 subBar.append(provenanceEl, validationEl, statusEl);
 
-// The scrolling content region: an inline error notification, the status filter,
-// first-run guidance, the results list, and the drill-in detail pane.
+// The scrolling content region: an inline error notification, the per-side
+// fidelity matrix, the status filter, first-run guidance, the results list, and
+// the drill-in detail pane.
 const contentEl = el("div", "content");
 const notificationEl = el("div", "notification");
+const fidelityEl = el("div", "fidelity-panel");
+fidelityEl.hidden = true;
 const filtersEl = el("div", "filters");
 const emptyStateEl = el("div", "empty-state");
 const tableWrap = el("div", "table-wrap");
 const detailEl = el("div", "detail-pane");
 detailEl.hidden = true;
-contentEl.append(notificationEl, filtersEl, emptyStateEl, tableWrap, detailEl);
+contentEl.append(
+    notificationEl,
+    fidelityEl,
+    filtersEl,
+    emptyStateEl,
+    tableWrap,
+    detailEl,
+);
 
 root.append(actionBar, subBar, contentEl);
 
@@ -433,6 +445,8 @@ function renderResult(
     provenanceEl.textContent = "";
     validationEl.textContent = "";
     validationEl.classList.remove("is-degraded");
+    fidelityEl.textContent = "";
+    fidelityEl.hidden = true;
     filtersEl.textContent = "";
     filtersEl.hidden = true;
     emptyStateEl.hidden = true;
@@ -470,6 +484,10 @@ function renderResult(
     // "unavailable" state when the validator couldn't load (e.g. packaged build).
     renderValidationNote(result.wildcardValidation);
 
+    // The per-side fidelity matrix: which deterministic layers actually ran on
+    // each version, plus a build-from-ref preflight when a side is source-only.
+    renderFidelity(result.sideFidelity);
+
     currentRows = toImpactRows(result.rows, result.methodA, result.methodB);
     currentTotal = result.summary.rowCount;
 
@@ -564,6 +582,85 @@ function describeValidation(diagnostics: WildcardValidationDiagnostic[]): {
     };
 }
 
+const FIDELITY_STATUS_ICON: Record<FidelityCell["status"], string> = {
+    ran: "pass-filled",
+    skipped: "circle-slash",
+    unavailable: "circle-large-outline",
+};
+
+/** Render the per-side fidelity matrix into a collapsible panel, or hide it.
+ *  Surfaces which deterministic layers ran on each version (A/B) with a status
+ *  icon + hover reason, plus a build-from-ref preflight when a side is source
+ *  -only — so the report is honest about exactly what it exercised. */
+function renderFidelity(
+    sideFidelity: StudioReplayResult["sideFidelity"] | undefined,
+): void {
+    fidelityEl.textContent = "";
+    fidelityEl.hidden = true;
+    const view = toFidelityMatrix(sideFidelity);
+    if (!view) {
+        return;
+    }
+
+    const details = document.createElement("details");
+    details.className = "fidelity-details";
+    const summary = document.createElement("summary");
+    summary.className = "fidelity-summary";
+    summary.append(
+        codicon("layers"),
+        text(
+            `Fidelity \u00b7 A: ${view.realizationA} \u00b7 B: ${view.realizationB}`,
+        ),
+    );
+    details.append(summary);
+
+    const grid = el("div", "fidelity-grid");
+    grid.append(
+        fidelityHeadCell("Layer"),
+        fidelityHeadCell("A"),
+        fidelityHeadCell("B"),
+    );
+    for (const row of view.rows) {
+        grid.append(
+            fidelityLayerCell(row.layer),
+            fidelityStatusCell(row.a),
+            fidelityStatusCell(row.b),
+        );
+    }
+    details.append(grid);
+
+    if (view.preflight) {
+        const preflight = el("div", "fidelity-preflight");
+        preflight.append(codicon("lightbulb"), text(view.preflight));
+        details.append(preflight);
+    }
+
+    fidelityEl.append(details);
+    fidelityEl.hidden = false;
+}
+
+function fidelityHeadCell(label: string): HTMLElement {
+    const cell = el("span", "fidelity-head");
+    cell.textContent = label;
+    return cell;
+}
+
+function fidelityLayerCell(label: string): HTMLElement {
+    const cell = el("span", "fidelity-layer");
+    cell.textContent = label;
+    return cell;
+}
+
+function fidelityStatusCell(cellInfo: FidelityCell): HTMLElement {
+    const cell = el("span", `fidelity-cell is-${cellInfo.status}`);
+    cell.title = cellInfo.reason;
+    cell.append(
+        codicon(FIDELITY_STATUS_ICON[cellInfo.status]),
+        text(cellInfo.status),
+    );
+    return cell;
+}
+
 /** Paint the status filter chips for the rows of the current result. */
 function renderFilters(): void {
     filtersEl.textContent = "";
diff --git a/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts b/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts
index eef46be0ac..6de9a8e512 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts
@@ -15,7 +15,12 @@ import type {
     ReplayCacheState,
     VersionSpec,
 } from "@typeagent/core/replay";
-import type { StudioReplayResult } from "@typeagent/core/runtime";
+import type {
+    SideFidelity,
+    FidelityLayer,
+    FidelityLayerStatus,
+    StudioReplayResult,
+} from "@typeagent/core/runtime";
 import {
     classifyReplayRow,
     type ReplayRowStatus,
@@ -296,6 +301,92 @@ export function toSideMethodLabel(method: StudioReplayMethod): string {
     return METHOD_LABEL[method];
 }
 
+/** One cell of the fidelity matrix: a layer's status on one side plus the
+ *  reason to show on hover. */
+export interface FidelityCell {
+    status: FidelityLayerStatus;
+    reason: string;
+}
+
+/** One row of the fidelity matrix: a deterministic layer and its A/B status. */
+export interface FidelityMatrixRow {
+    /** Display label for the layer, e.g. "Construction cache". */
+    layer: string;
+    a: FidelityCell;
+    b: FidelityCell;
+}
+
+/** Render-ready per-side fidelity matrix for the Impact Report. */
+export interface FidelityMatrixView {
+    /** How side A was realized, e.g. "built (live)". */
+    realizationA: string;
+    /** How side B was realized, e.g. "source (git ref)". */
+    realizationB: string;
+    rows: FidelityMatrixRow[];
+    /** A "build-from-ref would add X" hint shown when a side is source-only. */
+    preflight?: string;
+}
+
+const FIDELITY_LAYER_ORDER: { key: FidelityLayer; label: string }[] = [
+    { key: "grammar", label: "Grammar match" },
+    { key: "schemaEnrichment", label: "Schema enrichment" },
+    { key: "constructionCache", label: "Construction cache" },
+    { key: "wildcardValidation", label: "Wildcard validation" },
+    { key: "dispatch", label: "Full dispatch" },
+];
+
+const FIDELITY_REALIZATION_LABEL: Record<
+    SideFidelity["A"]["realization"],
+    string
+> = {
+    "built-live": "built (live)",
+    source: "source (git ref)",
+};
+
+/**
+ * Map the core {@link SideFidelity} descriptor into a render-ready matrix:
+ * one row per deterministic layer with each side's status + reason, plus a
+ * preflight hint when a side only materialized source at a git ref (so a build
+ * -from-ref pass would unlock the live-only layers). Pure + browser-neutral.
+ */
+export function toFidelityMatrix(
+    sideFidelity: SideFidelity | undefined,
+): FidelityMatrixView | undefined {
+    if (!sideFidelity) {
+        return undefined;
+    }
+    const rows: FidelityMatrixRow[] = FIDELITY_LAYER_ORDER.map(
+        ({ key, label }) => ({
+            layer: label,
+            a: toFidelityCell(sideFidelity.A.layers[key]),
+            b: toFidelityCell(sideFidelity.B.layers[key]),
+        }),
+    );
+    const sources: string[] = [];
+    if (sideFidelity.A.realization === "source") {
+        sources.push("A");
+    }
+    if (sideFidelity.B.realization === "source") {
+        sources.push("B");
+    }
+    const view: FidelityMatrixView = {
+        realizationA: FIDELITY_REALIZATION_LABEL[sideFidelity.A.realization],
+        realizationB: FIDELITY_REALIZATION_LABEL[sideFidelity.B.realization],
+        rows,
+    };
+    if (sources.length > 0) {
+        const plural = sources.length > 1;
+        view.preflight =
+            `Side ${sources.join(" & ")} ${plural ? "are" : "is"} materialized source read at a git ref. ` +
+            `Building from the ref would let ${plural ? "them" : "it"} run the construction cache and the agent's real wildcard validation — not yet available.`;
+    }
+    return view;
+}
+
+function toFidelityCell(report: FidelityCell): FidelityCell {
+    return { status: report.status, reason: report.reason };
+}
+
 /**
  * A version the user selected through a picker: the typed spec the host will run
  * plus the resolved, human-readable label/tooltip to show in the launch control.
diff --git a/ts/pnpm-lock.yaml b/ts/pnpm-lock.yaml
index 05fe69ef4b..88f1b1c561 100644
--- a/ts/pnpm-lock.yaml
+++ b/ts/pnpm-lock.yaml
@@ -5632,9 +5632,6 @@ importers:
       debug:
         specifier: ^4.4.0
         version: 4.4.3(supports-color@8.1.1)
-      default-agent-provider:
-        specifier: workspace:*
-        version: link:../defaultAgentProvider
       ws:
         specifier: ^8.21.0
         version: 8.21.0
@@ -5860,6 +5857,9 @@ importers:
       debug:
         specifier: ^4.4.0
         version: 4.4.3(supports-color@8.1.1)
+      default-agent-provider:
+        specifier: workspace:*
+        version: link:../defaultAgentProvider
       ws:
         specifier: ^8.21.0
         version: 8.21.0

From 38f5f068089e3eea094bc6d1b226a3768644ecb8 Mon Sep 17 00:00:00 2001
From: Tal Zaccai <talzacc@microsoft.com>
Date: Fri, 26 Jun 2026 21:59:21 -0600
Subject: [PATCH 06/15] Studio: replay options popover, corpus auto-refresh,
 fidelity matrix cleanup

Impact Report:

- Move the Cache and Wildcard validation toggles behind a gear 'Replay options' popover with VS Code-themed on/off pill switches (closes on outside-click/Escape, disabled mid-run). Add the settings-gear codicon glyph to the curated set so the icon-only button renders.

- Relabel the old Grammar/Cache mode toggle to a Cache on/off switch and Validate to Wildcard validation, with concise one-line tooltips.

- Remove the verbose source-side preflight hint from the fidelity matrix (and its CSS + tests) per UI 'avoid heavy text' guidance.

Corpora view:

- Add a FileSystemWatcher on **/*.utterances.jsonl so the tree refreshes on corpus create/change/delete instead of only on manual refresh (fixes the seed-then-save stale tree).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../typeagent-studio/media/impactReport.css   | 138 ++++++++++--
 ts/packages/typeagent-studio/src/extension.ts |  12 ++
 .../src/test/replayViewModel.spec.ts          |  22 --
 .../src/webviewKit/client/impactReport.ts     | 198 ++++++++++--------
 .../src/webviewKit/replayViewModel.ts         |  23 +-
 5 files changed, 253 insertions(+), 140 deletions(-)

diff --git a/ts/packages/typeagent-studio/media/impactReport.css b/ts/packages/typeagent-studio/media/impactReport.css
index 8569c3367e..1cc1c91056 100644
--- a/ts/packages/typeagent-studio/media/impactReport.css
+++ b/ts/packages/typeagent-studio/media/impactReport.css
@@ -107,6 +107,9 @@ body {
 .codicon-sync::before {
   content: "\ea77";
 }
+.codicon-settings-gear::before {
+  content: "\eb51";
+}
 
 /* --- Action bar (the native editor toolbar) ------------------------------ */
 .action-bar {
@@ -314,6 +317,126 @@ body {
   border-color: var(--vscode-inputOption-activeBorder, transparent);
 }
 
+/* --- Replay-options popover + switches ----------------------------------- */
+.settings-group {
+  position: relative;
+}
+
+.settings-popover {
+  position: absolute;
+  top: calc(100% + 4px);
+  right: 0;
+  z-index: 30;
+  display: flex;
+  flex-direction: column;
+  gap: 12px;
+  min-width: 220px;
+  padding: 12px 14px;
+  background: var(
+    --vscode-editorWidget-background,
+    var(--vscode-menu-background)
+  );
+  color: var(--vscode-editorWidget-foreground, var(--vscode-menu-foreground));
+  border: 1px solid
+    var(
+      --vscode-editorWidget-border,
+      var(--vscode-menu-border, var(--vscode-widget-border))
+    );
+  border-radius: 4px;
+  box-shadow: 0 2px 8px var(--vscode-widget-shadow, rgba(0, 0, 0, 0.36));
+}
+
+.switch-row {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 18px;
+}
+
+.switch-label {
+  font-size: 12px;
+  white-space: nowrap;
+}
+
+.switch {
+  position: relative;
+  flex: 0 0 auto;
+  width: 52px;
+  height: 22px;
+  padding: 0;
+  border: none;
+  border-radius: 11px;
+  cursor: pointer;
+  background: var(--vscode-input-background, rgba(128, 128, 128, 0.25));
+  box-shadow: inset 0 0 0 1px
+    var(--vscode-checkbox-border, var(--vscode-input-border, transparent));
+  transition: background 0.15s ease;
+}
+
+.switch.is-on {
+  background: var(--vscode-button-background);
+  box-shadow: none;
+}
+
+.switch:focus-visible {
+  outline: 1px solid var(--vscode-focusBorder);
+  outline-offset: 2px;
+}
+
+.switch:disabled {
+  opacity: 0.5;
+  cursor: default;
+}
+
+.switch-knob {
+  position: absolute;
+  top: 2px;
+  left: 2px;
+  width: 18px;
+  height: 18px;
+  border-radius: 50%;
+  background: var(--vscode-input-foreground, #fff);
+  transition:
+    transform 0.15s ease,
+    background 0.15s ease;
+}
+
+.switch.is-on .switch-knob {
+  transform: translateX(30px);
+  background: var(--vscode-button-foreground, #fff);
+}
+
+.switch-text {
+  position: absolute;
+  top: 0;
+  height: 22px;
+  line-height: 22px;
+  font-size: 9px;
+  font-weight: 600;
+  letter-spacing: 0.04em;
+  pointer-events: none;
+}
+
+.switch-on {
+  left: 8px;
+  color: var(--vscode-button-foreground, #fff);
+  opacity: 0;
+}
+
+.switch-off {
+  right: 7px;
+  color: var(--vscode-descriptionForeground);
+  opacity: 1;
+}
+
+.switch.is-on .switch-on {
+  opacity: 1;
+}
+
+.switch.is-on .switch-off {
+  opacity: 0;
+}
+
 /* --- Sub-bar: provenance + run status ------------------------------------ */
 .sub-bar {
   display: flex;
@@ -479,21 +602,6 @@ body {
   );
 }
 
-.fidelity-preflight {
-  display: flex;
-  align-items: flex-start;
-  gap: 6px;
-  padding: 0 0 10px;
-  color: var(--vscode-descriptionForeground);
-  font-size: 11px;
-  line-height: 1.4;
-}
-
-.fidelity-preflight .codicon {
-  flex: 0 0 auto;
-  color: var(--vscode-charts-yellow, var(--vscode-descriptionForeground));
-}
-
 /* --- Scrolling content area ---------------------------------------------- */
 .content {
   flex: 1 1 auto;
diff --git a/ts/packages/typeagent-studio/src/extension.ts b/ts/packages/typeagent-studio/src/extension.ts
index e13a3c3d82..32e96bf911 100644
--- a/ts/packages/typeagent-studio/src/extension.ts
+++ b/ts/packages/typeagent-studio/src/extension.ts
@@ -327,8 +327,20 @@ export function activate(context: vscode.ExtensionContext): void {
     );
 
     const corpusTree = new CorpusTreeProvider(serviceRuntime);
+    // In-repo corpus files (`<repoRoot>/corpus/<agent>.utterances.jsonl`) are
+    // edited outside the extension (e.g. after "Seed in-repo corpus..." opens the
+    // new file for the user to paste utterances). Watch them so the Corpora tree
+    // refreshes on save/create/delete instead of only on manual refresh.
+    const corpusWatcher = vscode.workspace.createFileSystemWatcher(
+        "**/*.utterances.jsonl",
+    );
+    const refreshOnCorpusChange = () => corpusTree.refresh();
+    corpusWatcher.onDidCreate(refreshOnCorpusChange);
+    corpusWatcher.onDidChange(refreshOnCorpusChange);
+    corpusWatcher.onDidDelete(refreshOnCorpusChange);
     context.subscriptions.push(
         corpusTree,
+        corpusWatcher,
         vscode.window.registerTreeDataProvider(CORPUS_VIEW_ID, corpusTree),
         vscode.commands.registerCommand("typeagent-studio.refreshCorpora", () =>
             corpusTree.refresh(),
diff --git a/ts/packages/typeagent-studio/src/test/replayViewModel.spec.ts b/ts/packages/typeagent-studio/src/test/replayViewModel.spec.ts
index 0a545db7aa..8d78d47586 100644
--- a/ts/packages/typeagent-studio/src/test/replayViewModel.spec.ts
+++ b/ts/packages/typeagent-studio/src/test/replayViewModel.spec.ts
@@ -465,8 +465,6 @@ test("toFidelityMatrix produces one row per layer in order", () => {
     );
     assert.equal(view!.realizationA, "built (live)");
     assert.equal(view!.realizationB, "built (live)");
-    // No source side → no preflight hint.
-    assert.equal(view!.preflight, undefined);
 });
 
 test("toFidelityMatrix carries per-side status and reason", () => {
@@ -486,23 +484,3 @@ test("toFidelityMatrix carries per-side status and reason", () => {
     assert.equal(cacheRow!.b.status, "unavailable");
     assert.equal(cacheRow!.b.reason, "no cache at a ref");
 });
-
-test("toFidelityMatrix adds a preflight hint for a single source side", () => {
-    const view = toFidelityMatrix({
-        A: fidelityReport("built-live"),
-        B: fidelityReport("source"),
-    });
-    assert.ok(view);
-    assert.ok(view!.preflight);
-    assert.ok(/Side B is /.test(view!.preflight!));
-});
-
-test("toFidelityMatrix pluralizes the preflight hint for two source sides", () => {
-    const view = toFidelityMatrix({
-        A: fidelityReport("source"),
-        B: fidelityReport("source"),
-    });
-    assert.ok(view);
-    assert.ok(view!.preflight);
-    assert.ok(/Side A & B are /.test(view!.preflight!));
-});
diff --git a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
index 8964da1ade..2fdc39d947 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
@@ -137,23 +137,15 @@ let mode: StudioReplayMode = "nfa-grammar";
 // `wildcardValidation.diagnostics`, which `renderValidationNote` surfaces.
 let validateWildcards = false;
 
-/** Short toolbar label for each replay mode. */
-const MODE_LABEL: Record<StudioReplayMode, string> = {
-    "nfa-grammar": "Grammar",
-    "completionBased-cache": "Cache",
+/** Concise hover copy for the Cache on/off toggle. */
+const CACHE_TOOLTIP: Record<"on" | "off", string> = {
+    on: "Cache on: the working-tree side consults the live construction cache before the grammar. Click to turn off.",
+    off: "Cache off: both sides match grammar only and stay symmetric. Click to turn on.",
 };
-/** Hover copy explaining each mode and that the button toggles between them. */
-const MODE_TOOLTIP: Record<StudioReplayMode, string> = {
-    "nfa-grammar":
-        "Grammar mode: both versions match against the compiled grammar only — " +
-        "the construction cache is never consulted, so A and B stay symmetric. " +
-        "Cleanest signal for what a grammar or schema edit changed. " +
-        "Click to switch to Cache mode.",
-    "completionBased-cache":
-        "Cache mode: the working-tree side consults the live construction cache " +
-        "before the grammar (the default dispatcher path), so a cache hit reflects " +
-        "what the dispatcher would serve. Asymmetric — a B-side cache hit can mask " +
-        "a grammar regression. Click to switch to Grammar mode.",
+/** Concise hover copy for the Wildcard validation on/off toggle. */
+const VALIDATE_TOOLTIP: Record<"on" | "off", string> = {
+    on: "Wildcard validation on: the working-tree side runs the agent's real validator over wildcard matches (allowlisted agents only). Click to turn off.",
+    off: "Wildcard validation off: matches come from the grammar alone. Click to turn on.",
 };
 
 const root = document.getElementById("root")!;
@@ -186,23 +178,34 @@ const swapButton = toolButton("arrow-swap", "Swap A and B", () =>
 );
 swapButton.title = "Swap the base (A) and compare (B) versions.";
 
-// A two-state toggle for the replay mode. Only two modes exist, so a click
-// flips between them (no dropdown). Built by hand (not `toolButton`) so we keep
-// a handle on the label node and update it in place from `renderModeButton`.
-const modeButton = document.createElement("button");
-modeButton.type = "button";
-modeButton.className = "tool-button";
-const modeButtonText = text("");
-modeButton.append(codicon("settings"), modeButtonText);
-modeButton.addEventListener("click", () => cycleMode());
-// A two-state toggle for opt-in wildcard validation (L4a). Built by hand like
-// `modeButton` so we keep a handle on its label and lit state.
-const validateButton = document.createElement("button");
-validateButton.type = "button";
-validateButton.className = "tool-button";
-const validateButtonText = text("");
-validateButton.append(codicon("verified"), validateButtonText);
-validateButton.addEventListener("click", () => toggleValidateWildcards());
+// Replay options (construction cache, wildcard validation) are advanced knobs
+// that most runs leave at their defaults, so they live behind a gear popover
+// rather than cluttering the toolbar. Each is a VS Code-themed on/off switch
+// (see `.switch` in the stylesheet); the gear lights up while the popover open.
+const cacheSwitch = settingsSwitch(
+    "Cache",
+    () => mode === "completionBased-cache",
+    () => toggleCache(),
+    (on) => CACHE_TOOLTIP[on ? "on" : "off"],
+);
+const validateSwitch = settingsSwitch(
+    "Wildcard validation",
+    () => validateWildcards,
+    () => toggleValidateWildcards(),
+    (on) => VALIDATE_TOOLTIP[on ? "on" : "off"],
+);
+const settingsPopover = el("div", "settings-popover");
+settingsPopover.setAttribute("role", "group");
+settingsPopover.setAttribute("aria-label", "Replay options");
+settingsPopover.hidden = true;
+settingsPopover.append(cacheSwitch.row, validateSwitch.row);
+const settingsButton = toolButton("settings-gear", "Replay options", () =>
+    toggleSettingsPopover(),
+);
+settingsButton.setAttribute("aria-haspopup", "true");
+settingsButton.setAttribute("aria-expanded", "false");
+const settingsGroup = el("div", "bar-group settings-group");
+settingsGroup.append(settingsButton, settingsPopover);
 const runButton = toolButton("play", "Run", () => runReplay(), {
     primary: true,
     text: "Run",
@@ -220,7 +223,7 @@ actionBar.append(
     separator(),
     group(versionAButton.button, swapButton, versionBButton.button),
     separator(),
-    group(modeButton, validateButton),
+    settingsGroup,
     spacer(),
     runButton,
     connectionPill,
@@ -261,8 +264,26 @@ setControlsEnabled(false);
 renderConnection("connecting");
 restoreSelection();
 renderVersionButtons();
-renderModeButton();
-renderValidateButton();
+cacheSwitch.render();
+validateSwitch.render();
+
+// Dismiss the replay-options popover on an outside click or Escape, the way a
+// native menu behaves. The gear's own click is inside `settingsGroup`, so it
+// toggles without this handler immediately re-closing it.
+document.addEventListener("click", (event) => {
+    if (
+        !settingsPopover.hidden &&
+        !settingsGroup.contains(event.target as Node)
+    ) {
+        toggleSettingsPopover(false);
+    }
+});
+document.addEventListener("keydown", (event) => {
+    if (event.key === "Escape" && !settingsPopover.hidden) {
+        toggleSettingsPopover(false);
+        settingsButton.focus();
+    }
+});
 
 // --- Messaging ------------------------------------------------------------
 window.addEventListener("message", (event: MessageEvent) => {
@@ -364,49 +385,27 @@ function runReplay(): void {
     });
 }
 
-/** Flip the replay mode between its two states and persist the choice. */
-function cycleMode(): void {
+/** Flip the construction cache on/off and persist the choice. */
+function toggleCache(): void {
     mode = mode === "nfa-grammar" ? "completionBased-cache" : "nfa-grammar";
-    renderModeButton();
+    cacheSwitch.render();
     persistState({ mode });
 }
 
 /** Toggle opt-in wildcard validation and persist the choice. */
 function toggleValidateWildcards(): void {
     validateWildcards = !validateWildcards;
-    renderValidateButton();
+    validateSwitch.render();
     persistState({ validateWildcards });
 }
 
-/** Paint the wildcard-validation toggle's label, tooltip, and lit state. */
-function renderValidateButton(): void {
-    validateButtonText.textContent = "Validate";
-    validateButton.classList.toggle("is-active", validateWildcards);
-    validateButton.setAttribute("aria-pressed", String(validateWildcards));
-    const tip = validateWildcards
-        ? "Wildcard validation ON: the working-tree side additionally runs the " +
-          "agent's real validateWildcardMatch over its wildcard matches (the " +
-          "dispatcher's post-match step), dropping a match the agent rejects. " +
-          "Only for allowlisted agents (timer, list); a no-op otherwise. " +
-          "Click to turn off."
-        : "Wildcard validation OFF: matches are taken from the grammar alone. " +
-          "Click to additionally run the agent's real validateWildcardMatch on " +
-          "wildcard matches (allowlisted agents only).";
-    validateButton.title = tip;
-    validateButton.setAttribute(
-        "aria-label",
-        `Wildcard validation ${validateWildcards ? "on" : "off"}. ${tip}`,
-    );
-}
-
-/** Paint the mode toggle's label and tooltip from the current `mode`. */
-function renderModeButton(): void {
-    modeButtonText.textContent = MODE_LABEL[mode];
-    modeButton.title = MODE_TOOLTIP[mode];
-    modeButton.setAttribute(
-        "aria-label",
-        `Replay mode: ${MODE_LABEL[mode]}. ${MODE_TOOLTIP[mode]}`,
-    );
+/** Open/close the replay-options popover. Pass `force` to set a specific
+ *  state; omit to flip the current one. */
+function toggleSettingsPopover(force?: boolean): void {
+    const open = force ?? settingsPopover.hidden;
+    settingsPopover.hidden = !open;
+    settingsButton.classList.toggle("is-active", open);
+    settingsButton.setAttribute("aria-expanded", String(open));
 }
 
 /** Apply a version selection from the host picker to one side. */
@@ -485,7 +484,7 @@ function renderResult(
     renderValidationNote(result.wildcardValidation);
 
     // The per-side fidelity matrix: which deterministic layers actually ran on
-    // each version, plus a build-from-ref preflight when a side is source-only.
+    // each version.
     renderFidelity(result.sideFidelity);
 
     currentRows = toImpactRows(result.rows, result.methodA, result.methodB);
@@ -590,8 +589,8 @@ const FIDELITY_STATUS_ICON: Record<FidelityCell["status"], string> = {
 
 /** Render the per-side fidelity matrix into a collapsible panel, or hide it.
  *  Surfaces which deterministic layers ran on each version (A/B) with a status
- *  icon + hover reason, plus a build-from-ref preflight when a side is source
- *  -only — so the report is honest about exactly what it exercised. */
+ *  icon + hover reason — so the report is honest about exactly what it
+ *  exercised. */
 function renderFidelity(
     sideFidelity: StudioReplayResult["sideFidelity"] | undefined,
 ): void {
@@ -629,12 +628,6 @@ function renderFidelity(
     }
     details.append(grid);
 
-    if (view.preflight) {
-        const preflight = el("div", "fidelity-preflight");
-        preflight.append(codicon("lightbulb"), text(view.preflight));
-        details.append(preflight);
-    }
-
     fidelityEl.append(details);
     fidelityEl.hidden = false;
 }
@@ -1059,8 +1052,13 @@ function setControlsEnabled(enabled: boolean): void {
     swapButton.disabled = !enabled;
     versionAButton.button.disabled = !enabled;
     versionBButton.button.disabled = !enabled;
-    modeButton.disabled = !enabled;
-    validateButton.disabled = !enabled;
+    settingsButton.disabled = !enabled;
+    cacheSwitch.button.disabled = !enabled;
+    validateSwitch.button.disabled = !enabled;
+    // A run shouldn't leave the options popover hanging open.
+    if (!enabled) {
+        toggleSettingsPopover(false);
+    }
 }
 
 function setStatus(text: string): void {
@@ -1203,9 +1201,43 @@ function toolButton(
     return b;
 }
 
-/** A dropdown-style picker button: an optional side tag (A/B), the current
- *  selection, and a trailing chevron. Asks the host to open a native QuickPick
- *  on click. */
+/** A labelled VS Code-themed on/off switch for the replay-options popover.
+ *  `getState`/`onToggle` bind it to a boolean; `tooltipFor` supplies concise
+ *  hover copy per state. Returns the row, the switch button (so callers can
+ *  disable it), and a `render` that repaints from the current state. */
+function settingsSwitch(
+    label: string,
+    getState: () => boolean,
+    onToggle: () => void,
+    tooltipFor: (on: boolean) => string,
+): { row: HTMLElement; button: HTMLButtonElement; render: () => void } {
+    const row = el("div", "switch-row");
+    const labelEl = el("span", "switch-label");
+    labelEl.textContent = label;
+    labelEl.id = `switch-${label.replace(/\s+/g, "-").toLowerCase()}`;
+    const button = document.createElement("button");
+    button.type = "button";
+    button.className = "switch";
+    button.setAttribute("role", "switch");
+    button.setAttribute("aria-labelledby", labelEl.id);
+    const offText = el("span", "switch-text switch-off");
+    offText.textContent = "OFF";
+    const onText = el("span", "switch-text switch-on");
+    onText.textContent = "ON";
+    button.append(offText, onText, el("span", "switch-knob"));
+    button.addEventListener("click", () => onToggle());
+    row.append(labelEl, button);
+    const render = () => {
+        const on = getState();
+        button.classList.toggle("is-on", on);
+        button.setAttribute("aria-checked", String(on));
+        const tip = tooltipFor(on);
+        button.title = tip;
+        row.title = tip;
+    };
+    return { row, button, render };
+}
+
 function picker(
     description: string,
     onClick: () => void,
diff --git a/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts b/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts
index 6de9a8e512..c039fb960b 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/replayViewModel.ts
@@ -323,8 +323,6 @@ export interface FidelityMatrixView {
     /** How side B was realized, e.g. "source (git ref)". */
     realizationB: string;
     rows: FidelityMatrixRow[];
-    /** A "build-from-ref would add X" hint shown when a side is source-only. */
-    preflight?: string;
 }
 
 const FIDELITY_LAYER_ORDER: { key: FidelityLayer; label: string }[] = [
@@ -345,9 +343,8 @@ const FIDELITY_REALIZATION_LABEL: Record<
 
 /**
  * Map the core {@link SideFidelity} descriptor into a render-ready matrix:
- * one row per deterministic layer with each side's status + reason, plus a
- * preflight hint when a side only materialized source at a git ref (so a build
- * -from-ref pass would unlock the live-only layers). Pure + browser-neutral.
+ * one row per deterministic layer with each side's status + reason. Pure +
+ * browser-neutral.
  */
 export function toFidelityMatrix(
     sideFidelity: SideFidelity | undefined,
@@ -362,25 +359,11 @@ export function toFidelityMatrix(
             b: toFidelityCell(sideFidelity.B.layers[key]),
         }),
     );
-    const sources: string[] = [];
-    if (sideFidelity.A.realization === "source") {
-        sources.push("A");
-    }
-    if (sideFidelity.B.realization === "source") {
-        sources.push("B");
-    }
-    const view: FidelityMatrixView = {
+    return {
         realizationA: FIDELITY_REALIZATION_LABEL[sideFidelity.A.realization],
         realizationB: FIDELITY_REALIZATION_LABEL[sideFidelity.B.realization],
         rows,
     };
-    if (sources.length > 0) {
-        const plural = sources.length > 1;
-        view.preflight =
-            `Side ${sources.join(" & ")} ${plural ? "are" : "is"} materialized source read at a git ref. ` +
-            `Building from the ref would let ${plural ? "them" : "it"} run the construction cache and the agent's real wildcard validation — not yet available.`;
-    }
-    return view;
 }
 
 function toFidelityCell(report: FidelityCell): FidelityCell {

From 43e6e455f7c019218fc96f1d067f11d04604e65c Mon Sep 17 00:00:00 2001
From: typeagent-bot <typeagent-bot[bot]@users.noreply.github.com>
Date: Sat, 27 Jun 2026 04:41:25 +0000
Subject: [PATCH 07/15] style: apply prettier formatting and policy fixes

---
 ts/docs/plans/vscode-devx/STATUS.md | 36 ++++++++++++++---------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/ts/docs/plans/vscode-devx/STATUS.md b/ts/docs/plans/vscode-devx/STATUS.md
index 3695087529..f44924fc0f 100644
--- a/ts/docs/plans/vscode-devx/STATUS.md
+++ b/ts/docs/plans/vscode-devx/STATUS.md
@@ -13,24 +13,24 @@ Legend: ✅ done · 🟡 partial · ❌ not started.
 "Wired to dispatcher" = backed by the real TypeAgent dispatcher/engine rather
 than an in-memory stand-in.
 
-| Capability                                                     | Core logic                                                                  | UI                                                                 | Wired to dispatcher                                                        | Tested |
-| -------------------------------------------------------------- | --------------------------------------------------------------------------- | ------------------------------------------------------------------ | -------------------------------------------------------------------------- | ------ |
-| Sandboxes (lifecycle, agent load/unload)                       | ✅ in-memory                                                                | ✅ tree view (channel-backed; agent runtime is source of truth)    | ❌ in-memory only (no subprocess/isolated dispatcher)                      | ✅     |
-| Sandbox persistence across reload/restart                      | ✅                                                                          | ✅ (auto-restore)                                                  | n/a                                                                        | ✅     |
-| Corpora (federation: in-repo / captures / external / feedback) | ✅ file-backed                                                              | ✅ tree view                                                       | n/a                                                                        | ✅     |
-| Event Log (structured event stream)                            | 🟡 in-memory ring buffer                                                    | ✅ tree view (+ channel-backed source)                             | ❌ most emit sites unwired                                                 | ✅     |
-| Agent health (status bar + findings)                           | 🟡 heuristic/filesystem checks                                              | ✅ status bar                                                      | ❌ no real schema parse / grammar compile                                  | ✅     |
-| Collisions (cross-schema grammar overlap)                      | ✅ real NFA scanner over compiled `.ag.json`                                | ✅ tree view + Skipped group + auto-scan (+ channel-backed source) | n/a (reads compiled grammars)                                              | ✅     |
-| Feedback (thumbs up/down → corpus)                             | ✅                                                                          | ✅ command                                                         | n/a                                                                        | ✅     |
-| Replay / compare engine                                        | ✅ schema-enriched grammar (L1), construction-cache (L2), selectable two-mode (grammar/cache) + opt-in live wildcard validation (L4a) | ✅ Impact Report webview (`ActionDelta[]`)                         | 🟡 grammar + live construction cache + working-tree wildcard validation; no two-version build-from-ref (L4b, deferred to P-7) | ✅     |
-| Onboarding bridge (snapshot/restore, stale detection)          | ✅                                                                          | ✅ commands                                                        | ❌ in-memory bridge                                                        | ✅     |
-| Repo-root detection (find `packages/agents`)                   | ✅                                                                          | ✅ warn toast + status bar                                         | n/a                                                                        | ✅     |
-| Webview infrastructure (`webviewKit`)                          | ✅ CSP/nonce host + typed protocol                                          | ✅ singleton-panel host                                            | —                                                                          | ✅     |
-| Impact Report webview                                          | ✅ `replayCorpus` over channel                                              | ✅ context header, A/B controls, Grammar/Cache + Validate toggles, durable state | 🟡 grammar + construction-cache + working-tree wildcard validation (L1–L2, L4a) | ✅     |
-| Player corpus capture                                          | ❌                                                                          | ❌                                                                 | ❌                                                                         | ❌     |
-| Schema Studio                                                  | ❌                                                                          | ❌                                                                 | ❌                                                                         | ❌     |
-| Live Trace                                                     | ❌                                                                          | ❌                                                                 | ❌                                                                         | ❌     |
-| `agr-language` / `vscode-shell` refactor onto core             | 🟡 dependency edge only                                                     | —                                                                  | ❌ no behavioral integration                                               | ❌     |
+| Capability                                                     | Core logic                                                                                                                            | UI                                                                               | Wired to dispatcher                                                                                                           | Tested |
+| -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ------ |
+| Sandboxes (lifecycle, agent load/unload)                       | ✅ in-memory                                                                                                                          | ✅ tree view (channel-backed; agent runtime is source of truth)                  | ❌ in-memory only (no subprocess/isolated dispatcher)                                                                         | ✅     |
+| Sandbox persistence across reload/restart                      | ✅                                                                                                                                    | ✅ (auto-restore)                                                                | n/a                                                                                                                           | ✅     |
+| Corpora (federation: in-repo / captures / external / feedback) | ✅ file-backed                                                                                                                        | ✅ tree view                                                                     | n/a                                                                                                                           | ✅     |
+| Event Log (structured event stream)                            | 🟡 in-memory ring buffer                                                                                                              | ✅ tree view (+ channel-backed source)                                           | ❌ most emit sites unwired                                                                                                    | ✅     |
+| Agent health (status bar + findings)                           | 🟡 heuristic/filesystem checks                                                                                                        | ✅ status bar                                                                    | ❌ no real schema parse / grammar compile                                                                                     | ✅     |
+| Collisions (cross-schema grammar overlap)                      | ✅ real NFA scanner over compiled `.ag.json`                                                                                          | ✅ tree view + Skipped group + auto-scan (+ channel-backed source)               | n/a (reads compiled grammars)                                                                                                 | ✅     |
+| Feedback (thumbs up/down → corpus)                             | ✅                                                                                                                                    | ✅ command                                                                       | n/a                                                                                                                           | ✅     |
+| Replay / compare engine                                        | ✅ schema-enriched grammar (L1), construction-cache (L2), selectable two-mode (grammar/cache) + opt-in live wildcard validation (L4a) | ✅ Impact Report webview (`ActionDelta[]`)                                       | 🟡 grammar + live construction cache + working-tree wildcard validation; no two-version build-from-ref (L4b, deferred to P-7) | ✅     |
+| Onboarding bridge (snapshot/restore, stale detection)          | ✅                                                                                                                                    | ✅ commands                                                                      | ❌ in-memory bridge                                                                                                           | ✅     |
+| Repo-root detection (find `packages/agents`)                   | ✅                                                                                                                                    | ✅ warn toast + status bar                                                       | n/a                                                                                                                           | ✅     |
+| Webview infrastructure (`webviewKit`)                          | ✅ CSP/nonce host + typed protocol                                                                                                    | ✅ singleton-panel host                                                          | —                                                                                                                             | ✅     |
+| Impact Report webview                                          | ✅ `replayCorpus` over channel                                                                                                        | ✅ context header, A/B controls, Grammar/Cache + Validate toggles, durable state | 🟡 grammar + construction-cache + working-tree wildcard validation (L1–L2, L4a)                                               | ✅     |
+| Player corpus capture                                          | ❌                                                                                                                                    | ❌                                                                               | ❌                                                                                                                            | ❌     |
+| Schema Studio                                                  | ❌                                                                                                                                    | ❌                                                                               | ❌                                                                                                                            | ❌     |
+| Live Trace                                                     | ❌                                                                                                                                    | ❌                                                                               | ❌                                                                                                                            | ❌     |
+| `agr-language` / `vscode-shell` refactor onto core             | 🟡 dependency edge only                                                                                                               | —                                                                                | ❌ no behavioral integration                                                                                                  | ❌     |
 
 ## The long pole
 

From 1a5fbb642c94383065e409a61a43976224f01bd6 Mon Sep 17 00:00:00 2001
From: Tal Zaccai <talzacc@microsoft.com>
Date: Mon, 29 Jun 2026 13:32:49 -0600
Subject: [PATCH 08/15] Studio: retain Impact Report webview context when
 hidden

The Impact Report panel was created with retainContextWhenHidden:false, so switching away from the tab tore down and reloaded the webview. On reload the client repainted its defaults and fired a single, no-retry 'ready' handshake to re-pull connection + agent + result from the host; when that reply failed to land on the reveal, the panel was stranded on 'Connecting...' with the agent/branch context blank.

Add an opt-in retainContextWhenHidden option to WebviewKitPanelOptions (default false, no change for other panels) and enable it for the Impact Report so the webview keeps its DOM, live connection, selection, and rendered result while hidden. Navigating away and back no longer reloads the panel or depends on the reveal handshake.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../typeagent-studio/src/impactReportView.ts     | 13 ++++++++++---
 .../typeagent-studio/src/webviewKit/host.ts      | 16 +++++++++++++---
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/ts/packages/typeagent-studio/src/impactReportView.ts b/ts/packages/typeagent-studio/src/impactReportView.ts
index 8b64855330..a4a9037889 100644
--- a/ts/packages/typeagent-studio/src/impactReportView.ts
+++ b/ts/packages/typeagent-studio/src/impactReportView.ts
@@ -59,9 +59,11 @@ export function openImpactReport(
     // Subscription to the shared connection's state, disposed with the panel.
     let stateSub: { dispose(): void } | undefined;
     // The last completed result + its request id. Re-posted whenever the webview
-    // signals `ready` so a run that finished while the iframe was torn down (the
-    // panel is `retainContextWhenHidden: false`, so hidden panels drop posts) is
-    // recovered on reveal — the webview dedupes by request id.
+    // signals `ready` so a run whose result arrived before the webview was
+    // listening (e.g. the first load, or a full extension reload) is recovered
+    // on the next `ready` — the webview dedupes by request id. With the panel
+    // retaining context while hidden, navigate-away/back no longer reloads it,
+    // so this mainly guards the initial-load and extension-reload cases.
     let lastResult:
         | {
               requestId: number;
@@ -82,6 +84,11 @@ export function openImpactReport(
         title: `Impact Report — ${agent}`,
         scriptPath: ["dist", "webview", "impactReport.js"],
         stylePath: ["media", "impactReport.css"],
+        // The panel holds a live service connection and a rendered result, and
+        // the agent/connection context is re-pushed only via a single reveal
+        // `ready` handshake. Retain the context so navigating away and back
+        // keeps that state instead of tearing the webview down and reloading.
+        retainContextWhenHidden: true,
         onMessage: (raw) => void handleMessage(raw),
         onDispose: () => {
             stateSub?.dispose();
diff --git a/ts/packages/typeagent-studio/src/webviewKit/host.ts b/ts/packages/typeagent-studio/src/webviewKit/host.ts
index f4a996b686..ec4c359db9 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/host.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/host.ts
@@ -15,6 +15,15 @@ export interface WebviewKitPanelOptions {
      */
     instanceKey?: string;
     title: string;
+    /**
+     * Keep the webview's context (DOM + script state + live messaging channel)
+     * alive while the panel is hidden instead of tearing it down and reloading
+     * on reveal. Defaults to `false`. Set `true` for panels that hold live
+     * state (an open connection, a rendered result) so navigating away and back
+     * doesn't drop the agent/connection context and strand a one-shot reveal
+     * handshake. Costs memory while hidden.
+     */
+    retainContextWhenHidden?: boolean;
     /** Path segments (from the extension root) to the client script bundle. */
     scriptPath: string[];
     /** Path segments (from the extension root) to the stylesheet. */
@@ -31,8 +40,8 @@ export interface WebviewKitPanelOptions {
  * existing one instead of opening a second — builds a locked-down CSP'd HTML
  * shell with a per-load nonce, restricts `localResourceRoots` to the bundle +
  * media dirs, and exposes typed `post`/`dispose`. State restore is left to the
- * webview via `setState` — the panel intentionally does not
- * `retainContextWhenHidden`.
+ * webview via `setState`; panels that hold live state can opt into
+ * `retainContextWhenHidden` so a reveal doesn't tear down and reload them.
  */
 export class WebviewKitPanel {
     private static readonly live = new Map<string, WebviewKitPanel>();
@@ -81,7 +90,8 @@ export class WebviewKitPanel {
             vscode.ViewColumn.Active,
             {
                 enableScripts: true,
-                retainContextWhenHidden: false,
+                retainContextWhenHidden:
+                    options.retainContextWhenHidden ?? false,
                 localResourceRoots: [scriptRoot, styleRoot],
             },
         );

From cf4ab1e8024e275a144672281215286154cbb19c Mon Sep 17 00:00:00 2001
From: Tal Zaccai <talzacc@microsoft.com>
Date: Mon, 29 Jun 2026 14:18:56 -0600
Subject: [PATCH 09/15] Studio: address PR review comments on tree providers,
 corpus watcher, and resolver doc

- Corpus watcher: scope the FileSystemWatcher to <repoRoot>/corpus (or a corpus-rooted glob when the repo root is unknown) instead of **/*.utterances.jsonl, so it no longer churns on unrelated utterance files across a large workspace.

- Event log and collisions trees: only await the connection while their buffer/cache is empty (initial load). Once events or scan results have been collected, render them immediately so a mid-session disconnect keeps the data visible instead of falling back to an indefinite loading bar on reconnect.

- grammarResolver: drop an accidental duplicate JSDoc block above selectValidatedMatchAction, keeping the single accurate one.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../src/replay/grammarResolver.ts             | 11 ----------
 .../src/collisionsTreeProvider.ts             | 22 +++++++++++--------
 .../src/eventLogTreeProvider.ts               | 14 +++++++-----
 ts/packages/typeagent-studio/src/extension.ts | 16 +++++++++-----
 4 files changed, 33 insertions(+), 30 deletions(-)

diff --git a/ts/packages/typeagent-core/src/replay/grammarResolver.ts b/ts/packages/typeagent-core/src/replay/grammarResolver.ts
index a02d7d45d6..53d9f495ea 100644
--- a/ts/packages/typeagent-core/src/replay/grammarResolver.ts
+++ b/ts/packages/typeagent-core/src/replay/grammarResolver.ts
@@ -406,17 +406,6 @@ function topMatchAction(results: MatchResult[]): unknown {
     return actions.length > 0 ? actions[0].action : undefined;
 }
 
-/**
- * Mirror the dispatcher's `getValidatedMatches` for the wildcard step: walk the
- * heuristically-ranked matches and return the action of the first one the agent
- * accepts. A match with no wildcard capture (`wildcardCharCount === 0`) is
- * accepted without consulting the validator (exactly as the dispatcher
- * short-circuits); a wildcard match is dropped only on an explicit `false`
- * verdict, and the walk continues to the next candidate (the dispatcher's
- * fall-back-to-a-lower-match behavior). Returns `undefined` when every candidate
- * was rejected — the row then becomes `needs-explanation`, the deterministic
- * stand-in for the dispatcher falling back to the LLM.
- */
 /**
  * Mirror the dispatcher's `getValidatedMatches` for the wildcard step: walk the
  * heuristically-ranked matches and return the action of the first one the agent
diff --git a/ts/packages/typeagent-studio/src/collisionsTreeProvider.ts b/ts/packages/typeagent-studio/src/collisionsTreeProvider.ts
index aaf6ea899f..767fc3f98c 100644
--- a/ts/packages/typeagent-studio/src/collisionsTreeProvider.ts
+++ b/ts/packages/typeagent-studio/src/collisionsTreeProvider.ts
@@ -108,15 +108,19 @@ export class CollisionsTreeProvider
 
     async getChildren(row?: CollisionRow): Promise<CollisionRow[]> {
         if (!row) {
-            // Show the native loading bar while connecting; once connected,
-            // render scanned collisions (or the "no collisions" placeholder).
-            await this.whenConnected();
-            if (
-                !this.connected &&
-                this.entries.length === 0 &&
-                this.skipped.length === 0
-            ) {
-                return [];
+            // Render the last known scan immediately so a mid-session
+            // disconnect keeps results visible instead of dropping back to the
+            // loading bar. Only gate on the connection (showing the native
+            // loading bar) during the initial load, before anything is scanned.
+            if (this.entries.length === 0 && this.skipped.length === 0) {
+                await this.whenConnected();
+                if (
+                    !this.connected &&
+                    this.entries.length === 0 &&
+                    this.skipped.length === 0
+                ) {
+                    return [];
+                }
             }
             return buildCollisionRows(this.entries, this.skipped);
         }
diff --git a/ts/packages/typeagent-studio/src/eventLogTreeProvider.ts b/ts/packages/typeagent-studio/src/eventLogTreeProvider.ts
index c172cb0099..614c039258 100644
--- a/ts/packages/typeagent-studio/src/eventLogTreeProvider.ts
+++ b/ts/packages/typeagent-studio/src/eventLogTreeProvider.ts
@@ -75,11 +75,15 @@ export class EventLogTreeProvider
         if (row) {
             return [];
         }
-        // Show the native loading bar while connecting; once connected, render
-        // buffered events (or the "no events yet" placeholder).
-        await this.whenConnected();
-        if (!this.connected && this.entries.length === 0) {
-            return [];
+        // Render buffered events immediately so a mid-session disconnect keeps
+        // the ring visible instead of dropping back to the loading bar. Only
+        // gate on the connection (showing the native loading bar) during the
+        // initial load, when nothing has been buffered yet.
+        if (this.entries.length === 0) {
+            await this.whenConnected();
+            if (!this.connected && this.entries.length === 0) {
+                return [];
+            }
         }
         return buildEventLogRows(this.entries);
     }
diff --git a/ts/packages/typeagent-studio/src/extension.ts b/ts/packages/typeagent-studio/src/extension.ts
index 32e96bf911..f9cdfe8f4c 100644
--- a/ts/packages/typeagent-studio/src/extension.ts
+++ b/ts/packages/typeagent-studio/src/extension.ts
@@ -329,11 +329,17 @@ export function activate(context: vscode.ExtensionContext): void {
     const corpusTree = new CorpusTreeProvider(serviceRuntime);
     // In-repo corpus files (`<repoRoot>/corpus/<agent>.utterances.jsonl`) are
     // edited outside the extension (e.g. after "Seed in-repo corpus..." opens the
-    // new file for the user to paste utterances). Watch them so the Corpora tree
-    // refreshes on save/create/delete instead of only on manual refresh.
-    const corpusWatcher = vscode.workspace.createFileSystemWatcher(
-        "**/*.utterances.jsonl",
-    );
+    // new file for the user to paste utterances). Watch only that directory so the
+    // Corpora tree refreshes on save/create/delete without churning on unrelated
+    // `*.utterances.jsonl` files elsewhere in a large workspace.
+    const corpusWatcherPattern = repoRootInfo.repoRoot
+        ? new vscode.RelativePattern(
+              path.join(repoRootInfo.repoRoot, "corpus"),
+              "*.utterances.jsonl",
+          )
+        : "**/corpus/*.utterances.jsonl";
+    const corpusWatcher =
+        vscode.workspace.createFileSystemWatcher(corpusWatcherPattern);
     const refreshOnCorpusChange = () => corpusTree.refresh();
     corpusWatcher.onDidCreate(refreshOnCorpusChange);
     corpusWatcher.onDidChange(refreshOnCorpusChange);

From 0c97d5fd66b1eef6c8076011e016a88208c94e3b Mon Sep 17 00:00:00 2001
From: Tal Zaccai <talzacc@microsoft.com>
Date: Mon, 29 Jun 2026 14:36:37 -0600
Subject: [PATCH 10/15] Studio: remove corpus file watcher; refresh the Corpora
 tree only on explicit UI actions

Drop the FileSystemWatcher on *.utterances.jsonl. The Corpora tree now refreshes only on in-extension actions (seeding an in-repo file, adding an external source, recording feedback) and the manual Refresh command -- each already calls corpusTree.refresh() directly -- rather than auto-updating on out-of-band edits to corpus files on disk. This supersedes the earlier watcher-scoping change for the same review thread.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 ts/packages/typeagent-studio/src/extension.ts | 23 ++++---------------
 1 file changed, 5 insertions(+), 18 deletions(-)

diff --git a/ts/packages/typeagent-studio/src/extension.ts b/ts/packages/typeagent-studio/src/extension.ts
index f9cdfe8f4c..25c6568485 100644
--- a/ts/packages/typeagent-studio/src/extension.ts
+++ b/ts/packages/typeagent-studio/src/extension.ts
@@ -327,26 +327,13 @@ export function activate(context: vscode.ExtensionContext): void {
     );
 
     const corpusTree = new CorpusTreeProvider(serviceRuntime);
-    // In-repo corpus files (`<repoRoot>/corpus/<agent>.utterances.jsonl`) are
-    // edited outside the extension (e.g. after "Seed in-repo corpus..." opens the
-    // new file for the user to paste utterances). Watch only that directory so the
-    // Corpora tree refreshes on save/create/delete without churning on unrelated
-    // `*.utterances.jsonl` files elsewhere in a large workspace.
-    const corpusWatcherPattern = repoRootInfo.repoRoot
-        ? new vscode.RelativePattern(
-              path.join(repoRootInfo.repoRoot, "corpus"),
-              "*.utterances.jsonl",
-          )
-        : "**/corpus/*.utterances.jsonl";
-    const corpusWatcher =
-        vscode.workspace.createFileSystemWatcher(corpusWatcherPattern);
-    const refreshOnCorpusChange = () => corpusTree.refresh();
-    corpusWatcher.onDidCreate(refreshOnCorpusChange);
-    corpusWatcher.onDidChange(refreshOnCorpusChange);
-    corpusWatcher.onDidDelete(refreshOnCorpusChange);
+    // The Corpora tree refreshes only on explicit in-extension actions (seeding
+    // an in-repo file, adding an external source, recording feedback) and the
+    // manual Refresh command -- each of those calls corpusTree.refresh()
+    // directly. We intentionally do not watch the filesystem for out-of-band
+    // edits to *.utterances.jsonl.
     context.subscriptions.push(
         corpusTree,
-        corpusWatcher,
         vscode.window.registerTreeDataProvider(CORPUS_VIEW_ID, corpusTree),
         vscode.commands.registerCommand("typeagent-studio.refreshCorpora", () =>
             corpusTree.refresh(),

From 4d91b839c5bb8544892627ea6b943fe4afcdadc4 Mon Sep 17 00:00:00 2001
From: Tal Zaccai <talzacc@microsoft.com>
Date: Mon, 29 Jun 2026 15:42:09 -0600
Subject: [PATCH 11/15] Replace wildcard-validation allowlist with a manifest
 capability

Agents now opt their validateWildcardMatch into replay via a new optional AppAgentManifest field (replaySafeWildcardValidator); timer and list set it, preserving current behavior. The host gates on the manifest flag (reading it lazily, failing closed to grammar-only), so core becomes pure load-and-run mechanism: the hardcoded allowlist, its option, and the agent-not-in-allowlist diagnostic are removed.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 ts/packages/agentSdk/src/agentInterface.ts    |  5 ++
 ts/packages/agents/list/src/listManifest.json |  1 +
 .../agents/timer/src/timerManifest.json       |  1 +
 .../studio-service/src/wildcardValidation.ts  | 47 +++++++++---
 .../src/replay/grammarResolver.ts             |  2 +-
 .../src/replay/wildcardValidator.ts           | 74 +++++--------------
 .../typeagent-core/src/runtime/index.ts       |  7 +-
 .../src/runtime/studioRuntimeCore.ts          | 41 +++++-----
 .../test/grammarReplayResolver.spec.ts        |  8 +-
 .../typeagent-core/test/sideFidelity.spec.ts  |  7 +-
 .../test/wildcardValidator.spec.ts            | 40 +---------
 .../typeagent-studio/src/impactReportView.ts  |  6 +-
 .../src/webviewKit/client/impactReport.ts     | 26 ++-----
 .../src/webviewKit/protocol.ts                |  4 +-
 14 files changed, 104 insertions(+), 165 deletions(-)

diff --git a/ts/packages/agentSdk/src/agentInterface.ts b/ts/packages/agentSdk/src/agentInterface.ts
index 24fc1ff258..abc86ecbd7 100644
--- a/ts/packages/agentSdk/src/agentInterface.ts
+++ b/ts/packages/agentSdk/src/agentInterface.ts
@@ -43,6 +43,11 @@ export type AppAgentManifest = {
     // Registered flow programs: actionName → path to .flow.json (relative to manifest file)
     flows?: Record<string, string>;
     allowDynamicAgents?: boolean; // whether this agent can add/remove dynamic sub-agents at runtime, default is false
+    // Whether this agent's validateWildcardMatch is safe to run during offline
+    // replay: deterministic, free of side effects, and tolerant of a stub
+    // SessionContext (no live agentContext, storage, or network). Default false.
+    // Studio replay only runs the real validator for agents that opt in here.
+    replaySafeWildcardValidator?: boolean;
 } & ActionManifest;
 
 export type SchemaTypeNames = {
diff --git a/ts/packages/agents/list/src/listManifest.json b/ts/packages/agents/list/src/listManifest.json
index 0b7495e0c4..23162190ab 100644
--- a/ts/packages/agents/list/src/listManifest.json
+++ b/ts/packages/agents/list/src/listManifest.json
@@ -1,6 +1,7 @@
 {
   "emojiChar": "📝",
   "description": "Agent to create and manage lists",
+  "replaySafeWildcardValidator": true,
   "schema": {
     "description": "List agent with actions to create lists, show list items, add and remove list items",
     "originalSchemaFile": "./listSchema.ts",
diff --git a/ts/packages/agents/timer/src/timerManifest.json b/ts/packages/agents/timer/src/timerManifest.json
index 98f0e2f907..d89d94aff7 100644
--- a/ts/packages/agents/timer/src/timerManifest.json
+++ b/ts/packages/agents/timer/src/timerManifest.json
@@ -1,6 +1,7 @@
 {
   "emojiChar": "⏰",
   "description": "Agent for setting reminders that fire as agent-initiated messages",
+  "replaySafeWildcardValidator": true,
   "schema": {
     "description": "Timer agent: set, list, and cancel reminders. When a reminder fires, the agent pushes a message to the chat via SessionContext.beginAgentThread (no preceding user request).",
     "originalSchemaFile": "./timerSchema.ts",
diff --git a/ts/packages/studio-service/src/wildcardValidation.ts b/ts/packages/studio-service/src/wildcardValidation.ts
index 341796a2f9..1b67692cbb 100644
--- a/ts/packages/studio-service/src/wildcardValidation.ts
+++ b/ts/packages/studio-service/src/wildcardValidation.ts
@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 /**
- * Production wiring for the replay wildcard validator (fidelity rung L4a).
+ * Production wiring for the replay wildcard validator.
  *
  * The core `@typeagent/core` runtime exposes the validation algorithm but is
  * dependency-light and does NOT know how to load a real agent module. The real
@@ -27,15 +27,20 @@
 
 import {
     createWildcardMatchValidator,
-    DEFAULT_WILDCARD_VALIDATION_ALLOWLIST,
     type ReplayAppAgentLoader,
     type ReplayValidatableAgent,
     type WildcardMatchValidator,
 } from "@typeagent/core/runtime";
 
+/** The single manifest field that gates replay wildcard validation. */
+interface ReplaySafeManifestLike {
+    replaySafeWildcardValidator?: boolean;
+}
+
 /** Minimal structural view of an `AppAgentProvider` (avoids a type-only dep). */
 interface AppAgentProviderLike {
     getAppAgentNames(): string[];
+    getAppAgentManifest(agentName: string): Promise<ReplaySafeManifestLike>;
     loadAppAgent(agentName: string): Promise<ReplayValidatableAgent>;
     unloadAppAgent(agentName: string): Promise<void>;
 }
@@ -118,20 +123,42 @@ function createDefaultLoader(): ReplayAppAgentLoader {
     };
 }
 
+/**
+ * Whether the agent's manifest opts its `validateWildcardMatch` into replay. Any
+ * lookup failure (provider unavailable, agent unknown, manifest read error) is
+ * treated as not safe, so replay stays grammar-only.
+ */
+async function isReplaySafe(agentName: string): Promise<boolean> {
+    try {
+        const providers = await loadProviders();
+        if (providers === undefined) {
+            return false;
+        }
+        const provider = findProvider(providers, agentName);
+        if (provider === undefined) {
+            return false;
+        }
+        const manifest = await provider.getAppAgentManifest(agentName);
+        return manifest.replaySafeWildcardValidator === true;
+    } catch {
+        return false;
+    }
+}
+
 /**
  * Build the runtime's `resolveWildcardValidator` option: returns a validator
- * only for allowlisted agents (so a non-allowlisted agent stays grammar-only
- * and the run doesn't claim a validation it can't perform), backed by the
- * lazy default loader. The dynamic import is deferred until a wildcard match
- * actually triggers `loadAppAgent`, so a run that never hits a wildcard never
- * pays for the provider.
+ * only for agents whose manifest declares `replaySafeWildcardValidator` (so any
+ * other agent stays grammar-only and the run doesn't claim a validation it can't
+ * perform), backed by the lazy default loader. The dynamic import is deferred
+ * until a wildcard match actually triggers a lookup, so a run that never hits a
+ * wildcard never pays for the provider.
  */
 export function createDefaultWildcardValidatorResolver(): (
     agentName: string,
-) => WildcardMatchValidator | undefined {
+) => Promise<WildcardMatchValidator | undefined> {
     const loader = createDefaultLoader();
-    return (agentName) => {
-        if (!DEFAULT_WILDCARD_VALIDATION_ALLOWLIST.includes(agentName)) {
+    return async (agentName) => {
+        if (!(await isReplaySafe(agentName))) {
             return undefined;
         }
         return createWildcardMatchValidator(agentName, { loader });
diff --git a/ts/packages/typeagent-core/src/replay/grammarResolver.ts b/ts/packages/typeagent-core/src/replay/grammarResolver.ts
index 53d9f495ea..278a4060af 100644
--- a/ts/packages/typeagent-core/src/replay/grammarResolver.ts
+++ b/ts/packages/typeagent-core/src/replay/grammarResolver.ts
@@ -544,7 +544,7 @@ export function createGrammarReplayResolver(
 
             const results = g.store.match(entry.utterance);
 
-            // Wildcard validation (L4a): on the working-tree side, run the
+            // Wildcard validation: on the working-tree side, run the
             // agent's real `validateWildcardMatch` over the ranked candidates and
             // resolve the first accepted match — mirroring the dispatcher's
             // post-match `getValidatedMatches`. Never on a git ref (we can't load
diff --git a/ts/packages/typeagent-core/src/replay/wildcardValidator.ts b/ts/packages/typeagent-core/src/replay/wildcardValidator.ts
index c1f839ffd2..c890ade187 100644
--- a/ts/packages/typeagent-core/src/replay/wildcardValidator.ts
+++ b/ts/packages/typeagent-core/src/replay/wildcardValidator.ts
@@ -2,36 +2,34 @@
 // Licensed under the MIT License.
 
 /**
- * Wildcard-match validation for the replay path (fidelity rung "L4a").
+ * Wildcard-match validation for the replay path.
  *
  * The dispatcher's only beyond-grammar determinism is a post-match step: for
  * each candidate match that captured a wildcard, it calls the agent's
  * `validateWildcardMatch(action, sessionContext)` and DROPS the match if the
  * agent returns `false` (then falls back to a lower-ranked match or the LLM).
- * See `dispatcher/.../translation/matchRequest.ts` `getValidatedMatches`.
  *
  * Grammar matching alone cannot reproduce this, so a `.agr` that still matches
  * an utterance can look unchanged in replay even though the real dispatcher
  * would reject the wildcard value. This module runs the agent's REAL validator
- * so that fidelity axis shows up in the Impact Report.
+ * so that axis shows up in the Impact Report.
  *
- * ## Scope & honesty (deliberate, see `files/replay-l4a-design.md`)
- * - **Working-tree side only.** Loading arbitrary git-ref agent code is L4b
- *   build machinery; the caller never runs this for a git ref.
+ * Scope and guarantees:
+ * - **Working-tree side only.** The caller never runs this for a git ref.
  * - **Built module.** {@link ReplayAppAgentLoader.loadAppAgent} returns the
  *   agent's BUILT module, so uncommitted validator `.ts` edits are reflected
- *   only after a rebuild (grammar edits — the usual regression experiment — are
- *   already read from source by the grammar resolver).
+ *   only after a rebuild (grammar edits are read from source by the grammar
+ *   resolver).
  * - **Fail-OPEN.** Only an explicit `=== false` from the agent rejects a match.
- *   A missing method / load failure / thrown validator / non-allowlisted agent
- *   all ACCEPT the match and record a diagnostic — replay must never fabricate a
- *   lost match from infrastructure noise.
- * - **Allowlist.** Only validators known to be deterministic + side-effect-free
- *   run by default ({@link DEFAULT_WILDCARD_VALIDATION_ALLOWLIST}); e.g.
- *   `weather`'s validator does live network geocoding and is intentionally
- *   excluded.
+ *   A missing method, load failure, or thrown validator all ACCEPT the match and
+ *   record a diagnostic — replay must never fabricate a lost match from
+ *   infrastructure noise.
  * - **Entity-wildcard validation is out of scope** (it needs conversation
  *   memory replay does not have).
+ *
+ * Which agents run here is decided by the host, not this module: the host only
+ * builds a validator for agents that declare they are replay-safe. This module
+ * always loads and runs the validator it is asked to build.
  */
 
 /**
@@ -64,7 +62,6 @@ export interface ReplayAppAgentLoader {
  * verdict — recorded so the run can honestly report that validation degraded.
  */
 export type WildcardValidationDiagnostic =
-    | "agent-not-in-allowlist"
     | "no-validator"
     | "load-failed"
     | "errored";
@@ -94,48 +91,17 @@ export interface WildcardMatchValidator {
     dispose(): Promise<void>;
 }
 
-/**
- * Agents whose `validateWildcardMatch` validates cleanly during replay — a
- * deterministic function of the action that does NOT depend on live session
- * state (verified end-to-end against the real built modules):
- * - `timer` — `tryParseWhen` on the action params; context ignored. Rejects a
- *   wildcard that isn't a real duration/timestamp.
- * - `list` — `simpleNoun` heuristic over the action; context ignored.
- *
- * `player` is deliberately EXCLUDED even though its source returns `true` when
- * `context.agentContext.spotify` is absent. It runs `execMode: "separate"`, so
- * the loader returns an RPC proxy to a child process that reconstructs its OWN
- * `SessionContext` — where `agentContext` is `undefined`, not our stub's `{}`.
- * The child then throws `Cannot read properties of undefined (reading 'spotify')`
- * before reaching its self-degrade guard. And even if it didn't throw, it can
- * only ever return `true` without a live Spotify client, so it adds no fidelity.
- * It would just fail-open with an `errored` diagnostic and a misleading
- * "degraded" indicator — so we keep it grammar-only.
- *
- * `weather`/`markdown`/`photo`/`androidMobile`/`taskflow`/`powershell` are NOT
- * here: weather geocodes over the network, and the others are unverified.
- */
-export const DEFAULT_WILDCARD_VALIDATION_ALLOWLIST: readonly string[] = [
-    "timer",
-    "list",
-];
-
 export interface CreateWildcardMatchValidatorOptions {
     loader: ReplayAppAgentLoader;
-    /** Defaults to {@link DEFAULT_WILDCARD_VALIDATION_ALLOWLIST}. */
-    allowlist?: readonly string[];
 }
 
 /**
- * A no-op `SessionContext`-shaped stub for replay validation. The validators we
- * allowlist (timer/list) ignore the context entirely; the stub still exposes
+ * A no-op `SessionContext`-shaped stub for replay validation. Deterministic
+ * validators ignore the context entirely; the stub still exposes
  * `agentContext: {}` — NOT `undefined` — so that any in-process validator that
  * defensively reads `context.agentContext.<x>` degrades gracefully instead of
- * throwing. (Out-of-process agents like `player` reconstruct their own context
- * in the child and never see this stub, which is why `player` is excluded from
- * the default allowlist — see {@link DEFAULT_WILDCARD_VALIDATION_ALLOWLIST}.)
- * The interactive methods throw, since a validator that calls them is out of the
- * supported set and should fail-open via the wrapper's try/catch.
+ * throwing. The interactive methods throw, since a validator that calls them is
+ * out of the supported set and should fail-open via the wrapper's try/catch.
  */
 export function createReplaySessionContextStub(): Record<string, unknown> {
     const unsupported = (name: string) => () => {
@@ -179,9 +145,6 @@ export function createWildcardMatchValidator(
     options: CreateWildcardMatchValidatorOptions,
 ): WildcardMatchValidator {
     const { loader } = options;
-    const allowlist =
-        options.allowlist ?? DEFAULT_WILDCARD_VALIDATION_ALLOWLIST;
-    const allowed = allowlist.includes(agentName);
     const diagnostics = new Set<WildcardValidationDiagnostic>();
     const stubContext = createReplaySessionContextStub();
 
@@ -220,9 +183,6 @@ export function createWildcardMatchValidator(
         },
 
         async validateMatch(actions): Promise<WildcardValidationOutcome> {
-            if (!allowed) {
-                return note("agent-not-in-allowlist");
-            }
             const agent = await ensureAgent();
             if (agent === undefined) {
                 return { rejected: false, diagnostic: "load-failed" };
diff --git a/ts/packages/typeagent-core/src/runtime/index.ts b/ts/packages/typeagent-core/src/runtime/index.ts
index 221fa9f385..687c3f8dba 100644
--- a/ts/packages/typeagent-core/src/runtime/index.ts
+++ b/ts/packages/typeagent-core/src/runtime/index.ts
@@ -17,14 +17,13 @@ export * from "./studioServiceAuth.js";
 export * from "./studioWorkspaceId.js";
 export { getDefaultPhaseInputs } from "./onboardingPhaseInputs.js";
 
-// Wildcard-match validation (replay fidelity rung L4a). Re-exported here so the
-// service host can wire a real `ReplayAppAgentLoader` (the production loader
-// lives outside this dependency-light package) without reaching into `replay/`.
+// Replay wildcard-match validation. Re-exported here so the service host can
+// wire a real `ReplayAppAgentLoader` (the production loader lives outside this
+// dependency-light package) without reaching into `replay/`.
 export {
     createWildcardMatchValidator,
     createReplaySessionContextStub,
     matchActionPayloads,
-    DEFAULT_WILDCARD_VALIDATION_ALLOWLIST,
 } from "../replay/wildcardValidator.js";
 export type {
     ReplayAppAgentLoader,
diff --git a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
index c84ed12f02..b54354e624 100644
--- a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
+++ b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
@@ -267,8 +267,8 @@ export interface StudioReplayRequest {
      * Opt-in: additionally run the agent's real `validateWildcardMatch` over the
      * working-tree side's wildcard grammar matches (the dispatcher's post-match
      * validation step). Default off. Only takes effect when the runtime was given
-     * a `resolveWildcardValidator` and the agent is in the validation allowlist;
-     * otherwise it is a no-op. Working-tree side only; fail-open. See
+     * a `resolveWildcardValidator` and the agent's manifest declares it
+     * replay-safe; otherwise it is a no-op. Working-tree side only; fail-open. See
      * {@link StudioReplayResult.wildcardValidation}.
      */
     validateWildcards?: boolean;
@@ -364,8 +364,8 @@ export interface StudioReplayResult {
     /**
      * Per-side fidelity descriptor: how each version was realized and which
      * deterministic layers actually ran. Derived purely from signals the replay
-     * already computed (per-side method, mode, version kind, the L4a validation
-     * pass) so the Impact Report can show an honest "what ran" matrix.
+     * already computed (per-side method, mode, version kind, the wildcard
+     * validation pass) so the Impact Report can show an honest "what ran" matrix.
      */
     sideFidelity: SideFidelity;
     /**
@@ -375,9 +375,8 @@ export interface StudioReplayResult {
 }
 
 /**
- * Reports the opt-in wildcard-validation pass (replay fidelity rung L4a). Only
- * present when validation was enabled AND consulted on at least one wildcard
- * match. `diagnostics` lists fail-open reasons (e.g. the agent's module
+ * Reports the opt-in wildcard-validation pass. Only present when validation was
+ * enabled AND consulted on at least one wildcard match. `diagnostics` lists fail-open reasons (e.g. the agent's module
  * couldn't load, or its validator threw), so the UI can show that validation
  * degraded rather than silently claiming full fidelity.
  */
@@ -545,12 +544,6 @@ function fidelityWildcardValidationLayer(
                 reason: "Agent module couldn't be loaded (e.g. a packaged build ships no agent code) — matches fell back to the grammar.",
             };
         }
-        if (diagnostics.includes("agent-not-in-allowlist")) {
-            return {
-                status: "skipped",
-                reason: "Agent isn't in the validation allowlist — its validator wasn't run.",
-            };
-        }
         if (diagnostics.includes("no-validator")) {
             return {
                 status: "skipped",
@@ -840,15 +833,19 @@ export interface CreateStudioRuntimeOptions {
     /**
      * Builds the agent's wildcard-match validator for an opt-in
      * (`validateWildcards`) replay run. Returns `undefined` when the host can't
-     * validate this agent (e.g. not in the allowlist, or no agent loader
-     * available), in which case replay stays grammar-only. Injected because the
-     * production loader pulls the dispatcher's agent providers, which live
-     * outside this dependency-light package; tests inject a fake-loader-backed
-     * validator. The runtime disposes the returned validator at run end.
+     * validate this agent (e.g. the agent's manifest doesn't declare it
+     * replay-safe, or no agent loader is available), in which case replay stays
+     * grammar-only. Injected because the production loader pulls the
+     * dispatcher's agent providers, which live outside this dependency-light
+     * package; tests inject a fake-loader-backed validator. The runtime disposes
+     * the returned validator at run end.
      */
     resolveWildcardValidator?: (
         agentName: string,
-    ) => WildcardMatchValidator | undefined;
+    ) =>
+        | WildcardMatchValidator
+        | undefined
+        | Promise<WildcardMatchValidator | undefined>;
     collisions?: CollisionService;
     /**
      * Scans agents' compiled grammars for collisions. Injected so tests can
@@ -1326,7 +1323,7 @@ export function createStudioRuntimeCore(
             let method: StudioReplayMethod = "identity";
             let methodA: StudioReplayMethod = "identity";
             let methodB: StudioReplayMethod = "identity";
-            // Opt-in wildcard validator (L4a); kept in scope so its diagnostics
+            // Opt-in wildcard validator; kept in scope so its diagnostics
             // can be read after the run and it can be disposed (unload the agent).
             let wildcardValidator: WildcardMatchValidator | undefined;
             let activeGrammarResolver: GrammarReplayResolver | undefined;
@@ -1361,11 +1358,11 @@ export function createStudioRuntimeCore(
                     // Opt-in: build the agent's wildcard validator so the
                     // working-tree side runs the dispatcher's real post-match
                     // `validateWildcardMatch`. `undefined` when the host has no
-                    // loader or the agent isn't validatable — replay then stays
+                    // loader or the agent doesn't opt in — replay then stays
                     // grammar-only (a silent no-op, not an error).
                     wildcardValidator =
                         request.validateWildcards === true
-                            ? options.resolveWildcardValidator?.(
+                            ? await options.resolveWildcardValidator?.(
                                   replayOptions.agent,
                               )
                             : undefined;
diff --git a/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts b/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts
index 4e01abe32b..5ed9fb935b 100644
--- a/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts
+++ b/ts/packages/typeagent-core/test/grammarReplayResolver.spec.ts
@@ -500,7 +500,7 @@ describe("resolveGrammarReplayTarget + schema enrichment (L1)", () => {
 });
 
 // ---------------------------------------------------------------------------
-// L4a — working-tree wildcard validation
+// Working-tree wildcard validation
 // ---------------------------------------------------------------------------
 
 // A grammar whose <PlayTrack> rule captures a greedy wildcard (so a match has
@@ -521,7 +521,7 @@ function wildcardLoader(agent: ReplayValidatableAgent): ReplayAppAgentLoader {
     };
 }
 
-/** A validator (allowlisted for "demo") that rejects a given track name. */
+/** A validator (for "demo") that rejects a given track name. */
 function rejectTrackValidator(reject: string): WildcardMatchValidator {
     const agent: ReplayValidatableAgent = {
         async validateWildcardMatch(a) {
@@ -532,11 +532,10 @@ function rejectTrackValidator(reject: string): WildcardMatchValidator {
     };
     return createWildcardMatchValidator("demo", {
         loader: wildcardLoader(agent),
-        allowlist: ["demo"],
     });
 }
 
-describe("grammar resolver wildcard validation (L4a)", () => {
+describe("grammar resolver wildcard validation", () => {
     let dir: string;
 
     beforeEach(() => {
@@ -670,7 +669,6 @@ describe("selectValidatedMatchAction (ranked-list contract)", () => {
         };
         return createWildcardMatchValidator("demo", {
             loader: wildcardLoader(agent),
-            allowlist: ["demo"],
         });
     }
 
diff --git a/ts/packages/typeagent-core/test/sideFidelity.spec.ts b/ts/packages/typeagent-core/test/sideFidelity.spec.ts
index 3288539ee7..c2ad95bb58 100644
--- a/ts/packages/typeagent-core/test/sideFidelity.spec.ts
+++ b/ts/packages/typeagent-core/test/sideFidelity.spec.ts
@@ -137,11 +137,8 @@ describe("deriveSideFidelity", () => {
         );
     });
 
-    it("wildcard validation maps allowlist/no-validator to skipped", () => {
-        for (const diagnostic of [
-            "agent-not-in-allowlist",
-            "no-validator",
-        ] as const) {
+    it("wildcard validation maps no-validator to skipped", () => {
+        for (const diagnostic of ["no-validator"] as const) {
             const skipped = baseInput({
                 validateWildcards: true,
                 wildcardValidation: {
diff --git a/ts/packages/typeagent-core/test/wildcardValidator.spec.ts b/ts/packages/typeagent-core/test/wildcardValidator.spec.ts
index 3f5fcbbbd8..c4cbea5941 100644
--- a/ts/packages/typeagent-core/test/wildcardValidator.spec.ts
+++ b/ts/packages/typeagent-core/test/wildcardValidator.spec.ts
@@ -5,7 +5,6 @@ import {
     createWildcardMatchValidator,
     createReplaySessionContextStub,
     matchActionPayloads,
-    DEFAULT_WILDCARD_VALIDATION_ALLOWLIST,
     type ReplayAppAgentLoader,
     type ReplayValidatableAgent,
 } from "../src/replay/wildcardValidator.js";
@@ -42,23 +41,6 @@ function action(actionName: string, parameters?: Record<string, unknown>) {
 }
 
 describe("createWildcardMatchValidator", () => {
-    it("accepts (no-op) and diagnoses when the agent is not in the allowlist", async () => {
-        const log = { loaded: [] as string[], unloaded: [] as string[] };
-        const validator = createWildcardMatchValidator("notallowed", {
-            loader: fakeLoader({ notallowed: {} }, log),
-            allowlist: ["timer"],
-        });
-        const outcome = await validator.validateMatch([action("doThing")]);
-        expect(outcome.rejected).toBe(false);
-        expect(outcome.diagnostic).toBe("agent-not-in-allowlist");
-        // Never even loaded the module.
-        expect(log.loaded).toEqual([]);
-        expect(validator.loaded).toBe(false);
-        expect([...validator.diagnostics]).toContain("agent-not-in-allowlist");
-        await validator.dispose();
-        expect(log.unloaded).toEqual([]);
-    });
-
     it("rejects when the agent validator returns false", async () => {
         const agent: ReplayValidatableAgent = {
             async validateWildcardMatch(a) {
@@ -67,7 +49,6 @@ describe("createWildcardMatchValidator", () => {
         };
         const validator = createWildcardMatchValidator("timer", {
             loader: fakeLoader({ timer: agent }),
-            allowlist: ["timer"],
         });
         expect((await validator.validateMatch([action("good")])).rejected).toBe(
             false,
@@ -85,7 +66,6 @@ describe("createWildcardMatchValidator", () => {
         };
         const validator = createWildcardMatchValidator("timer", {
             loader: fakeLoader({ timer: agent }),
-            allowlist: ["timer"],
         });
         const outcome = await validator.validateMatch([
             action("good"),
@@ -102,7 +82,6 @@ describe("createWildcardMatchValidator", () => {
         };
         const validator = createWildcardMatchValidator("timer", {
             loader: fakeLoader({ timer: agent }),
-            allowlist: ["timer"],
         });
         const outcome = await validator.validateMatch([action("x")]);
         expect(outcome.rejected).toBe(false);
@@ -113,7 +92,6 @@ describe("createWildcardMatchValidator", () => {
         const validator = createWildcardMatchValidator("timer", {
             // Loader has no "timer" → loadAppAgent throws.
             loader: fakeLoader({}),
-            allowlist: ["timer"],
         });
         const outcome = await validator.validateMatch([action("x")]);
         expect(outcome.rejected).toBe(false);
@@ -126,7 +104,6 @@ describe("createWildcardMatchValidator", () => {
     it("accepts (no-op) when the agent has no validateWildcardMatch", async () => {
         const validator = createWildcardMatchValidator("timer", {
             loader: fakeLoader({ timer: {} }),
-            allowlist: ["timer"],
         });
         const outcome = await validator.validateMatch([action("x")]);
         expect(outcome.rejected).toBe(false);
@@ -136,10 +113,7 @@ describe("createWildcardMatchValidator", () => {
     it("does not throw for an in-process validator reading agentContext (empty stub)", async () => {
         // A hypothetical in-process validator that reads context.agentContext
         // and returns true when its client is absent. Our stub uses
-        // `agentContext: {}` (NOT undefined), so this must not throw. (The real
-        // player validator has this shape but runs out-of-process, so it's
-        // excluded from the default allowlist; here we override the allowlist to
-        // exercise the stub's defensive behaviour directly.)
+        // `agentContext: {}` (NOT undefined), so this must not throw.
         const playerLike: ReplayValidatableAgent = {
             async validateWildcardMatch(_a, context) {
                 const spotify = (
@@ -153,7 +127,6 @@ describe("createWildcardMatchValidator", () => {
         };
         const validator = createWildcardMatchValidator("player", {
             loader: fakeLoader({ player: playerLike }),
-            allowlist: ["player"],
         });
         const outcome = await validator.validateMatch([action("playTrack")]);
         expect(outcome.rejected).toBe(false);
@@ -169,7 +142,6 @@ describe("createWildcardMatchValidator", () => {
         };
         const validator = createWildcardMatchValidator("timer", {
             loader: fakeLoader({ timer: agent }, log),
-            allowlist: ["timer"],
         });
         // Not loaded until first validateMatch.
         expect(log.loaded).toEqual([]);
@@ -183,20 +155,12 @@ describe("createWildcardMatchValidator", () => {
 
     it("dispose is a no-op when nothing was loaded", async () => {
         const log = { loaded: [] as string[], unloaded: [] as string[] };
-        const validator = createWildcardMatchValidator("notallowed", {
+        const validator = createWildcardMatchValidator("timer", {
             loader: fakeLoader({}, log),
-            allowlist: ["timer"],
         });
         await validator.dispose();
         expect(log.unloaded).toEqual([]);
     });
-
-    it("ships timer/list as the default allowlist", () => {
-        expect([...DEFAULT_WILDCARD_VALIDATION_ALLOWLIST].sort()).toEqual([
-            "list",
-            "timer",
-        ]);
-    });
 });
 
 describe("createReplaySessionContextStub", () => {
diff --git a/ts/packages/typeagent-studio/src/impactReportView.ts b/ts/packages/typeagent-studio/src/impactReportView.ts
index a4a9037889..14c5a1d703 100644
--- a/ts/packages/typeagent-studio/src/impactReportView.ts
+++ b/ts/packages/typeagent-studio/src/impactReportView.ts
@@ -371,9 +371,9 @@ export function openImpactReport(
                 // path to model (grammar-only vs construction-cache-first); the
                 // runtime defaults unknown/missing to the cache-free baseline.
                 mode: msg.mode,
-                // Opt-in wildcard validation (L4a): the runtime only acts on it
-                // when a `resolveWildcardValidator` is wired and the agent is in
-                // the validation allowlist; otherwise it is a no-op.
+                // Opt-in wildcard validation: the runtime only acts on it when a
+                // `resolveWildcardValidator` is wired and the agent's manifest
+                // declares it replay-safe; otherwise it is a no-op.
                 validateWildcards: msg.validateWildcards,
                 missPolicy: "needs-explanation",
             });
diff --git a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
index 2fdc39d947..5ea092c1b5 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
@@ -129,11 +129,11 @@ let versionB: ResolvedVersion = DEFAULT_VERSION_B;
 // version selection so a reload keeps the chosen mode.
 let mode: StudioReplayMode = "nfa-grammar";
 
-// Opt-in wildcard validation (replay fidelity rung L4a). When lit, the run asks
-// the host to additionally run the agent's real `validateWildcardMatch` over the
-// working-tree side's wildcard matches. Off by default; persisted with the rest
-// of the selection so a reload keeps the choice. The host/runtime no-op it when
-// no validator is wired (e.g. the packaged build) and report that back via
+// Opt-in wildcard validation. When lit, the run asks the host to additionally
+// run the agent's real `validateWildcardMatch` over the working-tree side's
+// wildcard matches. Off by default; persisted with the rest of the selection so
+// a reload keeps the choice. The host/runtime no-op it when no validator is
+// wired (e.g. the packaged build) and report that back via
 // `wildcardValidation.diagnostics`, which `renderValidationNote` surfaces.
 let validateWildcards = false;
 
@@ -144,7 +144,7 @@ const CACHE_TOOLTIP: Record<"on" | "off", string> = {
 };
 /** Concise hover copy for the Wildcard validation on/off toggle. */
 const VALIDATE_TOOLTIP: Record<"on" | "off", string> = {
-    on: "Wildcard validation on: the working-tree side runs the agent's real validator over wildcard matches (allowlisted agents only). Click to turn off.",
+    on: "Wildcard validation on: the working-tree side runs the agent's real validator over wildcard matches (agents that opt in only). Click to turn off.",
     off: "Wildcard validation off: matches come from the grammar alone. Click to turn on.",
 };
 
@@ -479,7 +479,7 @@ function renderResult(
         renderProvenance(provenance);
     }
 
-    // Surface the opt-in wildcard-validation outcome (L4a), including an honest
+    // Surface the opt-in wildcard-validation outcome, including an honest
     // "unavailable" state when the validator couldn't load (e.g. packaged build).
     renderValidationNote(result.wildcardValidation);
 
@@ -502,7 +502,7 @@ function renderResult(
     );
 }
 
-/** Paint the wildcard-validation outcome (L4a) into the sub-bar, or clear it.
+/** Paint the wildcard-validation outcome into the sub-bar, or clear it.
  *  Shown only when validation actually ran on a wildcard match; a clean pass is
  *  neutral, a degraded/unavailable pass warns so the report stays honest. */
 function renderValidationNote(
@@ -551,16 +551,6 @@ function describeValidation(diagnostics: WildcardValidationDiagnostic[]): {
             degraded: true,
         };
     }
-    if (diagnostics.includes("agent-not-in-allowlist")) {
-        return {
-            icon: "info",
-            label: "validation skipped",
-            detail:
-                "This agent isn't in the validation allowlist, so its validator " +
-                "wasn't run and matches came from the grammar alone.",
-            degraded: true,
-        };
-    }
     if (diagnostics.includes("no-validator")) {
         return {
             icon: "info",
diff --git a/ts/packages/typeagent-studio/src/webviewKit/protocol.ts b/ts/packages/typeagent-studio/src/webviewKit/protocol.ts
index 6a5f2f7de6..a31103b7eb 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/protocol.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/protocol.ts
@@ -78,8 +78,8 @@ export type WebviewToHostMessage =
           mode: StudioReplayMode;
           /**
            * Opt-in: additionally run the agent's real `validateWildcardMatch`
-           * over the working-tree side's wildcard matches (replay fidelity rung
-           * L4a). Off unless the user lit the validation toggle.
+           * over the working-tree side's wildcard matches. Off unless the user
+           * lit the validation toggle.
            */
           validateWildcards: boolean;
       }

From 0aec3c37dfb144f634c83f2198f12671f01b8548 Mon Sep 17 00:00:00 2001
From: Tal Zaccai <talzacc@microsoft.com>
Date: Tue, 30 Jun 2026 13:55:25 -0600
Subject: [PATCH 12/15] Replace replay-safe manifest flag with a
 wildcard-validator capability probe

The Impact Report's wildcard-validation toggle previously keyed off a
self-certified manifest flag (replaySafeWildcardValidator), which was an
unclear tri-state config for agent authors. Replace it with a runtime
capability probe: the host loads the agent and checks whether it exposes a
validateWildcardMatch method (canValidateWildcards).

- Drop the replaySafeWildcardValidator field from AppAgentManifest and the
  two manifests that set it (list, timer).
- Rename the RPC and runtime plumbing isReplaySafe -> canValidateWildcards
  end to end (core runtime/protocol, studio-service host/resolver/handler,
  client, stub, spec). The default validator resolver no longer gates on the
  manifest.
- Webview: disable the toggle (with a neutral 'no validator to run' note)
  when the agent has no validator; otherwise enable it, default off, and show
  a caution warning only when it is turned on (the real validator may have
  side effects or be non-deterministic, so replay results may not reproduce).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 ts/packages/agentSdk/src/agentInterface.ts    |  5 --
 ts/packages/agents/list/src/listManifest.json |  1 -
 .../agents/timer/src/timerManifest.json       |  1 -
 ts/packages/studio-service/src/runtime.ts     |  6 +-
 .../studio-service/src/studioRpcHandlers.ts   |  3 +
 .../studio-service/src/wildcardValidation.ts  | 70 +++++++++--------
 .../studio-service/test/studioService.spec.ts | 12 +++
 .../src/runtime/studioProtocol.ts             | 10 +++
 .../src/runtime/studioRuntimeCore.ts          | 19 +++++
 .../typeagent-studio/media/impactReport.css   | 31 ++++++++
 .../typeagent-studio/src/impactReportView.ts  | 25 +++++-
 .../src/studioServiceClient.ts                |  5 ++
 .../src/test/stubInvokeHandlers.ts            |  1 +
 .../src/webviewKit/client/impactReport.ts     | 78 +++++++++++++++++--
 .../src/webviewKit/protocol.ts                |  5 ++
 15 files changed, 222 insertions(+), 50 deletions(-)

diff --git a/ts/packages/agentSdk/src/agentInterface.ts b/ts/packages/agentSdk/src/agentInterface.ts
index abc86ecbd7..24fc1ff258 100644
--- a/ts/packages/agentSdk/src/agentInterface.ts
+++ b/ts/packages/agentSdk/src/agentInterface.ts
@@ -43,11 +43,6 @@ export type AppAgentManifest = {
     // Registered flow programs: actionName → path to .flow.json (relative to manifest file)
     flows?: Record<string, string>;
     allowDynamicAgents?: boolean; // whether this agent can add/remove dynamic sub-agents at runtime, default is false
-    // Whether this agent's validateWildcardMatch is safe to run during offline
-    // replay: deterministic, free of side effects, and tolerant of a stub
-    // SessionContext (no live agentContext, storage, or network). Default false.
-    // Studio replay only runs the real validator for agents that opt in here.
-    replaySafeWildcardValidator?: boolean;
 } & ActionManifest;
 
 export type SchemaTypeNames = {
diff --git a/ts/packages/agents/list/src/listManifest.json b/ts/packages/agents/list/src/listManifest.json
index 23162190ab..0b7495e0c4 100644
--- a/ts/packages/agents/list/src/listManifest.json
+++ b/ts/packages/agents/list/src/listManifest.json
@@ -1,7 +1,6 @@
 {
   "emojiChar": "📝",
   "description": "Agent to create and manage lists",
-  "replaySafeWildcardValidator": true,
   "schema": {
     "description": "List agent with actions to create lists, show list items, add and remove list items",
     "originalSchemaFile": "./listSchema.ts",
diff --git a/ts/packages/agents/timer/src/timerManifest.json b/ts/packages/agents/timer/src/timerManifest.json
index d89d94aff7..98f0e2f907 100644
--- a/ts/packages/agents/timer/src/timerManifest.json
+++ b/ts/packages/agents/timer/src/timerManifest.json
@@ -1,7 +1,6 @@
 {
   "emojiChar": "⏰",
   "description": "Agent for setting reminders that fire as agent-initiated messages",
-  "replaySafeWildcardValidator": true,
   "schema": {
     "description": "Timer agent: set, list, and cancel reminders. When a reminder fires, the agent pushes a message to the chat via SessionContext.beginAgentThread (no preceding user request).",
     "originalSchemaFile": "./timerSchema.ts",
diff --git a/ts/packages/studio-service/src/runtime.ts b/ts/packages/studio-service/src/runtime.ts
index 4ae42835ef..d8510b54b7 100644
--- a/ts/packages/studio-service/src/runtime.ts
+++ b/ts/packages/studio-service/src/runtime.ts
@@ -14,7 +14,10 @@ import {
     FileWorkspaceState,
     studioWorkspaceStateFile,
 } from "./fileWorkspaceState.js";
-import { createDefaultWildcardValidatorResolver } from "./wildcardValidation.js";
+import {
+    createDefaultWildcardValidatorResolver,
+    canValidateWildcards,
+} from "./wildcardValidation.js";
 
 /**
  * Candidate repository roots for Studio to inspect, most-specific first. The
@@ -140,6 +143,7 @@ export function getStudioRuntime(repoRoot?: string): StudioRuntime {
             {
                 resolveWildcardValidator:
                     createDefaultWildcardValidatorResolver(),
+                resolveCanValidateWildcards: canValidateWildcards,
             },
         );
         runtimeCache.set(key, runtime);
diff --git a/ts/packages/studio-service/src/studioRpcHandlers.ts b/ts/packages/studio-service/src/studioRpcHandlers.ts
index cc8a80b913..147b66177d 100644
--- a/ts/packages/studio-service/src/studioRpcHandlers.ts
+++ b/ts/packages/studio-service/src/studioRpcHandlers.ts
@@ -68,6 +68,9 @@ export function createStudioInvokeHandlers(
         async listCorpusAgents(repoRoot) {
             return conn.getRuntime(repoRoot).listCorpusAgents();
         },
+        async canValidateWildcards(repoRoot, agent) {
+            return conn.getRuntime(repoRoot).canValidateWildcards(agent);
+        },
         async listCorpusEntries(repoRoot, agent) {
             return conn.getRuntime(repoRoot).listCorpusEntries(agent);
         },
diff --git a/ts/packages/studio-service/src/wildcardValidation.ts b/ts/packages/studio-service/src/wildcardValidation.ts
index 1b67692cbb..e0eb891ad0 100644
--- a/ts/packages/studio-service/src/wildcardValidation.ts
+++ b/ts/packages/studio-service/src/wildcardValidation.ts
@@ -32,15 +32,9 @@ import {
     type WildcardMatchValidator,
 } from "@typeagent/core/runtime";
 
-/** The single manifest field that gates replay wildcard validation. */
-interface ReplaySafeManifestLike {
-    replaySafeWildcardValidator?: boolean;
-}
-
 /** Minimal structural view of an `AppAgentProvider` (avoids a type-only dep). */
 interface AppAgentProviderLike {
     getAppAgentNames(): string[];
-    getAppAgentManifest(agentName: string): Promise<ReplaySafeManifestLike>;
     loadAppAgent(agentName: string): Promise<ReplayValidatableAgent>;
     unloadAppAgent(agentName: string): Promise<void>;
 }
@@ -124,43 +118,55 @@ function createDefaultLoader(): ReplayAppAgentLoader {
 }
 
 /**
- * Whether the agent's manifest opts its `validateWildcardMatch` into replay. Any
- * lookup failure (provider unavailable, agent unknown, manifest read error) is
- * treated as not safe, so replay stays grammar-only.
+ * Whether wildcard validation can actually run for an agent: the agent loads and
+ * exposes a `validateWildcardMatch`. Drives the Impact Report's validation
+ * toggle — when false (no validator, or the provider is unavailable in the
+ * packaged build) there is nothing to run, so the toggle is disabled. The agent
+ * is loaded only to inspect for the method, then unloaded; any failure reports
+ * not-validatable so the toggle stays off.
  */
-async function isReplaySafe(agentName: string): Promise<boolean> {
+export async function canValidateWildcards(
+    agentName: string,
+): Promise<boolean> {
+    const providers = await loadProviders();
+    if (providers === undefined) {
+        return false;
+    }
+    const provider = findProvider(providers, agentName);
+    if (provider === undefined) {
+        return false;
+    }
+    let loaded = false;
     try {
-        const providers = await loadProviders();
-        if (providers === undefined) {
-            return false;
-        }
-        const provider = findProvider(providers, agentName);
-        if (provider === undefined) {
-            return false;
-        }
-        const manifest = await provider.getAppAgentManifest(agentName);
-        return manifest.replaySafeWildcardValidator === true;
+        const agent = await provider.loadAppAgent(agentName);
+        loaded = true;
+        return typeof agent.validateWildcardMatch === "function";
     } catch {
         return false;
+    } finally {
+        if (loaded) {
+            try {
+                await provider.unloadAppAgent(agentName);
+            } catch {
+                // Best-effort cleanup; the run-time validator reloads as needed.
+            }
+        }
     }
 }
 
 /**
- * Build the runtime's `resolveWildcardValidator` option: returns a validator
- * only for agents whose manifest declares `replaySafeWildcardValidator` (so any
- * other agent stays grammar-only and the run doesn't claim a validation it can't
- * perform), backed by the lazy default loader. The dynamic import is deferred
- * until a wildcard match actually triggers a lookup, so a run that never hits a
- * wildcard never pays for the provider.
+ * Build the runtime's `resolveWildcardValidator` option: returns a validator for
+ * any agent, backed by the lazy default loader. The validator fail-opens
+ * (load-failed / no-validator / errored diagnostics) so a run never fabricates a
+ * lost match; whether validation is attempted at all is the operator's per-run
+ * toggle, which the UI disables for agents that have no validator to run. The
+ * dynamic import is deferred until a wildcard match actually triggers a lookup,
+ * so a run that never hits a wildcard never pays for the provider.
  */
 export function createDefaultWildcardValidatorResolver(): (
     agentName: string,
 ) => Promise<WildcardMatchValidator | undefined> {
     const loader = createDefaultLoader();
-    return async (agentName) => {
-        if (!(await isReplaySafe(agentName))) {
-            return undefined;
-        }
-        return createWildcardMatchValidator(agentName, { loader });
-    };
+    return async (agentName) =>
+        createWildcardMatchValidator(agentName, { loader });
 }
diff --git a/ts/packages/studio-service/test/studioService.spec.ts b/ts/packages/studio-service/test/studioService.spec.ts
index fbfc9930dd..6759d9f59f 100644
--- a/ts/packages/studio-service/test/studioService.spec.ts
+++ b/ts/packages/studio-service/test/studioService.spec.ts
@@ -97,6 +97,7 @@ function createRuntimeStub(): {
         },
         queryRecentEvents: async () => [],
         listCorpusAgents: async () => ["player", "calendar"],
+        canValidateWildcards: async (agent: string) => agent === "player",
         listCorpusEntries: async (agent: string) => [
             { utterance: `hello ${agent}`, source: "in-repo" },
         ],
@@ -266,6 +267,17 @@ describe("studio service channel (in-memory rpc)", () => {
         expect(result.rows[0].utterance).toBe("play jazz");
     });
 
+    it("canValidateWildcards round-trips the per-agent capability", async () => {
+        const stub = createRuntimeStub();
+        const { client } = wireClientServer(stub);
+        expect(
+            await client.invoke("canValidateWildcards", undefined, "player"),
+        ).toBe(true);
+        expect(
+            await client.invoke("canValidateWildcards", undefined, "calendar"),
+        ).toBe(false);
+    });
+
     it("corpus/feedback methods round-trip", async () => {
         const stub = createRuntimeStub();
         const { client } = wireClientServer(stub);
diff --git a/ts/packages/typeagent-core/src/runtime/studioProtocol.ts b/ts/packages/typeagent-core/src/runtime/studioProtocol.ts
index f92efbb7c1..e5285ccbeb 100644
--- a/ts/packages/typeagent-core/src/runtime/studioProtocol.ts
+++ b/ts/packages/typeagent-core/src/runtime/studioProtocol.ts
@@ -72,6 +72,16 @@ export type StudioServiceInvokeFunctions = {
     ): Promise<StudioEvent[]>;
     /** Corpus agents available for replay in this workspace. */
     listCorpusAgents(repoRoot?: string): Promise<string[]>;
+    /**
+     * Whether wildcard validation can run for `agent` in replay — the agent
+     * loads and exposes a `validateWildcardMatch`. The Impact Report reads this
+     * before a run to disable its validation toggle when there is nothing to
+     * run.
+     */
+    canValidateWildcards(
+        repoRoot: string | undefined,
+        agent: string,
+    ): Promise<boolean>;
     /**
      * Federated corpus entries for an agent (in-repo, captures, external,
      * feedback) — what the Corpus tree expands.
diff --git a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
index b54354e624..c09fd2ccb8 100644
--- a/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
+++ b/ts/packages/typeagent-core/src/runtime/studioRuntimeCore.ts
@@ -721,6 +721,12 @@ export interface StudioRuntime {
      * union of agents loaded across running sandboxes.
      */
     listCorpusAgents(): Promise<string[]>;
+    /**
+     * Whether wildcard validation can actually run for `agent` in replay — the
+     * agent loads and exposes a `validateWildcardMatch`. Lets the Impact Report
+     * disable its validation toggle when there is nothing to run.
+     */
+    canValidateWildcards(agent: string): Promise<boolean>;
     /** Federated corpus entries for an agent (in-repo, captures, external, feedback). */
     listCorpusEntries(agent: string): Promise<CorpusEntry[]>;
     /**
@@ -846,6 +852,14 @@ export interface CreateStudioRuntimeOptions {
         | WildcardMatchValidator
         | undefined
         | Promise<WildcardMatchValidator | undefined>;
+    /**
+     * Reports whether wildcard validation can run for an agent — it loads and
+     * exposes a `validateWildcardMatch` — so `canValidateWildcards` can answer
+     * the Impact Report before a run. Injected for the same reason as
+     * `resolveWildcardValidator`: loading agent modules lives outside this
+     * dependency-light package. Absent ⇒ no agent is treated as validatable.
+     */
+    resolveCanValidateWildcards?: (agentName: string) => Promise<boolean>;
     collisions?: CollisionService;
     /**
      * Scans agents' compiled grammars for collisions. Injected so tests can
@@ -1278,6 +1292,11 @@ export function createStudioRuntimeCore(
             }
             return [...agents].sort((a, b) => a.localeCompare(b));
         },
+        async canValidateWildcards(agent) {
+            return (
+                (await options.resolveCanValidateWildcards?.(agent)) === true
+            );
+        },
         async listCorpusEntries(agent) {
             return corpus.list(agent);
         },
diff --git a/ts/packages/typeagent-studio/media/impactReport.css b/ts/packages/typeagent-studio/media/impactReport.css
index 1cc1c91056..e4eee84dad 100644
--- a/ts/packages/typeagent-studio/media/impactReport.css
+++ b/ts/packages/typeagent-studio/media/impactReport.css
@@ -437,6 +437,37 @@ body {
   opacity: 0;
 }
 
+/* A switch and its explanatory caption grouped as one popover item. */
+.switch-group {
+  display: flex;
+  flex-direction: column;
+  gap: 6px;
+}
+
+/* Caption under the wildcard-validation switch: states whether the scoped agent
+   is replay-safe so the toggle's effect is clear before a run. */
+.switch-note {
+  display: flex;
+  align-items: flex-start;
+  gap: 6px;
+  max-width: 240px;
+  font-size: 11px;
+  line-height: 1.4;
+  color: var(--vscode-descriptionForeground);
+}
+
+.switch-note.is-caution {
+  color: var(
+    --vscode-notificationsWarningIcon-foreground,
+    var(--vscode-editorWarning-foreground)
+  );
+}
+
+.switch-note .codicon {
+  flex: 0 0 auto;
+  margin-top: 1px;
+}
+
 /* --- Sub-bar: provenance + run status ------------------------------------ */
 .sub-bar {
   display: flex;
diff --git a/ts/packages/typeagent-studio/src/impactReportView.ts b/ts/packages/typeagent-studio/src/impactReportView.ts
index 14c5a1d703..026902cdf9 100644
--- a/ts/packages/typeagent-studio/src/impactReportView.ts
+++ b/ts/packages/typeagent-studio/src/impactReportView.ts
@@ -133,7 +133,13 @@ export function openImpactReport(
         post({ type: "status", text: "Connecting to the studio service…" });
         const c = await ensureClient();
         if (!c) {
-            post({ type: "init", agent, connected: false, available: false });
+            post({
+                type: "init",
+                agent,
+                connected: false,
+                available: false,
+                canValidateWildcards: false,
+            });
             return;
         }
         // Confirm this agent still has a corpus to replay so the webview can
@@ -145,7 +151,22 @@ export function openImpactReport(
         } catch {
             // Connected but listing failed; treat as unavailable for now.
         }
-        post({ type: "init", agent, connected: true, available });
+        // Check whether wildcard validation can actually run for this agent (it
+        // has a validator), so the webview can disable the toggle when there is
+        // nothing to run. A lookup failure stays conservative (toggle disabled).
+        let canValidateWildcards = false;
+        try {
+            canValidateWildcards = await c.canValidateWildcards(agent);
+        } catch {
+            // Leave the toggle disabled rather than offering a no-op.
+        }
+        post({
+            type: "init",
+            agent,
+            connected: true,
+            available,
+            canValidateWildcards,
+        });
         // Recover a result computed while the panel was hidden/reloaded.
         if (lastResult) {
             post({
diff --git a/ts/packages/typeagent-studio/src/studioServiceClient.ts b/ts/packages/typeagent-studio/src/studioServiceClient.ts
index e8597d964d..4f1451ede2 100644
--- a/ts/packages/typeagent-studio/src/studioServiceClient.ts
+++ b/ts/packages/typeagent-studio/src/studioServiceClient.ts
@@ -230,6 +230,11 @@ export class StudioServiceClient {
         return this.rpc.invoke("listCorpusAgents", this.repoRoot);
     }
 
+    /** Whether wildcard validation can run for `agent` (it has a validator). */
+    canValidateWildcards(agent: string): Promise<boolean> {
+        return this.rpc.invoke("canValidateWildcards", this.repoRoot, agent);
+    }
+
     /** Federated corpus entries for an agent (Corpus tree expansion). */
     listCorpusEntries(agent: string): Promise<CorpusEntry[]> {
         return this.rpc.invoke("listCorpusEntries", this.repoRoot, agent);
diff --git a/ts/packages/typeagent-studio/src/test/stubInvokeHandlers.ts b/ts/packages/typeagent-studio/src/test/stubInvokeHandlers.ts
index dd9af1a35a..4ed754bd8b 100644
--- a/ts/packages/typeagent-studio/src/test/stubInvokeHandlers.ts
+++ b/ts/packages/typeagent-studio/src/test/stubInvokeHandlers.ts
@@ -32,6 +32,7 @@ export function stubInvokeHandlers(
         clearCollisions: async () => 0,
         queryRecentEvents: async () => [],
         listCorpusAgents: async () => [],
+        canValidateWildcards: async () => false,
         listCorpusEntries: async () => [],
         seedInRepoCorpus: async () => ({
             path: "/repo/ts/corpus",
diff --git a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
index 5ea092c1b5..3ea4bb1b42 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/client/impactReport.ts
@@ -131,11 +131,16 @@ let mode: StudioReplayMode = "nfa-grammar";
 
 // Opt-in wildcard validation. When lit, the run asks the host to additionally
 // run the agent's real `validateWildcardMatch` over the working-tree side's
-// wildcard matches. Off by default; persisted with the rest of the selection so
-// a reload keeps the choice. The host/runtime no-op it when no validator is
-// wired (e.g. the packaged build) and report that back via
-// `wildcardValidation.diagnostics`, which `renderValidationNote` surfaces.
+// wildcard matches. Off by default and only available when the agent actually
+// has a validator to run (`agentCanValidate`); persisted with the rest of the
+// selection so a reload keeps the choice. The host/runtime fail-open it when no
+// validator runs and report that back via `wildcardValidation.diagnostics`,
+// which `renderValidationNote` surfaces.
 let validateWildcards = false;
+// Whether wildcard validation can run for the scoped agent (it has a validator),
+// from the host `init`. Enables the toggle; when false the toggle is disabled
+// because there is nothing to run.
+let agentCanValidate = false;
 
 /** Concise hover copy for the Cache on/off toggle. */
 const CACHE_TOOLTIP: Record<"on" | "off", string> = {
@@ -144,9 +149,13 @@ const CACHE_TOOLTIP: Record<"on" | "off", string> = {
 };
 /** Concise hover copy for the Wildcard validation on/off toggle. */
 const VALIDATE_TOOLTIP: Record<"on" | "off", string> = {
-    on: "Wildcard validation on: the working-tree side runs the agent's real validator over wildcard matches (agents that opt in only). Click to turn off.",
+    on: "Wildcard validation on: the working-tree side runs the agent's real validator over wildcard matches. Click to turn off.",
     off: "Wildcard validation off: matches come from the grammar alone. Click to turn on.",
 };
+/** Hover copy when the toggle is disabled because the agent has no wildcard
+ *  validator to run in replay. */
+const VALIDATE_DISABLED_TOOLTIP =
+    "Wildcard validation unavailable: this agent has no validateWildcardMatch to run in replay.";
 
 const root = document.getElementById("root")!;
 
@@ -192,13 +201,22 @@ const validateSwitch = settingsSwitch(
     "Wildcard validation",
     () => validateWildcards,
     () => toggleValidateWildcards(),
-    (on) => VALIDATE_TOOLTIP[on ? "on" : "off"],
+    (on) =>
+        agentCanValidate
+            ? VALIDATE_TOOLTIP[on ? "on" : "off"]
+            : VALIDATE_DISABLED_TOOLTIP,
 );
 const settingsPopover = el("div", "settings-popover");
 settingsPopover.setAttribute("role", "group");
 settingsPopover.setAttribute("aria-label", "Replay options");
 settingsPopover.hidden = true;
-settingsPopover.append(cacheSwitch.row, validateSwitch.row);
+// The validation switch carries a caption: a note when there's no validator to
+// run, and a caution when validation is turned on (the real validator may have
+// side effects or be non-deterministic, so replay results may not reproduce).
+const validateNote = el("div", "switch-note");
+const validateGroup = el("div", "switch-group");
+validateGroup.append(validateSwitch.row, validateNote);
+settingsPopover.append(cacheSwitch.row, validateGroup);
 const settingsButton = toolButton("settings-gear", "Replay options", () =>
     toggleSettingsPopover(),
 );
@@ -266,6 +284,7 @@ restoreSelection();
 renderVersionButtons();
 cacheSwitch.render();
 validateSwitch.render();
+renderValidateNote();
 
 // Dismiss the replay-options popover on an outside click or Escape, the way a
 // native menu behaves. The gear's own click is inside `settingsGroup`, so it
@@ -292,6 +311,15 @@ window.addEventListener("message", (event: MessageEvent) => {
         case "init":
             currentAgent = msg.agent;
             renderAgentName();
+            // Wildcard validation can only run when the agent has a validator;
+            // otherwise force the request off and disable the toggle. When a
+            // validator exists, keep the persisted choice (default off).
+            agentCanValidate = msg.canValidateWildcards;
+            if (!agentCanValidate) {
+                validateWildcards = false;
+            }
+            validateSwitch.render();
+            renderValidateNote();
             controlsAvailable = msg.connected && msg.available;
             setControlsEnabled(controlsAvailable);
             renderEmptyState();
@@ -396,9 +424,41 @@ function toggleCache(): void {
 function toggleValidateWildcards(): void {
     validateWildcards = !validateWildcards;
     validateSwitch.render();
+    renderValidateNote();
     persistState({ validateWildcards });
 }
 
+/** Caption the validation switch for the scoped agent. When there's no validator
+ *  to run, show a neutral note explaining why the toggle is disabled. When
+ *  validation is turned on, caution that the real validator may have side effects
+ *  or be non-deterministic, so replay results may not reproduce. Otherwise hide
+ *  the caption. */
+function renderValidateNote(): void {
+    validateNote.textContent = "";
+    if (!agentCanValidate) {
+        validateNote.hidden = false;
+        validateNote.classList.remove("is-caution");
+        validateNote.append(
+            codicon("info"),
+            text("No wildcard validator to run for this agent."),
+        );
+        return;
+    }
+    if (!validateWildcards) {
+        validateNote.hidden = true;
+        validateNote.classList.remove("is-caution");
+        return;
+    }
+    validateNote.hidden = false;
+    validateNote.classList.add("is-caution");
+    validateNote.append(
+        codicon("warning"),
+        text(
+            "May be unsafe: the validator can have side effects or be non-deterministic, so replay results may not reproduce.",
+        ),
+    );
+}
+
 /** Open/close the replay-options popover. Pass `force` to set a specific
  *  state; omit to flip the current one. */
 function toggleSettingsPopover(force?: boolean): void {
@@ -1044,7 +1104,9 @@ function setControlsEnabled(enabled: boolean): void {
     versionBButton.button.disabled = !enabled;
     settingsButton.disabled = !enabled;
     cacheSwitch.button.disabled = !enabled;
-    validateSwitch.button.disabled = !enabled;
+    // The validation toggle additionally requires the agent to have a validator
+    // to run; otherwise it stays disabled so the operator isn't offered a no-op.
+    validateSwitch.button.disabled = !enabled || !agentCanValidate;
     // A run shouldn't leave the options popover hanging open.
     if (!enabled) {
         toggleSettingsPopover(false);
diff --git a/ts/packages/typeagent-studio/src/webviewKit/protocol.ts b/ts/packages/typeagent-studio/src/webviewKit/protocol.ts
index a31103b7eb..2968ec03de 100644
--- a/ts/packages/typeagent-studio/src/webviewKit/protocol.ts
+++ b/ts/packages/typeagent-studio/src/webviewKit/protocol.ts
@@ -42,6 +42,11 @@ export type HostToWebviewMessage =
           /** Whether `agent` has a corpus available to replay (channel
            *  `listCorpusAgents`). */
           available: boolean;
+          /** Whether wildcard validation can run for `agent` — it exposes a
+           *  `validateWildcardMatch` and its module loads. Drives the validation
+           *  toggle: enabled when there is a validator to run, disabled (a no-op)
+           *  otherwise. */
+          canValidateWildcards: boolean;
       }
     /** A connection/loading status line for the webview to surface. */
     | { type: "status"; text: string }

From 2959d2f369b69dbb13dfe3737110c6126e73de92 Mon Sep 17 00:00:00 2001
From: Tal Zaccai <talzacc@microsoft.com>
Date: Tue, 30 Jun 2026 14:34:47 -0600
Subject: [PATCH 13/15] docs(studio): consolidate roadmap into STATUS.md gate
 spine + backlog

Add a single 'Roadmap at a glance' section to STATUS.md (gate spine A-E plus a tagged off-critical-path backlog) so depth (L4b) and breadth (multi-variant/multi-agent) are backlog rows under the gates rather than a parallel plan. Promote the L4b sandbox-convergence and multi-variant compare design docs into the plan folder and index them in README. Refresh the stale top-of-file branch note.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 ts/docs/plans/vscode-devx/README.md           |   2 +
 ts/docs/plans/vscode-devx/STATUS.md           |  51 +++-
 .../impact-report-multi-variant-design.md     | 256 ++++++++++++++++++
 .../plans/vscode-devx/replay-l4b-design.md    | 149 ++++++++++
 4 files changed, 456 insertions(+), 2 deletions(-)
 create mode 100644 ts/docs/plans/vscode-devx/impact-report-multi-variant-design.md
 create mode 100644 ts/docs/plans/vscode-devx/replay-l4b-design.md

diff --git a/ts/docs/plans/vscode-devx/README.md b/ts/docs/plans/vscode-devx/README.md
index 07f218bbe1..13fa523663 100644
--- a/ts/docs/plans/vscode-devx/README.md
+++ b/ts/docs/plans/vscode-devx/README.md
@@ -26,6 +26,8 @@
 | [USER-STORY.md](./USER-STORY.md)     | The authoring loop and the three interaction modes (human / AI-agent / hybrid).                              |
 | [STUDIO-AGENT.md](./STUDIO-AGENT.md) | The `studio` agent **action-surface reference** (groups A–F, tiers, approval boundary). Phasing lives in 05. |
 | [STATUS.md](./STATUS.md)             | What's built, known issues, and the ready-to-start next slices (pointing into the 05 §11 phasing).           |
+| [replay-l4b-design.md](./replay-l4b-design.md) | The replay **fidelity depth ladder** — sandbox-convergence framing and the build-from-git-ref (L4b) first-slice plan (Steps 1–3). |
+| [impact-report-multi-variant-design.md](./impact-report-multi-variant-design.md) | The Impact Report **breadth axis** — Baseline + N Variants compare matrix and regression bisect. |
 | [QUICKSTART.md](./QUICKSTART.md)     | Build/test/run commands and where each piece lives.                                                          |
 
 ---
diff --git a/ts/docs/plans/vscode-devx/STATUS.md b/ts/docs/plans/vscode-devx/STATUS.md
index f44924fc0f..1e3dcd9e13 100644
--- a/ts/docs/plans/vscode-devx/STATUS.md
+++ b/ts/docs/plans/vscode-devx/STATUS.md
@@ -4,8 +4,9 @@
 > the **intended** end state), this file tracks what is **actually built today**
 > and what remains. Update it whenever a capability changes state.
 
-Branch: feature work has merged to `main` via PR #2468; ongoing work continues
-on `dev/talzacc/typeagent_studio_part3` (the real replay path).
+Branch: earlier feature work merged to `main` via PR #2468; ongoing work
+continues on the `dev/talzacc/typeagent_studio_part*` stack (currently part5 —
+replay fidelity + Impact Report UX).
 
 ## Capability matrix
 
@@ -32,6 +33,52 @@ than an in-memory stand-in.
 | Live Trace                                                     | ❌                                                                                                                                    | ❌                                                                               | ❌                                                                                                                            | ❌     |
 | `agr-language` / `vscode-shell` refactor onto core             | 🟡 dependency edge only                                                                                                               | —                                                                                | ❌ no behavioral integration                                                                                                  | ❌     |
 
+## Roadmap at a glance — gate spine + backlog
+
+This is the single at-a-glance tracker. The MVP is fenced by five acceptance
+gates ([`04-mvp-slice.md` §3](./04-mvp-slice.md)); phase sequencing lives in
+[`05-implementation-plan.md` §11](./05-implementation-plan.md#11-phasing--concrete-sequencing).
+The **gate spine** is the critical path; the **backlog** holds everything
+deliberately off it, each row tagged with its parent gate and precondition so it
+is clear why it waits. There is no second roadmap — depth (L4b) and breadth
+(multi-variant / multi-agent) are backlog rows here, not a parallel plan.
+
+### Gate spine (MVP critical path)
+
+| Gate | Journey | Capability | Phase | Status |
+| ---- | ------- | ---------- | ----- | ------ |
+| **A** | J1 Stand up an agent | New-Agent Wizard | P-2 | ❌ not started (onboarding bridge ✅) |
+| **B** | J2 Tune schema | Schema Studio | P-4 | ❌ not started |
+| **C** | J4 Find a regression | Impact Report ≥ 80% red/green on hand-labelled `player` | P-3 | 🟡 **long pole** — engine L1–L4a ✅; corpus capture + predicate + measurement ❌ |
+| **D** | J5 Debug a trace | Trace Viewer | P-3 | ❌ not started |
+| **E** | J6 Observe live | Live Trace + status bar | P-5 | ❌ not started |
+
+**Critical path now: close Gate C** — player corpus capture → predicate tuning →
+run the ≥ 80% validation. It is the only *tunable* gate and §7's top risk.
+
+### Backlog (off the critical path — tagged, not a separate plan)
+
+Each item is a child of a gate/journey; **none is required to pass A–E**. Track
+tags: **depth** = how faithfully one replay side is realized (improves Gate C
+accuracy); **breadth** = how many things are compared at once (multiplies cells);
+**infra** = enabling plumbing.
+
+| Item | Parent | Track | Precondition | Status |
+| ---- | ------ | ----- | ------------ | ------ |
+| Fidelity transparency (per-side readout + "Sandbox A/B" relabel) | Gate C / J4 | depth | — | next (low-risk, no build) |
+| **L4b** build-from-ref sandboxes (real compiled ref side) | Gate C / J4 | depth (P-7) | Gate C banked | deferred flagged epic |
+| Multi-variant compare (Baseline vs N variants; bisect / first-divergence) | J4 | breadth — versions | Gate C banked; must preserve Gate C | post-MVP |
+| Multi-agent / multi-corpus replay (`code`, 684 utterances) | J4 | breadth — agents | Gate C banked | post-MVP (§2 exclude) |
+| Active-sandbox selector + per-sandbox scoping | — | infra | single-sandbox E2E | P-7a |
+| Sandbox copy-on-write overlay (true sandbox-local A/B) | — | infra | P-7a | P-7b |
+
+Why depth and breadth are distinct dials on the same `replayCorpus` (F4.1)
+primitive: **depth** raises the fidelity of each compared *cell* (and thus Gate C
+accuracy); the two **breadth** axes raise the *number* of cells (versions ×
+agents). They compose and are independent. Detailed slice breakdowns live in the
+design docs ([the L4b sandbox-convergence plan](./replay-l4b-design.md) and
+[the multi-variant compare design](./impact-report-multi-variant-design.md)).
+
 ## The long pole
 
 The headline "find a regression" journey (capture → replay two versions →
diff --git a/ts/docs/plans/vscode-devx/impact-report-multi-variant-design.md b/ts/docs/plans/vscode-devx/impact-report-multi-variant-design.md
new file mode 100644
index 0000000000..dec4a6ac1f
--- /dev/null
+++ b/ts/docs/plans/vscode-devx/impact-report-multi-variant-design.md
@@ -0,0 +1,256 @@
+# Impact Report — multi-variant compare (Baseline + N Variants) — design
+
+**Builds on:** the existing binary A/B Impact Report (replay engine, `replayCorpus`,
+`ActionDelta`, `ReplaySummary`) and the sandbox-convergence framing in
+[`replay-l4b-design.md`](./replay-l4b-design.md) (A/B reframed as "Sandbox A / Sandbox B").
+
+## One-line idea
+Replace the fixed two-target compare (A vs B) with **one Baseline compared against
+an ordered list of Variants** (A vs B1, B2, … Bn), rendered as a matrix. Binary
+A/B becomes the N=1 special case.
+
+## Why — the journeys this unlocks
+- **Bisect a regression.** "Which of these N commits changed the answer?" Baseline =
+  known-good ref; variants = suspect commits. One run instead of N binary compares.
+- **Track against history.** Working tree vs the last few releases at once.
+- **Compare candidate fixes.** One baseline vs several feature branches.
+- **Fidelity sweep.** Same version, varied replay options (cache on/off, validation
+  on/off) — see how the answer moves as fidelity climbs.
+
+## Two orthogonal axes (this is the key framing)
+The Impact Report has two independent dimensions:
+
+- **Depth / realization axis** (the L4b ladder): how faithfully ONE side is
+  realized — grammar → schema enrichment → construction cache → wildcard validation
+  → full built dispatch. This is [`replay-l4b-design.md`](./replay-l4b-design.md) (Steps 1–3).
+- **Breadth axis** (THIS doc): how MANY comparison targets sit beside the baseline.
+
+They compose. Each variant is a sandbox realized at some depth and carries its own
+fidelity descriptor. Multi-variant needs **none** of the deferred build epic — it
+works entirely at today's L1 grammar fidelity and gets richer for free as the depth
+axis advances.
+
+```
+                depth (realization, L4b) ──▶
+  breadth   B0(baseline)  grammar | +cache | +validation | built
+  (this     B1            grammar | +cache | …
+  doc)      B2            grammar | …
+   │        …
+   ▼
+```
+
+## Grounding in the MVP plan and the gates (A–E)
+Canonical source: [`04-mvp-slice.md`](./04-mvp-slice.md) §3 (gates) + §2 (excludes);
+journeys in [`02-journeys.md`](./02-journeys.md).
+
+The MVP is fenced by five gates; the load-bearing one for replay is:
+
+- **Gate C — headline / validation gate (journey J4, persona P4 "Regression Owner").**
+  The Impact Report, run with the default "likely-bad change" predicate (F4.4) on a
+  **hand-labelled `player` regression set**, agrees with developer judgment on **≥ 80%
+  of rows** (red = regression, green = improvement), under the **deterministic
+  `needs-explanation` miss policy**. Evaluated on the **binary** compare
+  (`working tree` vs `HEAD~1`). This is "the single hardest gate."
+
+Where multi-variant sits relative to the gates:
+
+- **It is POST-MVP and on none of A–E's critical path.** Every gate (A stand-up, B
+  schema loop, C headline, D drill-in, E live mirror) is satisfied by the binary
+  compare. Multi-variant adds **breadth**, which no MVP gate requires.
+- **It is the sibling of the existing §2 exclude.** §2 already defers
+  **multi-agent / multi-corpus simultaneous replay** (one agent → many agents) as
+  post-MVP. Multi-variant is the *other* replay-scaling axis — **one agent → many
+  versions**. Both are post-MVP breadth extensions of the same `replayCorpus` engine;
+  they are independent (you can have either without the other).
+- **It must PRESERVE Gate C, not move it.** Because binary is exactly N=1, the
+  slice-1 result-shape refactor has to keep single-variant `ActionDelta`/row semantics
+  byte-for-byte identical, so the ≥ 80% agreement number doesn't shift. The F4.4
+  red/green predicate and the deterministic `needs-explanation` policy run **per cell**
+  unchanged — so each variant column carries the same Gate-C-validated semantics.
+- **It is the natural post-MVP evolution of the J4 headline.** J4 is literally "find a
+  regression." The bisect / first-divergence feature (range-expand `good..HEAD`,
+  per-row first `=`→`Δ` column = the introducing commit) is the strongest realization
+  of that journey — it turns "is there a regression vs HEAD~1" into "which commit
+  introduced it," for persona P4, once the MVP gate is banked.
+
+### Two scaling axes vs one depth axis (the 2×2)
+`replayCorpus` (F4.1) is the primitive. Three independent dials sit on top:
+
+| Dial | What it scales | Doc | MVP status |
+| --- | --- | --- | --- |
+| **Fidelity / depth** (L1–L4b) | how faithfully each *side* is realized → improves Gate C row accuracy | [`replay-l4b-design.md`](./replay-l4b-design.md) | L1–L3 in; L4b deferred |
+| **Breadth — versions** (this) | how many *variants* beside the baseline | this doc | post-MVP |
+| **Breadth — agents/corpora** (§2) | how many *agents* replayed at once | [`04-mvp-slice.md` §2](./04-mvp-slice.md) | post-MVP |
+
+Depth improves the *quality* of each cell (and thus Gate C); the two breadth axes
+multiply the *number* of cells. Multi-variant only touches breadth-versions, composes
+with depth per-cell, and is orthogonal to breadth-agents.
+
+## The reframe: Baseline + Variants
+- **A → Baseline**: the fixed reference column.
+- **B → Variants**: an ordered list of comparison targets.
+- A variant is not just a version — it is **version + replay options**:
+
+```ts
+interface ReplayVariant {
+    version: VersionSpec;           // { kind:"git"; ref } | { kind:"workingTree" }
+    mode: StudioReplayMode;         // "nfa-grammar" | "completionBased-cache"
+    validateWildcards: boolean;
+    label?: string;                 // display label; defaults from version/options
+}
+```
+
+Folding options into the variant unifies *version* comparison and *fidelity*
+comparison under one abstraction — the option-sweep journey is then free.
+
+## Data model generalization
+The engine's binary pairs (`actionA`/`actionB`, `cacheStateA`/`cacheStateB`,
+`collisionsA`/`collisionsB`, `latencyA`/`latencyB`) become baseline + a variants
+array. Sketch:
+
+```ts
+interface VariantCell {
+    action?: unknown;
+    cacheState: ReplayCacheState;
+    collisions: CollisionDetectedEvent[];
+    latency: number;
+    requestId: string;
+    relation: "equal" | "changed" | "new-match" | "lost-match"; // vs baseline
+    fidelity: FidelityReport;       // per-variant (composes with the depth axis)
+}
+
+interface RowResult {
+    utterance: string;
+    utteranceId: string;
+    source: CorpusSource;
+    baseline: { action?: unknown; cacheState: ReplayCacheState;
+                collisions: CollisionDetectedEvent[]; latency: number; requestId: string };
+    variants: VariantCell[];        // index-aligned with the request's variants[]
+}
+
+interface VariantSummary {           // one per variant, vs baseline
+    equalCount: number; changedCount: number;
+    newMatchCount: number; lostMatchCount: number;
+    collisionDelta: number; duration: number;
+}
+```
+
+- **Baseline is computed once** and reused across all variants → cost is linear in N,
+  not N². 
+- `variants.length === 1` serializes/renders identically to today's A/B.
+
+### Coordination with L4b Step 1 (important — shape it once)
+L4b Step 1 plans to add `sideFidelity: { A: FidelityReport; B: FidelityReport }` to
+`StudioReplayResult`. That is the exact `{A, B}` pair this redesign turns into
+`baseline + variants[]`. To avoid refactoring the same field twice, when Step 1
+lands the fidelity descriptor should be modelled **per sandbox** —
+`baselineFidelity: FidelityReport` + `variants[i].fidelity` — rather than a fixed
+A/B pair. Even before multi-variant ships, expressing it as "baseline + a list of
+one" makes the later generalization a no-op on the type.
+
+## UI design
+
+### Action bar
+```
+Baseline:[▾ Working tree]   Variants:[HEAD ✕][HEAD~1 ✕][feat/x ✕][＋]   ⚙ options  ▶ Run
+```
+- Variants are chips; `＋` opens today's version picker. The swap button retires
+  (the compare is asymmetric now).
+- Per-variant options (mode/validation) hang off each chip's own little gear, or a
+  shared default with per-chip overrides.
+
+### Per-variant summary strip
+```
+  HEAD          HEAD~1        feat/x
+  ✓ 48 = · 2Δ   ✓ 47 = · 3Δ   ⚠ 31 = · 19Δ · 4−
+```
+One card per variant: equal/changed/new/lost counts vs baseline; click to focus.
+
+### Matrix (primary view)
+```
+Utterance                     │ Baseline │ HEAD │ HEAD~1 │ feat/x
+"play despacito by fonsi"     │ playTrack│  =   │   =    │   Δ
+"turn on the lights"          │ toggle   │  =   │   Δ    │   Δ
+"set a 5 minute timer"        │ setTimer │  =   │   =    │   =
+"queue some jazz"             │ (none)   │  =   │   =    │   + new
+```
+- Sticky baseline column; one column per variant; horizontal scroll past ~4–5.
+- Cell glyph relative to baseline — `=` equal, `Δ` changed, `+` new, `−` lost —
+  reusing today's vocabulary, just per column.
+- Click a **cell** → drawer with the full baseline-vs-that-variant action diff
+  (reuses today's binary detail view verbatim).
+
+### Two view modifiers
+- **Divergence filter** — hide rows that are `=` across every variant (essential at
+  scale; most rows are stable).
+- **Row fingerprint** (compact) — collapse columns to `●●○` per row (matches B1,B2;
+  differs B3) when there are many variants.
+
+## The payoff: ordered variants → first-divergence / bisect
+If variants are an ordered chronological range (old→new), the matrix answers "find
+the regression" directly: per row, the column where it first flips `=`→`Δ` **is the
+commit that changed it**.
+- **Range-expand source**: give a range `good..HEAD`; auto-create one variant per
+  commit (capped, e.g. ≤8); highlight each row's first-divergence column.
+- Per-row "introduced at →" annotation pointing at that commit.
+
+This makes the headline journey ("I edited grammar; which commit regressed it?") a
+single run.
+
+## Where variants come from
+1. **Manual** — add ref / pick commit / working tree (today's picker, N times).
+2. **Range-expand** — `A..B` → a variant per commit (bisect mode above).
+3. **Option sweep** — same version, varied `mode`/`validateWildcards` (fidelity
+   sweep). Free once options live on the variant.
+
+## Cost, perf, back-compat
+- Baseline computed once; each variant is one extra pass. Cap N with a warning;
+  range-expand caps commits.
+- **Progressive fill** — stream columns in as each variant completes (the matrix is
+  column-addressable) rather than blocking on the whole sweep.
+- **Cache by `(agent, version, mode)`** so adding a variant doesn't recompute
+  existing columns.
+- Fully backward compatible: N=1 renders exactly like today; the simple two-target
+  bar is the default and the matrix is progressive disclosure via `＋`.
+
+## Delivery slices
+1. **Result-shape generalization** — baseline + `variants[]` in the core result
+   types + engine, N=1 rendering identical. Pure refactor; the risk-bearing slice.
+   Best landed together with (or immediately after) L4b Step 1 so the fidelity
+   descriptor is shaped per-sandbox once.
+2. **Matrix UI** — variant chips + per-variant summary cards + matrix columns
+   (manual variants only).
+3. **Divergence filter + cell drill-down drawer** (reuses binary detail view).
+4. **Range-expand + first-divergence** (the bisect payoff).
+5. **Option-sweep variants** (fidelity sweep).
+
+## Where it fits in the work plan
+A **new workstream on the breadth-versions axis**, parallel to the L4b realization
+ladder, and **gated behind the MVP**:
+- **Precondition: Gate C is banked first.** Multi-variant is post-MVP (§2 sibling of
+  the deferred multi-agent axis). Do not start it until the binary Impact Report has
+  passed Gate C on the hand-labelled `player` set — otherwise slice 1's refactor risks
+  perturbing the very rows the gate measures.
+- **Depends on:** the sandbox framing — best sequenced *after* L4b Steps 1–2
+  (per-side fidelity readout + "Sandbox A/B" relabel), because multi-variant is the
+  natural generalization of "Sandbox A / Sandbox B" to "Baseline + N variant
+  sandboxes," and slice 1 shares the `StudioReplayResult` fidelity refactor with
+  Step 1.
+- **Independent of:** L4b Step 3 (build-from-ref) and the §2 multi-agent axis.
+  Multi-variant ships entirely at L1 grammar fidelity and simply gets richer cells as
+  the depth axis advances.
+- **Serves:** persona P4 (Quality / Regression Owner) and journey J4 — it extends the
+  validated headline ("find a regression vs HEAD~1") into bisect ("which commit
+  introduced it").
+- **Sequencing recommendation:** [MVP Gate C banked] → L4b Step 1 (shape fidelity
+  per-sandbox) → L4b Step 2 (relabel) → this slice 1 (baseline + variants[] result
+  shape) → this slice 2 (matrix) → slices 3–5. Step 3 (build epic) stays deferred and
+  orthogonal.
+
+## Open questions
+- Per-variant options vs one shared option set with overrides — how much per-chip
+  control is worth the UI weight?
+- Default column cap (N) and range-expand commit cap?
+- Is symmetric variant-vs-variant comparison ever wanted, or is baseline-relative
+  always sufficient? (Baseline-relative keeps the model simple; a "distinct
+  outcomes per row" cluster count could cover the rest without full N×N.)
diff --git a/ts/docs/plans/vscode-devx/replay-l4b-design.md b/ts/docs/plans/vscode-devx/replay-l4b-design.md
new file mode 100644
index 0000000000..ccca77e66b
--- /dev/null
+++ b/ts/docs/plans/vscode-devx/replay-l4b-design.md
@@ -0,0 +1,149 @@
+# Replay L4b — build-from-git-ref — first-slice design
+
+**Builds on:** L1 (schema-enriched grammar), L2 (cache, working-tree), L3 (deterministic dispatch),
+L4a (live working-tree wildcard validation). This is the deepest rung of the replay fidelity ladder.
+
+## The honest scope problem
+The fidelity ladder classifies L4 as the largest, fully greenfield rung — multi-week and behind a
+flag — needing a product steer before it is attempted. A faithful full L4b is genuinely multi-week
+and should NOT be attempted in one pass. So this doc scopes a **tractable first slice** plus the
+decision that gates which slice to build.
+
+## What actually changes between two versions (verified)
+- **Grammar (`.agr`)** — committed, versionable. Already compared at a ref via `git show`.
+- **Action schema (`.ts` source)** — committed, versionable. Parsed from SOURCE (no build needed:
+  `parseActionSchemaSource` reads TS text), already read at a ref via `git show` (L1 enrichment).
+- **Validator / handler code (`.ts` → built `.js`)** — committed source, but only the **working
+  tree** is BUILT. L4a runs the real `validateWildcardMatch` only on the working-tree side because
+  that's the only side whose JS exists. The git-ref side can't run validators without a build.
+- **Construction cache (`.json`)** — NOT committed (runtime artifact). Can NEVER be read at a ref.
+  So "frozen cache at a ref" is impossible by construction; L4b cannot deliver it.
+
+⇒ The ONLY thing a full build-from-ref unlocks that we don't already have is **running the ref's
+compiled validator/handler JS** (and a freshly-warmed cache, which isn't a frozen artifact anyway).
+
+## Current materialization limits (what a worktree would fix)
+- `git show <ref>:<path>` reads ONE file at a time. Imported `.agr` grammars (a grammar that
+  `import`s another `.agr`) aren't resolvable at a ref — each import needs its own `git show` with a
+  versioned FileLoader (deferred since the real-replay slice). The doc notes 57/65 `.agr` have
+  imports but **none import another `.agr`**, so this is rare.
+- The `ReplayActionResolver` contract has **no `dispose()`** — `engine.ts` `cancel()` just flips a
+  flag; the runtime consumes rows without a `finally` cleanup hook. Any worktree/checkout slice MUST
+  add a disposer first (a hard prerequisite before any build-from-ref work).
+
+## Three candidate first slices
+
+### Option A — Worktree materializer + resolver `dispose()` lifecycle (infra-first, NO build)
+Replace per-file `git show` with a single `git worktree add --detach <ref> <tmp>` per git-ref side;
+read all grammar/schema/paramSpec files from the checkout; `git worktree remove` in a new
+`dispose()`/`finally` path threaded through resolver → runtime.
+- **Unlocks:** imported-`.agr` + whole-tree schema at a ref; establishes the disposer lifecycle that
+  the full build rung mandates. De-risks Option C's hardest plumbing.
+- **Honest gain:** modest today (imported-`.agr` is rare), but it is the **required foundation** for
+  any build-from-ref and removes the "no disposer" blocker.
+- **Cost:** small–medium. Main risks: worktree cleanup on crash, lock contention with the user's live
+  repo, Windows path/locking. Mitigated by `--detach` (no branch checkout) + a temp dir under the OS
+  temp root + best-effort `git worktree prune` on dispose.
+
+### Option B — Ref-side wildcard validation via "unchanged-validator" shortcut (NO build)
+Run L4a on the git-ref side too, but ONLY when `git diff <ref>..worktree -- <validator file>` is
+empty (the agent's validator JS is unchanged between the ref and the built working tree), so the
+already-built working-tree module faithfully represents the ref's validator. If the validator
+changed, skip ref-side validation (honest, labelled "ref validation: skipped (validator changed)").
+- **Unlocks:** symmetric wildcard validation for the common case (validator untouched, only grammar/
+  schema edited — exactly the regression journey) with ZERO build machinery.
+- **Honest gain:** real for the headline journey; limited to validators (timer/list).
+- **Cost:** small. Risk: the "unchanged ⇒ safe to reuse working-tree module" inference must be
+  conservative (compare the whole built package's source closure, not just one file, or restrict to
+  the allowlisted agents whose validator lives in one known file).
+
+### Option C — Full build-from-ref (the real L4b, multi-week, behind a flag)
+`git worktree add` → `pnpm install --filter <agent>...` → scoped `tsc -b`/`asc`/`agc` → load the
+built dist via a provider pointed at the worktree → full deterministic dispatch at the ref.
+- **Unlocks:** true arbitrary-ref full-dispatch compare.
+- **Cost:** multi-week, fully greenfield. Minutes per run (install + build). Many failure modes
+  (install flakiness, native deps, disk). Clearly a flagged epic, not a one-session slice.
+
+## Recommendation
+Ship **Option A** now (the lifecycle + worktree foundation — the mandatory step 1 of the L4 epic),
+and optionally **Option B** as the first user-visible fidelity win on top of it. Defer **Option C**'s
+install/build orchestration to a tracked, flagged epic, started only if a concrete journey needs
+ref-vs-ref full dispatch.
+
+## Open decision
+Which first slice? **A (infra foundation)**, **A+B (foundation + ref-side validation win)**, or
+**commit to C (the multi-week build epic) now**?
+
+---
+
+## Design review — verdict & resolution
+- **Option A — CUT as standalone / defer.** Worktree + disposer is infra ahead of need; `git show`
+  is strictly better while we only READ committed source (no checkout cleanup, no repo-lock
+  contention vs the user's live checkout, no Windows file-lock/crash-recovery burden). Build the
+  lifecycle only when something real needs disposing (i.e. inside C).
+- **Option B — SHIP ONLY IF very conservative + clearly labelled.** "Validator file unchanged" is
+  NOT sound alone (a changed transitive import / helper / lockfile / tsconfig / data file silently
+  breaks it). Acceptable only if the whole validator source CLOSURE is unchanged, only for the tiny
+  timer/list validators, and surfaced as "opportunistic," not full ref fidelity. If closure
+  detection starts to grow → cut (it becomes a partial rebuild system).
+- **Option C — DEFER to a flagged epic** pending a real product journey. Guardrails before starting:
+  explicit flag; cache built refs by commit SHA; hard timeout + clear failure modes; disposable
+  worktree lifecycle first; per-agent build recipes (not guessed generic builds); strong UI
+  labelling when a ref can't be built.
+- **Better immediate slice:** ship **fidelity transparency** — per side/version report which
+  fidelity layers ran (grammar / schema enrichment / construction cache / wildcard validation /
+  dispatch) and the exact skip reason, plus a preflight "build-from-ref would add X (unsupported
+  today)." Honest user value now, near-zero risk, and the right scaffold that makes L4b's eventual
+  payoff visible/measurable.
+
+**Resolution:** sandboxes are the design's intended EXECUTION CONTAINERS (today they
+are metadata-only shells); a VersionSpec is a *recipe* and L4b is the *build* that realizes a recipe
+into a populated sandbox. The UI evolves "pick 2 commits" → "pick 2 sandboxes" along a single
+**realization axis**, in three incremental steps. Near-term = Steps 1–2 (transparency, no build);
+Step 3 (build-from-ref) stays the deferred flagged epic.
+
+## The sandbox-convergence plan (chosen direction)
+
+```
+VersionSpec (recipe)        L4b build (Step 3)            Sandbox (instance)
+  {git, ref}      ──▶  worktree → tsc/asc → load dist  ──▶  built agent + dispatcher + cache
+  {workingTree}   ──▶  (already built locally)          ──▶  the live agent
+```
+
+### Step 1 — per-side fidelity readout (SHIP NOW, no build, no new replay behavior)
+Surface signals the resolver ALREADY computes. Add to `StudioReplayResult`:
+```ts
+interface FidelityReport {
+  realization: "source" | "built-live";          // git ref ⇒ source; working tree ⇒ built-live
+  layers: Record<
+    "grammar" | "schemaEnrichment" | "constructionCache" | "wildcardValidation" | "dispatch",
+    { status: "ran" | "skipped" | "unavailable"; reason?: string }
+  >;
+}
+interface StudioReplayResult { /* …existing… */ sideFidelity: { A: FidelityReport; B: FidelityReport }; }
+```
+Derive from existing `methodA`/`methodB` (`static-grammar`/`schema-grammar`/`construction-cache`),
+`grammarResolver.enriched`, `wildcardValidation`, and `versionA/B.kind`. Pure mapping + unit test.
+Webview: collapsible Fidelity matrix (5 rows × A/B) + a preflight line when a side is `source`-only
+and a richer realization is theoretically available. Reuse `renderValidationNote` styling.
+
+### Step 2 — relabel A/B as "Sandbox A / Sandbox B" (light)
+A/B group header becomes Sandbox A / Sandbox B with a `recipe · realization` sub-label. Cements the
+"2 sandboxes" model. Mostly labels on top of Step 1's descriptor.
+
+### Step 3 — L4b build-from-ref (DEFERRED, flagged epic = old Option C)
+Picker gains a "build from ref" opt-in → host realizes a real sandbox (worktree → build → load) →
+ref side climbs the Step-1 matrix to `built`. Needs the resolver `dispose()`/`finally` lifecycle,
+per-agent build recipes, SHA-keyed build cache, timeouts. Subprocess-mode + the L4a RPC-context
+boundary remain the hardest sub-problem (context-dependent agents).
+
+## Implementation plan — near-term (Steps 1–2)
+1. **core/studioRuntimeCore.ts** — add `SideFidelity`/`FidelityReport` types + a pure
+   `deriveSideFidelity(methodA, methodB, enriched, wildcardValidation, versionA, versionB)` mapper;
+   populate `result.sideFidelity` (and in the error-result path). Unit test the mapper.
+2. **webviewKit/protocol.ts** — carry `sideFidelity` on `replay.result` (additive/optional).
+3. **webviewKit/client/impactReport.ts + media/impactReport.css** — Fidelity matrix panel +
+   preflight line + Sandbox A/B relabel. Render test.
+4. **Verify**: core `tsc -b` + jest, studio `tsx --test`, studio-service `npm test`, prettier,
+   esbuild. No commit/push without explicit user consent.
+Deferred (Step 3): worktree materializer, scoped build, resolver `dispose()`, ref-side execution.

From 4de0a7b09d95718c02229dc087150a90ca1d3c82 Mon Sep 17 00:00:00 2001
From: typeagent-bot <typeagent-bot[bot]@users.noreply.github.com>
Date: Tue, 30 Jun 2026 20:38:15 +0000
Subject: [PATCH 14/15] style: apply prettier formatting and policy fixes

---
 ts/docs/plans/vscode-devx/README.md           | 18 ++--
 ts/docs/plans/vscode-devx/STATUS.md           | 36 ++++----
 .../impact-report-multi-variant-design.md     | 92 +++++++++++++------
 .../plans/vscode-devx/replay-l4b-design.md    | 34 ++++++-
 4 files changed, 118 insertions(+), 62 deletions(-)

diff --git a/ts/docs/plans/vscode-devx/README.md b/ts/docs/plans/vscode-devx/README.md
index 13fa523663..a71938395a 100644
--- a/ts/docs/plans/vscode-devx/README.md
+++ b/ts/docs/plans/vscode-devx/README.md
@@ -20,15 +20,15 @@
 
 ### Companion docs (architecture, agent-drivability, status)
 
-| Doc                                  | What it answers                                                                                              |
-| ------------------------------------ | ------------------------------------------------------------------------------------------------------------ |
-| [DESIGN.md](./DESIGN.md)             | Architecture; **§3.0** the guiding principle — headless core, thin presenters, three audiences.              |
-| [USER-STORY.md](./USER-STORY.md)     | The authoring loop and the three interaction modes (human / AI-agent / hybrid).                              |
-| [STUDIO-AGENT.md](./STUDIO-AGENT.md) | The `studio` agent **action-surface reference** (groups A–F, tiers, approval boundary). Phasing lives in 05. |
-| [STATUS.md](./STATUS.md)             | What's built, known issues, and the ready-to-start next slices (pointing into the 05 §11 phasing).           |
-| [replay-l4b-design.md](./replay-l4b-design.md) | The replay **fidelity depth ladder** — sandbox-convergence framing and the build-from-git-ref (L4b) first-slice plan (Steps 1–3). |
-| [impact-report-multi-variant-design.md](./impact-report-multi-variant-design.md) | The Impact Report **breadth axis** — Baseline + N Variants compare matrix and regression bisect. |
-| [QUICKSTART.md](./QUICKSTART.md)     | Build/test/run commands and where each piece lives.                                                          |
+| Doc                                                                              | What it answers                                                                                                                   |
+| -------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- |
+| [DESIGN.md](./DESIGN.md)                                                         | Architecture; **§3.0** the guiding principle — headless core, thin presenters, three audiences.                                   |
+| [USER-STORY.md](./USER-STORY.md)                                                 | The authoring loop and the three interaction modes (human / AI-agent / hybrid).                                                   |
+| [STUDIO-AGENT.md](./STUDIO-AGENT.md)                                             | The `studio` agent **action-surface reference** (groups A–F, tiers, approval boundary). Phasing lives in 05.                      |
+| [STATUS.md](./STATUS.md)                                                         | What's built, known issues, and the ready-to-start next slices (pointing into the 05 §11 phasing).                                |
+| [replay-l4b-design.md](./replay-l4b-design.md)                                   | The replay **fidelity depth ladder** — sandbox-convergence framing and the build-from-git-ref (L4b) first-slice plan (Steps 1–3). |
+| [impact-report-multi-variant-design.md](./impact-report-multi-variant-design.md) | The Impact Report **breadth axis** — Baseline + N Variants compare matrix and regression bisect.                                  |
+| [QUICKSTART.md](./QUICKSTART.md)                                                 | Build/test/run commands and where each piece lives.                                                                               |
 
 ---
 
diff --git a/ts/docs/plans/vscode-devx/STATUS.md b/ts/docs/plans/vscode-devx/STATUS.md
index 1e3dcd9e13..7162c570be 100644
--- a/ts/docs/plans/vscode-devx/STATUS.md
+++ b/ts/docs/plans/vscode-devx/STATUS.md
@@ -45,16 +45,16 @@ is clear why it waits. There is no second roadmap — depth (L4b) and breadth
 
 ### Gate spine (MVP critical path)
 
-| Gate | Journey | Capability | Phase | Status |
-| ---- | ------- | ---------- | ----- | ------ |
-| **A** | J1 Stand up an agent | New-Agent Wizard | P-2 | ❌ not started (onboarding bridge ✅) |
-| **B** | J2 Tune schema | Schema Studio | P-4 | ❌ not started |
-| **C** | J4 Find a regression | Impact Report ≥ 80% red/green on hand-labelled `player` | P-3 | 🟡 **long pole** — engine L1–L4a ✅; corpus capture + predicate + measurement ❌ |
-| **D** | J5 Debug a trace | Trace Viewer | P-3 | ❌ not started |
-| **E** | J6 Observe live | Live Trace + status bar | P-5 | ❌ not started |
+| Gate  | Journey              | Capability                                              | Phase | Status                                                                           |
+| ----- | -------------------- | ------------------------------------------------------- | ----- | -------------------------------------------------------------------------------- |
+| **A** | J1 Stand up an agent | New-Agent Wizard                                        | P-2   | ❌ not started (onboarding bridge ✅)                                            |
+| **B** | J2 Tune schema       | Schema Studio                                           | P-4   | ❌ not started                                                                   |
+| **C** | J4 Find a regression | Impact Report ≥ 80% red/green on hand-labelled `player` | P-3   | 🟡 **long pole** — engine L1–L4a ✅; corpus capture + predicate + measurement ❌ |
+| **D** | J5 Debug a trace     | Trace Viewer                                            | P-3   | ❌ not started                                                                   |
+| **E** | J6 Observe live      | Live Trace + status bar                                 | P-5   | ❌ not started                                                                   |
 
 **Critical path now: close Gate C** — player corpus capture → predicate tuning →
-run the ≥ 80% validation. It is the only *tunable* gate and §7's top risk.
+run the ≥ 80% validation. It is the only _tunable_ gate and §7's top risk.
 
 ### Backlog (off the critical path — tagged, not a separate plan)
 
@@ -63,18 +63,18 @@ tags: **depth** = how faithfully one replay side is realized (improves Gate C
 accuracy); **breadth** = how many things are compared at once (multiplies cells);
 **infra** = enabling plumbing.
 
-| Item | Parent | Track | Precondition | Status |
-| ---- | ------ | ----- | ------------ | ------ |
-| Fidelity transparency (per-side readout + "Sandbox A/B" relabel) | Gate C / J4 | depth | — | next (low-risk, no build) |
-| **L4b** build-from-ref sandboxes (real compiled ref side) | Gate C / J4 | depth (P-7) | Gate C banked | deferred flagged epic |
-| Multi-variant compare (Baseline vs N variants; bisect / first-divergence) | J4 | breadth — versions | Gate C banked; must preserve Gate C | post-MVP |
-| Multi-agent / multi-corpus replay (`code`, 684 utterances) | J4 | breadth — agents | Gate C banked | post-MVP (§2 exclude) |
-| Active-sandbox selector + per-sandbox scoping | — | infra | single-sandbox E2E | P-7a |
-| Sandbox copy-on-write overlay (true sandbox-local A/B) | — | infra | P-7a | P-7b |
+| Item                                                                      | Parent      | Track              | Precondition                        | Status                    |
+| ------------------------------------------------------------------------- | ----------- | ------------------ | ----------------------------------- | ------------------------- |
+| Fidelity transparency (per-side readout + "Sandbox A/B" relabel)          | Gate C / J4 | depth              | —                                   | next (low-risk, no build) |
+| **L4b** build-from-ref sandboxes (real compiled ref side)                 | Gate C / J4 | depth (P-7)        | Gate C banked                       | deferred flagged epic     |
+| Multi-variant compare (Baseline vs N variants; bisect / first-divergence) | J4          | breadth — versions | Gate C banked; must preserve Gate C | post-MVP                  |
+| Multi-agent / multi-corpus replay (`code`, 684 utterances)                | J4          | breadth — agents   | Gate C banked                       | post-MVP (§2 exclude)     |
+| Active-sandbox selector + per-sandbox scoping                             | —           | infra              | single-sandbox E2E                  | P-7a                      |
+| Sandbox copy-on-write overlay (true sandbox-local A/B)                    | —           | infra              | P-7a                                | P-7b                      |
 
 Why depth and breadth are distinct dials on the same `replayCorpus` (F4.1)
-primitive: **depth** raises the fidelity of each compared *cell* (and thus Gate C
-accuracy); the two **breadth** axes raise the *number* of cells (versions ×
+primitive: **depth** raises the fidelity of each compared _cell_ (and thus Gate C
+accuracy); the two **breadth** axes raise the _number_ of cells (versions ×
 agents). They compose and are independent. Detailed slice breakdowns live in the
 design docs ([the L4b sandbox-convergence plan](./replay-l4b-design.md) and
 [the multi-variant compare design](./impact-report-multi-variant-design.md)).
diff --git a/ts/docs/plans/vscode-devx/impact-report-multi-variant-design.md b/ts/docs/plans/vscode-devx/impact-report-multi-variant-design.md
index dec4a6ac1f..fa0fee2b6e 100644
--- a/ts/docs/plans/vscode-devx/impact-report-multi-variant-design.md
+++ b/ts/docs/plans/vscode-devx/impact-report-multi-variant-design.md
@@ -5,11 +5,13 @@
 [`replay-l4b-design.md`](./replay-l4b-design.md) (A/B reframed as "Sandbox A / Sandbox B").
 
 ## One-line idea
+
 Replace the fixed two-target compare (A vs B) with **one Baseline compared against
 an ordered list of Variants** (A vs B1, B2, … Bn), rendered as a matrix. Binary
 A/B becomes the N=1 special case.
 
 ## Why — the journeys this unlocks
+
 - **Bisect a regression.** "Which of these N commits changed the answer?" Baseline =
   known-good ref; variants = suspect commits. One run instead of N binary compares.
 - **Track against history.** Working tree vs the last few releases at once.
@@ -18,6 +20,7 @@ A/B becomes the N=1 special case.
   on/off) — see how the answer moves as fidelity climbs.
 
 ## Two orthogonal axes (this is the key framing)
+
 The Impact Report has two independent dimensions:
 
 - **Depth / realization axis** (the L4b ladder): how faithfully ONE side is
@@ -40,6 +43,7 @@ axis advances.
 ```
 
 ## Grounding in the MVP plan and the gates (A–E)
+
 Canonical source: [`04-mvp-slice.md`](./04-mvp-slice.md) §3 (gates) + §2 (excludes);
 journeys in [`02-journeys.md`](./02-journeys.md).
 
@@ -59,7 +63,7 @@ Where multi-variant sits relative to the gates:
   compare. Multi-variant adds **breadth**, which no MVP gate requires.
 - **It is the sibling of the existing §2 exclude.** §2 already defers
   **multi-agent / multi-corpus simultaneous replay** (one agent → many agents) as
-  post-MVP. Multi-variant is the *other* replay-scaling axis — **one agent → many
+  post-MVP. Multi-variant is the _other_ replay-scaling axis — **one agent → many
   versions**. Both are post-MVP breadth extensions of the same `replayCorpus` engine;
   they are independent (you can have either without the other).
 - **It must PRESERVE Gate C, not move it.** Because binary is exactly N=1, the
@@ -74,72 +78,85 @@ Where multi-variant sits relative to the gates:
   introduced it," for persona P4, once the MVP gate is banked.
 
 ### Two scaling axes vs one depth axis (the 2×2)
+
 `replayCorpus` (F4.1) is the primitive. Three independent dials sit on top:
 
-| Dial | What it scales | Doc | MVP status |
-| --- | --- | --- | --- |
-| **Fidelity / depth** (L1–L4b) | how faithfully each *side* is realized → improves Gate C row accuracy | [`replay-l4b-design.md`](./replay-l4b-design.md) | L1–L3 in; L4b deferred |
-| **Breadth — versions** (this) | how many *variants* beside the baseline | this doc | post-MVP |
-| **Breadth — agents/corpora** (§2) | how many *agents* replayed at once | [`04-mvp-slice.md` §2](./04-mvp-slice.md) | post-MVP |
+| Dial                              | What it scales                                                        | Doc                                              | MVP status             |
+| --------------------------------- | --------------------------------------------------------------------- | ------------------------------------------------ | ---------------------- |
+| **Fidelity / depth** (L1–L4b)     | how faithfully each _side_ is realized → improves Gate C row accuracy | [`replay-l4b-design.md`](./replay-l4b-design.md) | L1–L3 in; L4b deferred |
+| **Breadth — versions** (this)     | how many _variants_ beside the baseline                               | this doc                                         | post-MVP               |
+| **Breadth — agents/corpora** (§2) | how many _agents_ replayed at once                                    | [`04-mvp-slice.md` §2](./04-mvp-slice.md)        | post-MVP               |
 
-Depth improves the *quality* of each cell (and thus Gate C); the two breadth axes
-multiply the *number* of cells. Multi-variant only touches breadth-versions, composes
+Depth improves the _quality_ of each cell (and thus Gate C); the two breadth axes
+multiply the _number_ of cells. Multi-variant only touches breadth-versions, composes
 with depth per-cell, and is orthogonal to breadth-agents.
 
 ## The reframe: Baseline + Variants
+
 - **A → Baseline**: the fixed reference column.
 - **B → Variants**: an ordered list of comparison targets.
 - A variant is not just a version — it is **version + replay options**:
 
 ```ts
 interface ReplayVariant {
-    version: VersionSpec;           // { kind:"git"; ref } | { kind:"workingTree" }
-    mode: StudioReplayMode;         // "nfa-grammar" | "completionBased-cache"
-    validateWildcards: boolean;
-    label?: string;                 // display label; defaults from version/options
+  version: VersionSpec; // { kind:"git"; ref } | { kind:"workingTree" }
+  mode: StudioReplayMode; // "nfa-grammar" | "completionBased-cache"
+  validateWildcards: boolean;
+  label?: string; // display label; defaults from version/options
 }
 ```
 
-Folding options into the variant unifies *version* comparison and *fidelity*
+Folding options into the variant unifies _version_ comparison and _fidelity_
 comparison under one abstraction — the option-sweep journey is then free.
 
 ## Data model generalization
+
 The engine's binary pairs (`actionA`/`actionB`, `cacheStateA`/`cacheStateB`,
 `collisionsA`/`collisionsB`, `latencyA`/`latencyB`) become baseline + a variants
 array. Sketch:
 
 ```ts
 interface VariantCell {
+  action?: unknown;
+  cacheState: ReplayCacheState;
+  collisions: CollisionDetectedEvent[];
+  latency: number;
+  requestId: string;
+  relation: "equal" | "changed" | "new-match" | "lost-match"; // vs baseline
+  fidelity: FidelityReport; // per-variant (composes with the depth axis)
+}
+
+interface RowResult {
+  utterance: string;
+  utteranceId: string;
+  source: CorpusSource;
+  baseline: {
     action?: unknown;
     cacheState: ReplayCacheState;
     collisions: CollisionDetectedEvent[];
     latency: number;
     requestId: string;
-    relation: "equal" | "changed" | "new-match" | "lost-match"; // vs baseline
-    fidelity: FidelityReport;       // per-variant (composes with the depth axis)
+  };
+  variants: VariantCell[]; // index-aligned with the request's variants[]
 }
 
-interface RowResult {
-    utterance: string;
-    utteranceId: string;
-    source: CorpusSource;
-    baseline: { action?: unknown; cacheState: ReplayCacheState;
-                collisions: CollisionDetectedEvent[]; latency: number; requestId: string };
-    variants: VariantCell[];        // index-aligned with the request's variants[]
-}
-
-interface VariantSummary {           // one per variant, vs baseline
-    equalCount: number; changedCount: number;
-    newMatchCount: number; lostMatchCount: number;
-    collisionDelta: number; duration: number;
+interface VariantSummary {
+  // one per variant, vs baseline
+  equalCount: number;
+  changedCount: number;
+  newMatchCount: number;
+  lostMatchCount: number;
+  collisionDelta: number;
+  duration: number;
 }
 ```
 
 - **Baseline is computed once** and reused across all variants → cost is linear in N,
-  not N². 
+  not N².
 - `variants.length === 1` serializes/renders identically to today's A/B.
 
 ### Coordination with L4b Step 1 (important — shape it once)
+
 L4b Step 1 plans to add `sideFidelity: { A: FidelityReport; B: FidelityReport }` to
 `StudioReplayResult`. That is the exact `{A, B}` pair this redesign turns into
 `baseline + variants[]`. To avoid refactoring the same field twice, when Step 1
@@ -151,22 +168,27 @@ one" makes the later generalization a no-op on the type.
 ## UI design
 
 ### Action bar
+
 ```
 Baseline:[▾ Working tree]   Variants:[HEAD ✕][HEAD~1 ✕][feat/x ✕][＋]   ⚙ options  ▶ Run
 ```
+
 - Variants are chips; `＋` opens today's version picker. The swap button retires
   (the compare is asymmetric now).
 - Per-variant options (mode/validation) hang off each chip's own little gear, or a
   shared default with per-chip overrides.
 
 ### Per-variant summary strip
+
 ```
   HEAD          HEAD~1        feat/x
   ✓ 48 = · 2Δ   ✓ 47 = · 3Δ   ⚠ 31 = · 19Δ · 4−
 ```
+
 One card per variant: equal/changed/new/lost counts vs baseline; click to focus.
 
 ### Matrix (primary view)
+
 ```
 Utterance                     │ Baseline │ HEAD │ HEAD~1 │ feat/x
 "play despacito by fonsi"     │ playTrack│  =   │   =    │   Δ
@@ -174,6 +196,7 @@ Utterance                     │ Baseline │ HEAD │ HEAD~1 │ feat/x
 "set a 5 minute timer"        │ setTimer │  =   │   =    │   =
 "queue some jazz"             │ (none)   │  =   │   =    │   + new
 ```
+
 - Sticky baseline column; one column per variant; horizontal scroll past ~4–5.
 - Cell glyph relative to baseline — `=` equal, `Δ` changed, `+` new, `−` lost —
   reusing today's vocabulary, just per column.
@@ -181,15 +204,18 @@ Utterance                     │ Baseline │ HEAD │ HEAD~1 │ feat/x
   (reuses today's binary detail view verbatim).
 
 ### Two view modifiers
+
 - **Divergence filter** — hide rows that are `=` across every variant (essential at
   scale; most rows are stable).
 - **Row fingerprint** (compact) — collapse columns to `●●○` per row (matches B1,B2;
   differs B3) when there are many variants.
 
 ## The payoff: ordered variants → first-divergence / bisect
+
 If variants are an ordered chronological range (old→new), the matrix answers "find
 the regression" directly: per row, the column where it first flips `=`→`Δ` **is the
 commit that changed it**.
+
 - **Range-expand source**: give a range `good..HEAD`; auto-create one variant per
   commit (capped, e.g. ≤8); highlight each row's first-divergence column.
 - Per-row "introduced at →" annotation pointing at that commit.
@@ -198,12 +224,14 @@ This makes the headline journey ("I edited grammar; which commit regressed it?")
 single run.
 
 ## Where variants come from
+
 1. **Manual** — add ref / pick commit / working tree (today's picker, N times).
 2. **Range-expand** — `A..B` → a variant per commit (bisect mode above).
 3. **Option sweep** — same version, varied `mode`/`validateWildcards` (fidelity
    sweep). Free once options live on the variant.
 
 ## Cost, perf, back-compat
+
 - Baseline computed once; each variant is one extra pass. Cap N with a warning;
   range-expand caps commits.
 - **Progressive fill** — stream columns in as each variant completes (the matrix is
@@ -214,6 +242,7 @@ single run.
   bar is the default and the matrix is progressive disclosure via `＋`.
 
 ## Delivery slices
+
 1. **Result-shape generalization** — baseline + `variants[]` in the core result
    types + engine, N=1 rendering identical. Pure refactor; the risk-bearing slice.
    Best landed together with (or immediately after) L4b Step 1 so the fidelity
@@ -225,13 +254,15 @@ single run.
 5. **Option-sweep variants** (fidelity sweep).
 
 ## Where it fits in the work plan
+
 A **new workstream on the breadth-versions axis**, parallel to the L4b realization
 ladder, and **gated behind the MVP**:
+
 - **Precondition: Gate C is banked first.** Multi-variant is post-MVP (§2 sibling of
   the deferred multi-agent axis). Do not start it until the binary Impact Report has
   passed Gate C on the hand-labelled `player` set — otherwise slice 1's refactor risks
   perturbing the very rows the gate measures.
-- **Depends on:** the sandbox framing — best sequenced *after* L4b Steps 1–2
+- **Depends on:** the sandbox framing — best sequenced _after_ L4b Steps 1–2
   (per-side fidelity readout + "Sandbox A/B" relabel), because multi-variant is the
   natural generalization of "Sandbox A / Sandbox B" to "Baseline + N variant
   sandboxes," and slice 1 shares the `StudioReplayResult` fidelity refactor with
@@ -248,6 +279,7 @@ ladder, and **gated behind the MVP**:
   orthogonal.
 
 ## Open questions
+
 - Per-variant options vs one shared option set with overrides — how much per-chip
   control is worth the UI weight?
 - Default column cap (N) and range-expand commit cap?
diff --git a/ts/docs/plans/vscode-devx/replay-l4b-design.md b/ts/docs/plans/vscode-devx/replay-l4b-design.md
index ccca77e66b..11a5d61b19 100644
--- a/ts/docs/plans/vscode-devx/replay-l4b-design.md
+++ b/ts/docs/plans/vscode-devx/replay-l4b-design.md
@@ -4,12 +4,14 @@
 L4a (live working-tree wildcard validation). This is the deepest rung of the replay fidelity ladder.
 
 ## The honest scope problem
+
 The fidelity ladder classifies L4 as the largest, fully greenfield rung — multi-week and behind a
 flag — needing a product steer before it is attempted. A faithful full L4b is genuinely multi-week
 and should NOT be attempted in one pass. So this doc scopes a **tractable first slice** plus the
 decision that gates which slice to build.
 
 ## What actually changes between two versions (verified)
+
 - **Grammar (`.agr`)** — committed, versionable. Already compared at a ref via `git show`.
 - **Action schema (`.ts` source)** — committed, versionable. Parsed from SOURCE (no build needed:
   `parseActionSchemaSource` reads TS text), already read at a ref via `git show` (L1 enrichment).
@@ -23,6 +25,7 @@ decision that gates which slice to build.
 compiled validator/handler JS** (and a freshly-warmed cache, which isn't a frozen artifact anyway).
 
 ## Current materialization limits (what a worktree would fix)
+
 - `git show <ref>:<path>` reads ONE file at a time. Imported `.agr` grammars (a grammar that
   `import`s another `.agr`) aren't resolvable at a ref — each import needs its own `git show` with a
   versioned FileLoader (deferred since the real-replay slice). The doc notes 57/65 `.agr` have
@@ -34,9 +37,11 @@ compiled validator/handler JS** (and a freshly-warmed cache, which isn't a froze
 ## Three candidate first slices
 
 ### Option A — Worktree materializer + resolver `dispose()` lifecycle (infra-first, NO build)
+
 Replace per-file `git show` with a single `git worktree add --detach <ref> <tmp>` per git-ref side;
 read all grammar/schema/paramSpec files from the checkout; `git worktree remove` in a new
 `dispose()`/`finally` path threaded through resolver → runtime.
+
 - **Unlocks:** imported-`.agr` + whole-tree schema at a ref; establishes the disposer lifecycle that
   the full build rung mandates. De-risks Option C's hardest plumbing.
 - **Honest gain:** modest today (imported-`.agr` is rare), but it is the **required foundation** for
@@ -46,10 +51,12 @@ read all grammar/schema/paramSpec files from the checkout; `git worktree remove`
   temp root + best-effort `git worktree prune` on dispose.
 
 ### Option B — Ref-side wildcard validation via "unchanged-validator" shortcut (NO build)
+
 Run L4a on the git-ref side too, but ONLY when `git diff <ref>..worktree -- <validator file>` is
 empty (the agent's validator JS is unchanged between the ref and the built working tree), so the
 already-built working-tree module faithfully represents the ref's validator. If the validator
 changed, skip ref-side validation (honest, labelled "ref validation: skipped (validator changed)").
+
 - **Unlocks:** symmetric wildcard validation for the common case (validator untouched, only grammar/
   schema edited — exactly the regression journey) with ZERO build machinery.
 - **Honest gain:** real for the headline journey; limited to validators (timer/list).
@@ -58,25 +65,30 @@ changed, skip ref-side validation (honest, labelled "ref validation: skipped (va
   the allowlisted agents whose validator lives in one known file).
 
 ### Option C — Full build-from-ref (the real L4b, multi-week, behind a flag)
+
 `git worktree add` → `pnpm install --filter <agent>...` → scoped `tsc -b`/`asc`/`agc` → load the
 built dist via a provider pointed at the worktree → full deterministic dispatch at the ref.
+
 - **Unlocks:** true arbitrary-ref full-dispatch compare.
 - **Cost:** multi-week, fully greenfield. Minutes per run (install + build). Many failure modes
   (install flakiness, native deps, disk). Clearly a flagged epic, not a one-session slice.
 
 ## Recommendation
+
 Ship **Option A** now (the lifecycle + worktree foundation — the mandatory step 1 of the L4 epic),
 and optionally **Option B** as the first user-visible fidelity win on top of it. Defer **Option C**'s
 install/build orchestration to a tracked, flagged epic, started only if a concrete journey needs
 ref-vs-ref full dispatch.
 
 ## Open decision
+
 Which first slice? **A (infra foundation)**, **A+B (foundation + ref-side validation win)**, or
 **commit to C (the multi-week build epic) now**?
 
 ---
 
 ## Design review — verdict & resolution
+
 - **Option A — CUT as standalone / defer.** Worktree + disposer is infra ahead of need; `git show`
   is strictly better while we only READ committed source (no checkout cleanup, no repo-lock
   contention vs the user's live checkout, no Windows file-lock/crash-recovery burden). Build the
@@ -97,7 +109,7 @@ Which first slice? **A (infra foundation)**, **A+B (foundation + ref-side valida
   payoff visible/measurable.
 
 **Resolution:** sandboxes are the design's intended EXECUTION CONTAINERS (today they
-are metadata-only shells); a VersionSpec is a *recipe* and L4b is the *build* that realizes a recipe
+are metadata-only shells); a VersionSpec is a _recipe_ and L4b is the _build_ that realizes a recipe
 into a populated sandbox. The UI evolves "pick 2 commits" → "pick 2 sandboxes" along a single
 **realization axis**, in three incremental steps. Near-term = Steps 1–2 (transparency, no build);
 Step 3 (build-from-ref) stays the deferred flagged epic.
@@ -111,33 +123,45 @@ VersionSpec (recipe)        L4b build (Step 3)            Sandbox (instance)
 ```
 
 ### Step 1 — per-side fidelity readout (SHIP NOW, no build, no new replay behavior)
+
 Surface signals the resolver ALREADY computes. Add to `StudioReplayResult`:
+
 ```ts
 interface FidelityReport {
-  realization: "source" | "built-live";          // git ref ⇒ source; working tree ⇒ built-live
+  realization: "source" | "built-live"; // git ref ⇒ source; working tree ⇒ built-live
   layers: Record<
-    "grammar" | "schemaEnrichment" | "constructionCache" | "wildcardValidation" | "dispatch",
+    | "grammar"
+    | "schemaEnrichment"
+    | "constructionCache"
+    | "wildcardValidation"
+    | "dispatch",
     { status: "ran" | "skipped" | "unavailable"; reason?: string }
   >;
 }
-interface StudioReplayResult { /* …existing… */ sideFidelity: { A: FidelityReport; B: FidelityReport }; }
+interface StudioReplayResult {
+  /* …existing… */ sideFidelity: { A: FidelityReport; B: FidelityReport };
+}
 ```
+
 Derive from existing `methodA`/`methodB` (`static-grammar`/`schema-grammar`/`construction-cache`),
 `grammarResolver.enriched`, `wildcardValidation`, and `versionA/B.kind`. Pure mapping + unit test.
 Webview: collapsible Fidelity matrix (5 rows × A/B) + a preflight line when a side is `source`-only
 and a richer realization is theoretically available. Reuse `renderValidationNote` styling.
 
 ### Step 2 — relabel A/B as "Sandbox A / Sandbox B" (light)
+
 A/B group header becomes Sandbox A / Sandbox B with a `recipe · realization` sub-label. Cements the
 "2 sandboxes" model. Mostly labels on top of Step 1's descriptor.
 
 ### Step 3 — L4b build-from-ref (DEFERRED, flagged epic = old Option C)
+
 Picker gains a "build from ref" opt-in → host realizes a real sandbox (worktree → build → load) →
 ref side climbs the Step-1 matrix to `built`. Needs the resolver `dispose()`/`finally` lifecycle,
 per-agent build recipes, SHA-keyed build cache, timeouts. Subprocess-mode + the L4a RPC-context
 boundary remain the hardest sub-problem (context-dependent agents).
 
 ## Implementation plan — near-term (Steps 1–2)
+
 1. **core/studioRuntimeCore.ts** — add `SideFidelity`/`FidelityReport` types + a pure
    `deriveSideFidelity(methodA, methodB, enriched, wildcardValidation, versionA, versionB)` mapper;
    populate `result.sideFidelity` (and in the error-result path). Unit test the mapper.
@@ -146,4 +170,4 @@ boundary remain the hardest sub-problem (context-dependent agents).
    preflight line + Sandbox A/B relabel. Render test.
 4. **Verify**: core `tsc -b` + jest, studio `tsx --test`, studio-service `npm test`, prettier,
    esbuild. No commit/push without explicit user consent.
-Deferred (Step 3): worktree materializer, scoped build, resolver `dispose()`, ref-side execution.
+   Deferred (Step 3): worktree materializer, scoped build, resolver `dispose()`, ref-side execution.

From 4cac18936db34e1cb2750bbb95362054e1018461 Mon Sep 17 00:00:00 2001
From: Tal Zaccai <talzacc@microsoft.com>
Date: Tue, 30 Jun 2026 14:43:37 -0600
Subject: [PATCH 15/15] =?UTF-8?q?docs(studio):=20correct=20roadmap=20statu?=
 =?UTF-8?q?s=20=E2=80=94=20fidelity=20matrix=20shipped;=20reframe=20corpus?=
 =?UTF-8?q?=20capture=20as=20agent-agnostic?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The per-side fidelity matrix (L4b Step 1) is already built and shipped, so split that backlog row out as done and leave only the optional Sandbox A/B relabel. Reword 'player corpus capture' to the general 'corpus capture' capability (works for any agent); player is only the anchor set Gate C is scored on.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 ts/docs/plans/vscode-devx/STATUS.md | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/ts/docs/plans/vscode-devx/STATUS.md b/ts/docs/plans/vscode-devx/STATUS.md
index 7162c570be..d838b168ed 100644
--- a/ts/docs/plans/vscode-devx/STATUS.md
+++ b/ts/docs/plans/vscode-devx/STATUS.md
@@ -28,7 +28,7 @@ than an in-memory stand-in.
 | Repo-root detection (find `packages/agents`)                   | ✅                                                                                                                                    | ✅ warn toast + status bar                                                       | n/a                                                                                                                           | ✅     |
 | Webview infrastructure (`webviewKit`)                          | ✅ CSP/nonce host + typed protocol                                                                                                    | ✅ singleton-panel host                                                          | —                                                                                                                             | ✅     |
 | Impact Report webview                                          | ✅ `replayCorpus` over channel                                                                                                        | ✅ context header, A/B controls, Grammar/Cache + Validate toggles, durable state | 🟡 grammar + construction-cache + working-tree wildcard validation (L1–L2, L4a)                                               | ✅     |
-| Player corpus capture                                          | ❌                                                                                                                                    | ❌                                                                               | ❌                                                                                                                            | ❌     |
+| Corpus capture (real utterances → actions, any agent)          | ❌                                                                                                                                    | ❌                                                                               | ❌                                                                                                                            | ❌     |
 | Schema Studio                                                  | ❌                                                                                                                                    | ❌                                                                               | ❌                                                                                                                            | ❌     |
 | Live Trace                                                     | ❌                                                                                                                                    | ❌                                                                               | ❌                                                                                                                            | ❌     |
 | `agr-language` / `vscode-shell` refactor onto core             | 🟡 dependency edge only                                                                                                               | —                                                                                | ❌ no behavioral integration                                                                                                  | ❌     |
@@ -53,7 +53,8 @@ is clear why it waits. There is no second roadmap — depth (L4b) and breadth
 | **D** | J5 Debug a trace     | Trace Viewer                                            | P-3   | ❌ not started                                                                   |
 | **E** | J6 Observe live      | Live Trace + status bar                                 | P-5   | ❌ not started                                                                   |
 
-**Critical path now: close Gate C** — player corpus capture → predicate tuning →
+**Critical path now: close Gate C** — corpus capture (the agent-agnostic capture
+path; `player` is just the anchor set the gate is scored on) → predicate tuning →
 run the ≥ 80% validation. It is the only _tunable_ gate and §7's top risk.
 
 ### Backlog (off the critical path — tagged, not a separate plan)
@@ -63,14 +64,15 @@ tags: **depth** = how faithfully one replay side is realized (improves Gate C
 accuracy); **breadth** = how many things are compared at once (multiplies cells);
 **infra** = enabling plumbing.
 
-| Item                                                                      | Parent      | Track              | Precondition                        | Status                    |
-| ------------------------------------------------------------------------- | ----------- | ------------------ | ----------------------------------- | ------------------------- |
-| Fidelity transparency (per-side readout + "Sandbox A/B" relabel)          | Gate C / J4 | depth              | —                                   | next (low-risk, no build) |
-| **L4b** build-from-ref sandboxes (real compiled ref side)                 | Gate C / J4 | depth (P-7)        | Gate C banked                       | deferred flagged epic     |
-| Multi-variant compare (Baseline vs N variants; bisect / first-divergence) | J4          | breadth — versions | Gate C banked; must preserve Gate C | post-MVP                  |
-| Multi-agent / multi-corpus replay (`code`, 684 utterances)                | J4          | breadth — agents   | Gate C banked                       | post-MVP (§2 exclude)     |
-| Active-sandbox selector + per-sandbox scoping                             | —           | infra              | single-sandbox E2E                  | P-7a                      |
-| Sandbox copy-on-write overlay (true sandbox-local A/B)                    | —           | infra              | P-7a                                | P-7b                      |
+| Item                                                                      | Parent      | Track              | Precondition                        | Status                         |
+| ------------------------------------------------------------------------- | ----------- | ------------------ | ----------------------------------- | ------------------------------ |
+| Per-side fidelity matrix (which layers ran on each side + skip reasons)   | Gate C / J4 | depth              | —                                   | ✅ shipped (L4b Step 1)        |
+| "Sandbox A / Sandbox B" relabel of the A/B columns                        | Gate C / J4 | depth              | —                                   | optional cosmetic (L4b Step 2) |
+| **L4b** build-from-ref sandboxes (real compiled ref side)                 | Gate C / J4 | depth (P-7)        | Gate C banked                       | deferred flagged epic (Step 3) |
+| Multi-variant compare (Baseline vs N variants; bisect / first-divergence) | J4          | breadth — versions | Gate C banked; must preserve Gate C | post-MVP                       |
+| Multi-agent / multi-corpus replay (`code`, 684 utterances)                | J4          | breadth — agents   | Gate C banked                       | post-MVP (§2 exclude)          |
+| Active-sandbox selector + per-sandbox scoping                             | —           | infra              | single-sandbox E2E                  | P-7a                           |
+| Sandbox copy-on-write overlay (true sandbox-local A/B)                    | —           | infra              | P-7a                                | P-7b                           |
 
 Why depth and breadth are distinct dials on the same `replayCorpus` (F4.1)
 primitive: **depth** raises the fidelity of each compared _cell_ (and thus Gate C
@@ -218,8 +220,9 @@ Ready to start (smallest → larger):
    Impact Report webview that drives `replayCorpus` over the service channel and
    renders the `ActionDelta[]` contract. The webview never opens a socket
    (webview → extension host → channel → agent runtime).
-3. **Player corpus capture** — wire `vscode-shell` request/feedback IDs into the
-   core corpus.
+3. **Corpus capture** — wire `vscode-shell` request/feedback IDs into the
+   core corpus. Agent-agnostic capture path (works for any agent); `player` is
+   simply the first corpus captured because Gate C is scored on it.
 4. **One real replay path** — one agent, one utterance, working tree vs. HEAD,
    real dispatch; validate the Impact Report `ActionDelta[]` contract (which the
    agent's `ValidateChange` and the webview both consume). **Largely done:** the
@@ -227,7 +230,7 @@ Ready to start (smallest → larger):
    modes, L4a opt-in wildcard validation) is live and validated against the
    contract. The remaining fidelity rung **L4b** (build-from-ref two-version
    sandboxes) is **deferred to P-7** (post-Gate-C). **Live priority is now #3
-   (player corpus capture) → Gate C measurement** — the headline acceptance bar.
+   (corpus capture) → Gate C measurement** — the headline acceptance bar.
 5. **Active-sandbox selector + per-sandbox scoping** (plan phase **P-7a**;
    sequenced **after** the single-sandbox E2E closes) — collisions and corpora are
    intrinsically per-sandbox (a collision is a function of the co-loaded agent