From 5ec4e1a2f8ef89c7e6163506a3f8056bd42e52d3 Mon Sep 17 00:00:00 2001
From: Yash Raj Pandey <yashpn62@gmail.com>
Date: Mon, 12 Oct 2026 15:16:59 -0400
Subject: [PATCH 1/5] Name a run corpus by its content, not by the frozen
 catalog

run_measurement copied the id and revision of the frozen wic-50 v1
catalog into every run, but it hashed the scenarios that actually ran.
A run with --scenarios therefore claimed to be wic-50 v1. Default runs
made the same claim, because the embedded set no longer matches the v1
hash since 964f6e5.

A run now keeps the catalog id and revision only when its corpus hash
equals the catalog hash. Any other corpus records id "custom" and its
hash as the revision. The M7 spec states the rule under T1.2.

The evidence test pinned the old label for a one-scenario corpus. It
now expects "custom". A new test runs the frozen catalog scenarios and
expects wic-50 v1. The CLI test checks the full corpus block of a
--scenarios run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 crates/wic-core/src/runner.rs                 | 14 ++++--
 crates/wic-core/tests/evidence.rs             | 49 +++++++++++++++++--
 crates/willitcall/tests/run_cli.rs            | 16 +++---
 .../2026-08-05-m7-measurement-contract.md     |  5 ++
 4 files changed, 71 insertions(+), 13 deletions(-)

diff --git a/crates/wic-core/src/runner.rs b/crates/wic-core/src/runner.rs
index 3978072..24d3552 100644
--- a/crates/wic-core/src/runner.rs
+++ b/crates/wic-core/src/runner.rs
@@ -457,10 +457,18 @@ pub async fn run_measurement(
     let catalog = load_frozen_v1_catalog().map_err(|error| io::Error::other(error.to_string()))?;
     let scenario_count = u32::try_from(scenarios.len())
         .map_err(|_| io::Error::other("scenario count does not fit in u32"))?;
+    let sha256 = corpus_identity(scenarios);
+    // Only a frozen catalog names a corpus. Any other set, including an edited
+    // embedded set, is named by its hash, so distinct corpora never share a label.
+    let (id, revision) = if sha256 == catalog.sha256 {
+        (catalog.id, catalog.revision)
+    } else {
+        ("custom".to_owned(), sha256.clone())
+    };
     let corpus = CorpusMetadata {
-        id: catalog.id,
-        revision: catalog.revision,
-        sha256: corpus_identity(scenarios),
+        id,
+        revision,
+        sha256,
         scenario_count,
         scoring_version: "v2".to_owned(),
     };
diff --git a/crates/wic-core/tests/evidence.rs b/crates/wic-core/tests/evidence.rs
index 624d5f4..4eb8210 100644
--- a/crates/wic-core/tests/evidence.rs
+++ b/crates/wic-core/tests/evidence.rs
@@ -8,7 +8,7 @@ use reqwest::header::{HeaderValue, AUTHORIZATION};
 use ring::digest::{digest, SHA256};
 use serde_json::{json, Value};
 use support::{MockServer, ScriptedResponse};
-use wic_core::corpus::corpus_identity;
+use wic_core::corpus::{corpus_identity, load_frozen_v1_catalog};
 use wic_core::result::{
     write_result_atomic, DecodeMode, IdentityStatus, ReplicationMetadata, ReplicationMode,
 };
@@ -285,8 +285,8 @@ async fn structured_http_failure_is_captured_at_the_request_boundary() {
         .as_ref()
         .expect("runtime corpus metadata");
 
-    assert_eq!(corpus.id, "wic-50");
-    assert_eq!(corpus.revision, "v1");
+    assert_eq!(corpus.id, "custom");
+    assert_eq!(corpus.revision, expected_corpus_sha256);
     assert_eq!(corpus.sha256, expected_corpus_sha256);
     assert_eq!(corpus.scenario_count, 1);
     assert_eq!(corpus.scoring_version, "v2");
@@ -319,6 +319,49 @@ async fn structured_http_failure_is_captured_at_the_request_boundary() {
     assert_eq!(failure.failed_turn_index, Some(1));
 }
 
+#[tokio::test]
+async fn a_run_of_the_frozen_catalog_keeps_the_catalog_name() {
+    let catalog = load_frozen_v1_catalog().expect("frozen catalog");
+    let server = MockServer::start_scripted(
+        "fixture-model",
+        catalog
+            .scenarios
+            .iter()
+            .map(|_| ScriptedResponse::Status(400, "bad request".to_owned()))
+            .collect(),
+    )
+    .await;
+    let directory = tempfile::tempdir().expect("temp directory");
+    let config = RunConfig::new(
+        server.endpoint(),
+        "fixture-model".to_owned(),
+        Duration::from_secs(5),
+        42,
+        0.0,
+    );
+
+    let measurement = run_measurement(
+        &config,
+        &catalog.scenarios,
+        &directory.path().join("result.json"),
+    )
+    .await
+    .expect("run frozen catalog");
+    let corpus = measurement
+        .metadata
+        .corpus
+        .as_ref()
+        .expect("runtime corpus metadata");
+
+    assert_eq!(corpus.id, "wic-50");
+    assert_eq!(corpus.revision, "v1");
+    assert_eq!(
+        corpus.sha256,
+        "sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"
+    );
+    assert_eq!(corpus.scenario_count, 50);
+}
+
 fn hex(bytes: &[u8]) -> String {
     const DIGITS: &[u8; 16] = b"0123456789abcdef";
     let mut encoded = String::with_capacity(bytes.len() * 2);
diff --git a/crates/willitcall/tests/run_cli.rs b/crates/willitcall/tests/run_cli.rs
index 0f5be6a..c339da3 100644
--- a/crates/willitcall/tests/run_cli.rs
+++ b/crates/willitcall/tests/run_cli.rs
@@ -6,6 +6,7 @@ use std::process::Command;
 
 use serde_json::{json, Value};
 use support::{MockServer, ScriptedResponse};
+use wic_core::corpus::corpus_identity;
 use wic_core::result::{parse_and_validate_measurement, IdentityStatus, Status};
 
 fn completion(calls: Value, content: Value) -> String {
@@ -650,14 +651,15 @@ content = "Reply ready."
     assert_eq!(v3.schema_version, 3);
     assert_eq!(v3.metadata.model.endpoint_id, "qwen2.5:7b-instruct");
     assert_eq!(v3.metadata.model.identity_status, IdentityStatus::Declared);
-    assert_eq!(
-        v3.metadata
-            .corpus
-            .as_ref()
-            .expect("v3 corpus metadata")
-            .scoring_version,
-        "v2"
+    let corpus = v3.metadata.corpus.as_ref().expect("v3 corpus metadata");
+    let custom_sha256 = corpus_identity(
+        &wic_core::load_scenarios_from_dir(&scenario_path).expect("custom scenarios"),
     );
+    assert_eq!(corpus.id, "custom");
+    assert_eq!(corpus.revision, custom_sha256);
+    assert_eq!(corpus.sha256, custom_sha256);
+    assert_eq!(corpus.scenario_count, 1);
+    assert_eq!(corpus.scoring_version, "v2");
     assert!(v3.metadata.replication.is_none());
     assert!(v3.metadata.arm_fingerprint.is_none());
 
diff --git a/docs/superpowers/specs/2026-08-05-m7-measurement-contract.md b/docs/superpowers/specs/2026-08-05-m7-measurement-contract.md
index 99eb989..d24dead 100644
--- a/docs/superpowers/specs/2026-08-05-m7-measurement-contract.md
+++ b/docs/superpowers/specs/2026-08-05-m7-measurement-contract.md
@@ -145,6 +145,11 @@ Model identity alone is not sufficient to identify a comparable arm.
 | `scenarios[].failure` | `{stage, code, http_status, failed_turn_index}`. `"turn 1: server returned HTTP 400"` is not queryable. Keep `failure_reason` for humans. |
 | Unique run id | `runner.rs:566` hashes timestamp, endpoint, and model selector, so two identical runs in the same second collide. Use UUIDv7 or a random nonce. |
 
+A run records the `id` and `revision` of a frozen catalog only when its corpus
+hash equals the catalog hash. Any other corpus, including an edited embedded
+set, records `id` `custom` and its hash as the `revision`. Distinct corpora
+therefore never share an `id` and `revision`.
+
 ### T1.3 Migration of the 32 published files
 
 A deterministic migration command plus an explicit 32-entry mapping manifest.

From 28c9264fe74e9801d0764be00eed3cb488b87e03 Mon Sep 17 00:00:00 2001
From: Yash Raj Pandey <yashpn62@gmail.com>
Date: Mon, 12 Oct 2026 15:22:48 -0400
Subject: [PATCH 2/5] Check every result against the schema of its declared
 version

validate deserialized a result with serde and checked only the totals.
Serde accepts values that the published schema rejects, so validate
printed "valid" for a failure_class of null, a failure.http_status of 0,
or a preflight_override with forced false. The same gap existed for v1
and v2: a v2 failure_class of "bogus" and a v1 run_id passed too.

The parser now checks the raw document against the embedded schema for
its version, after serde so the serde messages stay the same. The check
covers validate, site, annotate and rescore, because all of them parse
through it. The hand-written v2 property check is removed, because the
v2 schema covers it. jsonschema moves from a dev-dependency to a
dependency. Cargo.lock does not change.

annotate now refuses a v1 result. The v1 schema has no cause field, so
the edited file always failed its own schema.

All 187 tracked result files still validate, and the generated site is
byte-identical. The release binary grows from 7.9 MB to 12.3 MB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 CONTRIBUTING.md                    |   3 +-
 README.md                          |   6 +-
 crates/wic-core/Cargo.toml         |   2 +-
 crates/wic-core/src/result.rs      |  83 ++++++++++-----------
 crates/willitcall/src/main.rs      |   6 ++
 crates/willitcall/tests/run_cli.rs | 113 +++++++++++++++++++++++++++++
 6 files changed, 165 insertions(+), 48 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c4545a2..f274d4f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -79,7 +79,8 @@ record_id = "rec-17"
    filename.
 3. Run `willitcall validate results/<file>.json`; new runs must pass as schema v3
    against `schemas/result-v3.schema.json`. The CLI continues to accept schema v1
-   and v2 files without upgrading them during `annotate` or `rescore`.
+   and v2 files without upgrading them during `rescore`, and v2 files during
+   `annotate`.
 4. Open a pull request that adds the file and its `evidence/` directory under `results/`. State the hardware and the server version.
 
 Never hand-edit a result file. Each scenario record carries an evidence hash, so edited results are not comparable.
diff --git a/README.md b/README.md
index ae3b998..ebc4124 100644
--- a/README.md
+++ b/README.md
@@ -60,8 +60,10 @@ cargo run -p willitcall -- validate willitcall-result.json
 New runs use result schema v3 (`schemas/result-v3.schema.json`). The CLI
 resolves the model selector through `registry/models-v1.json` and embeds the
 resolved model and artifact metadata in the result. Legacy schema v1 and v2
-files are still valid input to `validate`, `site`, `annotate`, and `rescore`.
-Editing a legacy file keeps its original schema version.
+files are still valid input to `validate`, `site`, and `rescore`, and v2 files
+to `annotate`. The v1 schema has no `cause` field. Each file must pass the
+schema of its own version. Editing a legacy file keeps its original schema
+version.
 
 `--server` selects a preset (`llamacpp`, `ollama`, `mlx-lm`, `lmstudio`,
 `vllm`, `custom`). The preset only supplies request defaults. The result file
diff --git a/crates/wic-core/Cargo.toml b/crates/wic-core/Cargo.toml
index c0fbee5..b6d3f78 100644
--- a/crates/wic-core/Cargo.toml
+++ b/crates/wic-core/Cargo.toml
@@ -5,6 +5,7 @@ edition.workspace = true
 
 [dependencies]
 include_dir.workspace = true
+jsonschema.workspace = true
 reqwest.workspace = true
 ring.workspace = true
 serde.workspace = true
@@ -16,7 +17,6 @@ unicode-normalization = "0.1"
 
 [dev-dependencies]
 axum.workspace = true
-jsonschema.workspace = true
 
 [lints]
 workspace = true
diff --git a/crates/wic-core/src/result.rs b/crates/wic-core/src/result.rs
index 39fe929..b6d0ad7 100644
--- a/crates/wic-core/src/result.rs
+++ b/crates/wic-core/src/result.rs
@@ -10,6 +10,9 @@ use serde_json::Value;
 use crate::ScenarioCategory;
 
 pub const RESULT_SCHEMA_VERSION: u32 = 2;
+const RESULT_V1_SCHEMA: &str = include_str!("../../../schemas/result-v1.schema.json");
+const RESULT_V2_SCHEMA: &str = include_str!("../../../schemas/result-v2.schema.json");
+const RESULT_V3_SCHEMA: &str = include_str!("../../../schemas/result-v3.schema.json");
 
 #[derive(Clone, Debug, Deserialize, Serialize)]
 #[serde(deny_unknown_fields)]
@@ -639,30 +642,17 @@ pub fn parse_and_validate_result(bytes: &[u8]) -> Result<RunResult, String> {
             "unsupported schema_version {schema_version}; expected 1 or {RESULT_SCHEMA_VERSION}"
         ));
     }
-    let result: RunResultV1V2 = serde_json::from_value(document.clone())
-        .map_err(|error| format!("invalid result document: {error}"))?;
-    validate_result(&result)?;
-    if schema_version == 2 {
-        validate_v2_required_properties(&document)?;
-    }
-    Ok(result)
+    parse_legacy_result(&document, schema_version)
 }
 
 pub fn parse_and_validate_measurement(bytes: &[u8]) -> Result<Measurement, String> {
     let (document, schema_version) = inspect_result_document(bytes)?;
     let measurement = match schema_version {
-        1 | 2 => {
-            let result: RunResultV1V2 = serde_json::from_value(document.clone())
-                .map_err(|error| format!("invalid result document: {error}"))?;
-            validate_result(&result)?;
-            if schema_version == 2 {
-                validate_v2_required_properties(&document)?;
-            }
-            Measurement::from(result)
-        }
+        1 | 2 => Measurement::from(parse_legacy_result(&document, schema_version)?),
         3 => {
-            let result: RunResultV3 = serde_json::from_value(document)
+            let result = RunResultV3::deserialize(&document)
                 .map_err(|error| format!("invalid result document: {error}"))?;
+            validate_against_schema(&document, RESULT_V3_SCHEMA)?;
             let measurement = Measurement::from(result);
             validate_measurement(&measurement)?;
             measurement
@@ -689,31 +679,30 @@ fn inspect_result_document(bytes: &[u8]) -> Result<(Value, u32), String> {
     Ok((document, schema_version))
 }
 
-fn validate_v2_required_properties(document: &Value) -> Result<(), String> {
-    let metadata = document
-        .get("metadata")
-        .and_then(Value::as_object)
-        .ok_or_else(|| "invalid result document: metadata must be an object".to_owned())?;
-    if !metadata.contains_key("run_id") {
-        return Err(
-            "invalid result document: metadata.run_id is required for schema_version 2".to_owned(),
-        );
-    }
-    let scenarios = document
-        .get("scenarios")
-        .and_then(Value::as_array)
-        .ok_or_else(|| "invalid result document: scenarios must be an array".to_owned())?;
-    if scenarios.iter().any(|scenario| {
-        !scenario
-            .as_object()
-            .is_some_and(|scenario| scenario.contains_key("evidence_path"))
-    }) {
-        return Err(
-            "invalid result document: scenario evidence_path is required for schema_version 2"
-                .to_owned(),
-        );
-    }
-    Ok(())
+fn parse_legacy_result(document: &Value, schema_version: u32) -> Result<RunResultV1V2, String> {
+    let result = RunResultV1V2::deserialize(document)
+        .map_err(|error| format!("invalid result document: {error}"))?;
+    let schema = if schema_version == 1 {
+        RESULT_V1_SCHEMA
+    } else {
+        RESULT_V2_SCHEMA
+    };
+    validate_against_schema(document, schema)?;
+    validate_result(&result)?;
+    Ok(result)
+}
+
+// Serde runs first so its messages stay the same. The schema then rejects what
+// serde lets through, such as a null where the schema allows only an enum value.
+fn validate_against_schema(document: &Value, schema: &str) -> Result<(), String> {
+    let schema: Value = serde_json::from_str(schema).expect("checked-in result schema is JSON");
+    let validator = jsonschema::validator_for(&schema).expect("checked-in result schema compiles");
+    validator.validate(document).map_err(|error| {
+        format!(
+            "invalid result document: at '{}': {error}",
+            error.instance_path()
+        )
+    })
 }
 
 pub fn validate_result(result: &RunResult) -> Result<(), String> {
@@ -1535,7 +1524,10 @@ mod tests {
             &serde_json::to_vec(&missing_run_id).expect("encode result"),
         )
         .expect_err("v2 run_id is required");
-        assert!(error.contains("metadata.run_id is required"), "{error}");
+        assert_eq!(
+            error,
+            "invalid result document: at '/metadata': \"run_id\" is a required property"
+        );
 
         let mut missing_evidence_path =
             serde_json::to_value(sample_result()).expect("serialize result");
@@ -1547,7 +1539,10 @@ mod tests {
             &serde_json::to_vec(&missing_evidence_path).expect("encode result"),
         )
         .expect_err("v2 evidence_path is required");
-        assert!(error.contains("evidence_path is required"), "{error}");
+        assert_eq!(
+            error,
+            "invalid result document: at '/scenarios/0': \"evidence_path\" is a required property"
+        );
     }
 
     #[test]
diff --git a/crates/willitcall/src/main.rs b/crates/willitcall/src/main.rs
index 3d1149b..1bf7574 100644
--- a/crates/willitcall/src/main.rs
+++ b/crates/willitcall/src/main.rs
@@ -359,6 +359,12 @@ fn write_updated_result(path: &Path, result: &EditableResult) -> Result<(), Exec
 
 fn annotate(args: AnnotateArgs) -> Result<usize, ExecuteError> {
     let mut result = read_result(&args.result)?;
+    if result.measurement.schema_version == 1 {
+        return Err(ExecuteError::Usage(
+            "annotate needs a schema_version 2 or 3 result; the v1 schema has no cause field"
+                .to_owned(),
+        ));
+    }
     let cause = Cause {
         kind: args.cause.into(),
         reference: args.reference,
diff --git a/crates/willitcall/tests/run_cli.rs b/crates/willitcall/tests/run_cli.rs
index c339da3..18700d6 100644
--- a/crates/willitcall/tests/run_cli.rs
+++ b/crates/willitcall/tests/run_cli.rs
@@ -1169,6 +1169,119 @@ async fn validate_rejects_a_corrupt_document_with_a_nonzero_exit() {
     );
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn validate_rejects_what_the_published_schema_rejects() {
+    type Mutation = fn(&mut Value);
+    let directory = tempfile::tempdir().expect("temp directory");
+    let cases: [(u32, &str, Mutation, &str); 5] = [
+        (
+            3,
+            "v3-failure-class-null",
+            |document| document["scenarios"][0]["failure_class"] = Value::Null,
+            "at '/scenarios/0/failure_class': null is not one of \"empty_response\" or \"unparsed_tool_call\"",
+        ),
+        (
+            3,
+            "v3-override-not-forced",
+            |document| {
+                document["metadata"]["preflight_override"] =
+                    json!({"forced": false, "foreign_endpoints": []});
+            },
+            "at '/metadata/preflight_override/forced': true was expected",
+        ),
+        (
+            3,
+            "v3-http-status-zero",
+            |document| {
+                document["scenarios"][0]["failure"] = json!({
+                    "stage": "request",
+                    "code": "http_error",
+                    "http_status": 0,
+                    "failed_turn_index": 1
+                });
+            },
+            "at '/scenarios/0/failure': {\"code\":\"http_error\",\"failed_turn_index\":1,\"http_status\":0,\"stage\":\"request\"} is not valid under any of the schemas listed in the 'anyOf' keyword",
+        ),
+        (
+            2,
+            "v2-unknown-failure-class",
+            |document| document["scenarios"][0]["failure_class"] = json!("bogus"),
+            "at '/scenarios/0/failure_class': \"bogus\" is not one of \"empty_response\" or \"unparsed_tool_call\"",
+        ),
+        (
+            1,
+            "v1-run-id",
+            |document| document["metadata"]["run_id"] = json!("20260720T120000Z-fixture"),
+            "at '/metadata': Additional properties are not allowed ('run_id' was unexpected)",
+        ),
+    ];
+
+    for (schema_version, name, mutate, expected) in cases {
+        let path = directory.path().join(format!("{name}.json"));
+        let scenarios = vec![scenario_fixture(
+            "schema-case",
+            None,
+            Some("evidence/schema-case.json"),
+        )];
+        if schema_version == 3 {
+            write_v3_result_fixture(&path, scenarios);
+        } else {
+            write_result_fixture(&path, schema_version, scenarios);
+        }
+        let valid = run_binary(vec!["validate".to_owned(), path.display().to_string()]).await;
+        assert_eq!(valid.status.code(), Some(0), "{name} before the edit");
+
+        let mut document: Value =
+            serde_json::from_slice(&fs::read(&path).expect("result bytes")).expect("result JSON");
+        mutate(&mut document);
+        fs::write(
+            &path,
+            serde_json::to_vec_pretty(&document).expect("encode result"),
+        )
+        .expect("write result");
+        let output = run_binary(vec!["validate".to_owned(), path.display().to_string()]).await;
+
+        assert_eq!(output.status.code(), Some(2), "{name}");
+        assert!(output.stdout.is_empty(), "{name}");
+        assert_eq!(
+            String::from_utf8_lossy(&output.stderr),
+            format!("error: invalid result document: {expected}\n"),
+            "{name}"
+        );
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn annotate_refuses_a_v1_result_and_leaves_it_unchanged() {
+    let directory = tempfile::tempdir().expect("temp directory");
+    let result_path = directory.path().join("result.json");
+    write_result_fixture(
+        &result_path,
+        1,
+        vec![scenario_fixture("legacy", None, None)],
+    );
+    let before = fs::read(&result_path).expect("result bytes");
+
+    let output = run_binary(vec![
+        "annotate".to_owned(),
+        "--result".to_owned(),
+        result_path.display().to_string(),
+        "--scenario".to_owned(),
+        "legacy".to_owned(),
+        "--cause".to_owned(),
+        "unknown".to_owned(),
+        "--force".to_owned(),
+    ])
+    .await;
+
+    assert_eq!(output.status.code(), Some(2));
+    assert_eq!(
+        String::from_utf8_lossy(&output.stderr),
+        "error: annotate needs a schema_version 2 or 3 result; the v1 schema has no cause field\n"
+    );
+    assert_eq!(fs::read(result_path).expect("result bytes"), before);
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn known_broken_double_encoded_arguments_are_red_with_a_precise_reason() {
     let encoded_object = serde_json::to_string(&json!({"city": "Boston"})).expect("encode object");

From b020834e4609fda6f6b4f085e08b4b9c3fbc4b19 Mon Sep 17 00:00:00 2001
From: Yash Raj Pandey <yashpn62@gmail.com>
Date: Mon, 12 Oct 2026 15:26:16 -0400
Subject: [PATCH 3/5] Refuse local paths in evidence_path and cause.reference

The parser accepted any string in evidence_path. The site then wrote it
verbatim into results.json and results.csv, and linked it from the
index and the appendix. A result with /Users/alice/... therefore
published the author's home directory. annotate --reference had the
same defect: it accepted /Users/alice/notes.md, and validate and site
both passed the edited file.

validate_result and validate_measurement now check both fields. Every
parse runs one of them, and every write runs them first. An
evidence_path must be a relative path under evidence/. A cause
reference must be an http(s) URL or a relative repository path. Each
path segment must be non-empty, must not start with '.', and may use
only ASCII letters, digits, '.', '_' and '-'. The check does not use
std::path, because a Unix host does not see C:\ as absolute.

All 187 tracked result files pass. One site fixture had a leading '/'
on its evidence path. It is fixed, and the site golden does not change.
The leading '/' trims in evidence_url and reference_url are removed,
because no such value reaches them now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 crates/wic-core/src/result.rs                 | 139 +++++++++++++++++-
 crates/willitcall/src/site/html.rs            |  10 +-
 .../02-v2-unresolved.json                     |   2 +-
 crates/willitcall/tests/run_cli.rs            |  35 +++++
 crates/willitcall/tests/site_cli.rs           |  37 +++++
 5 files changed, 212 insertions(+), 11 deletions(-)

diff --git a/crates/wic-core/src/result.rs b/crates/wic-core/src/result.rs
index b6d0ad7..67e7966 100644
--- a/crates/wic-core/src/result.rs
+++ b/crates/wic-core/src/result.rs
@@ -716,7 +716,14 @@ pub fn validate_result(result: &RunResult) -> Result<(), String> {
         &result.totals,
         result.scenarios.iter().map(|outcome| outcome.status),
         result.scenarios.len(),
-    )
+    )?;
+    validate_scenario_paths(result.scenarios.iter().map(|outcome| {
+        (
+            outcome.id.as_str(),
+            outcome.evidence_path.as_deref(),
+            outcome.cause.as_ref(),
+        )
+    }))
 }
 
 pub fn validate_measurement(measurement: &Measurement) -> Result<(), String> {
@@ -730,7 +737,58 @@ pub fn validate_measurement(measurement: &Measurement) -> Result<(), String> {
         &measurement.totals,
         measurement.scenarios.iter().map(|outcome| outcome.status),
         measurement.scenarios.len(),
-    )
+    )?;
+    validate_scenario_paths(measurement.scenarios.iter().map(|outcome| {
+        (
+            outcome.id.as_str(),
+            outcome.evidence_path.as_deref(),
+            outcome.cause.as_ref(),
+        )
+    }))
+}
+
+// The site publishes both fields as links and in its JSON and CSV exports, so a
+// local path in either one leaks the author's file system.
+fn validate_scenario_paths<'a>(
+    outcomes: impl Iterator<Item = (&'a str, Option<&'a str>, Option<&'a Cause>)>,
+) -> Result<(), String> {
+    for (id, evidence_path, cause) in outcomes {
+        if let Some(path) = evidence_path.filter(|path| !is_safe_evidence_path(path)) {
+            return Err(format!(
+                "scenario {id:?} evidence_path {path:?} is not a relative path under evidence/"
+            ));
+        }
+        let reference = cause.and_then(|cause| cause.reference.as_deref());
+        if let Some(reference) = reference.filter(|reference| !is_safe_reference(reference)) {
+            return Err(format!(
+                "scenario {id:?} cause reference {reference:?} is not a web URL or a relative repository path"
+            ));
+        }
+    }
+    Ok(())
+}
+
+fn is_safe_evidence_path(path: &str) -> bool {
+    path.strip_prefix("evidence/")
+        .is_some_and(is_safe_relative_path)
+}
+
+fn is_safe_reference(reference: &str) -> bool {
+    reference.starts_with("https://")
+        || reference.starts_with("http://")
+        || is_safe_relative_path(reference)
+}
+
+// Platform independent on purpose: a result written on Windows is checked on
+// Unix, where std::path does not see C:\ or \\host as absolute.
+fn is_safe_relative_path(path: &str) -> bool {
+    path.split('/').all(|part| {
+        !part.is_empty()
+            && !part.starts_with('.')
+            && part
+                .bytes()
+                .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
+    })
 }
 
 fn validate_totals(
@@ -1545,6 +1603,83 @@ mod tests {
         );
     }
 
+    #[test]
+    fn validators_reject_local_and_traversing_paths() {
+        let unsafe_evidence_paths = [
+            "/Users/alice/run/single-weather.json",
+            "C:\\Users\\alice\\run.json",
+            "C:/Users/alice/run.json",
+            "~/run.json",
+            "file:///Users/alice/run.json",
+            "Users/alice/run.json",
+            "evidence/../../Users/alice/run.json",
+            "evidence/..\\..\\run.json",
+            "evidence/%2e%2e/run.json",
+            "evidence//run.json",
+            "evidence/",
+            "",
+        ];
+        for path in unsafe_evidence_paths {
+            let mut result = sample_result();
+            result.scenarios[0].evidence_path = Some(path.to_owned());
+            let expected = format!(
+                "scenario \"single-weather\" evidence_path {path:?} is not a relative path under evidence/"
+            );
+            assert_eq!(super::validate_result(&result), Err(expected.clone()));
+            assert_eq!(
+                super::validate_measurement(&super::Measurement::from(result)),
+                Err(expected)
+            );
+        }
+
+        let unsafe_references = [
+            "/Users/alice/notes.md",
+            "C:\\Users\\alice\\notes.md",
+            "~/notes.md",
+            "file:///Users/alice/notes.md",
+            "../notes.md",
+            "docs/../../notes.md",
+        ];
+        for reference in unsafe_references {
+            let mut result = sample_result();
+            result.scenarios[0].cause = Some(Cause {
+                kind: CauseKind::Unknown,
+                reference: Some(reference.to_owned()),
+                note: None,
+            });
+            let expected = format!(
+                "scenario \"single-weather\" cause reference {reference:?} is not a web URL or a relative repository path"
+            );
+            assert_eq!(super::validate_result(&result), Err(expected.clone()));
+            assert_eq!(
+                super::validate_measurement(&super::Measurement::from(result)),
+                Err(expected)
+            );
+        }
+
+        for (evidence_path, reference) in [
+            (Some("evidence/empty.json"), None),
+            (None, Some("https://example.test/issue/1")),
+            (
+                None,
+                Some("docs/case-studies/2026-07-19-ollama-drops-valid-tool-calls.md"),
+            ),
+        ] {
+            let mut result = sample_result();
+            result.scenarios[0].evidence_path = evidence_path.map(str::to_owned);
+            result.scenarios[0].cause = Some(Cause {
+                kind: CauseKind::Unknown,
+                reference: reference.map(str::to_owned),
+                note: None,
+            });
+            assert_eq!(super::validate_result(&result), Ok(()));
+            assert_eq!(
+                super::validate_measurement(&super::Measurement::from(result)),
+                Ok(())
+            );
+        }
+    }
+
     #[test]
     fn redaction_covers_sensitive_headers_query_parameters_and_raw_bytes() {
         let turn = super::redact_transcript_turn(
diff --git a/crates/willitcall/src/site/html.rs b/crates/willitcall/src/site/html.rs
index 59aa1f0..2cffd13 100644
--- a/crates/willitcall/src/site/html.rs
+++ b/crates/willitcall/src/site/html.rs
@@ -1327,20 +1327,14 @@ fn display_status(status: Status) -> &'static str {
 }
 
 fn evidence_url(repo_base: &str, evidence_path: &str) -> String {
-    format!(
-        "{repo_base}/blob/main/results/{}",
-        evidence_path.trim_start_matches('/')
-    )
+    format!("{repo_base}/blob/main/results/{evidence_path}")
 }
 
 fn reference_url(repo_base: &str, reference: &str) -> String {
     if reference.starts_with("https://") || reference.starts_with("http://") {
         reference.to_owned()
     } else {
-        format!(
-            "{repo_base}/blob/main/{}",
-            reference.trim_start_matches('/')
-        )
+        format!("{repo_base}/blob/main/{reference}")
     }
 }
 
diff --git a/crates/willitcall/tests/fixtures/site-contract-results/02-v2-unresolved.json b/crates/willitcall/tests/fixtures/site-contract-results/02-v2-unresolved.json
index 60f15d7..a4688d1 100644
--- a/crates/willitcall/tests/fixtures/site-contract-results/02-v2-unresolved.json
+++ b/crates/willitcall/tests/fixtures/site-contract-results/02-v2-unresolved.json
@@ -45,7 +45,7 @@
         "note": "fixture cause & note"
       },
       "evidence_hash": "sha256:v2-fail-fixture",
-      "evidence_path": "/evidence/site-contract/v2-fail.json",
+      "evidence_path": "evidence/site-contract/v2-fail.json",
       "retried": false
     },
     {
diff --git a/crates/willitcall/tests/run_cli.rs b/crates/willitcall/tests/run_cli.rs
index 18700d6..76950d0 100644
--- a/crates/willitcall/tests/run_cli.rs
+++ b/crates/willitcall/tests/run_cli.rs
@@ -277,6 +277,41 @@ async fn annotate_scenario_adds_a_cause() {
     assert_eq!(result["scenarios"][0]["cause"]["note"], "known server bug");
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn annotate_refuses_a_local_reference_and_leaves_the_result_unchanged() {
+    let directory = tempfile::tempdir().expect("temp directory");
+    let result_path = directory.path().join("result.json");
+    write_v3_result_fixture(
+        &result_path,
+        vec![scenario_fixture(
+            "empty",
+            Some("empty_response"),
+            Some("evidence/empty.json"),
+        )],
+    );
+    let before = fs::read(&result_path).expect("result bytes");
+
+    let output = run_binary(vec![
+        "annotate".to_owned(),
+        "--result".to_owned(),
+        result_path.display().to_string(),
+        "--scenario".to_owned(),
+        "empty".to_owned(),
+        "--cause".to_owned(),
+        "server-defect".to_owned(),
+        "--reference".to_owned(),
+        "/Users/alice/private/notes.md".to_owned(),
+    ])
+    .await;
+
+    assert_eq!(output.status.code(), Some(2));
+    assert_eq!(
+        String::from_utf8_lossy(&output.stderr),
+        "error: scenario \"empty\" cause reference \"/Users/alice/private/notes.md\" is not a web URL or a relative repository path\n"
+    );
+    assert_eq!(fs::read(result_path).expect("result bytes"), before);
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn annotate_refuses_a_non_empty_scenario_without_force() {
     let directory = tempfile::tempdir().expect("temp directory");
diff --git a/crates/willitcall/tests/site_cli.rs b/crates/willitcall/tests/site_cli.rs
index d40cd63..fd836fc 100644
--- a/crates/willitcall/tests/site_cli.rs
+++ b/crates/willitcall/tests/site_cli.rs
@@ -537,6 +537,43 @@ fn site_never_renders_absolute_model_paths() {
     assert!(index.contains("mlx-community/Qwen3-8B-4bit"));
 }
 
+#[test]
+fn site_refuses_a_local_evidence_path_before_writing_any_asset() {
+    let directory = tempfile::tempdir().expect("temp directory");
+    let results = directory.path().join("results");
+    let output = directory.path().join("site");
+    fs::create_dir(&results).expect("results directory");
+    write_result(
+        &results.join("leaky.json"),
+        2,
+        "fixture-model",
+        None,
+        "llamacpp",
+        None,
+        vec![scenario(
+            "single-bad",
+            "single_call",
+            "fail",
+            Some("bad"),
+            Some("/Users/someone/run/single-bad.json"),
+        )],
+    );
+
+    let generated = run_site(&results, &output, None);
+
+    assert_eq!(generated.status.code(), Some(4));
+    let stderr = String::from_utf8_lossy(&generated.stderr);
+    assert!(
+        stderr.contains(
+            "scenario \"single-bad\" evidence_path \"/Users/someone/run/single-bad.json\" is not a relative path under evidence/"
+        ),
+        "{stderr}"
+    );
+    for asset in ["index.html", "appendix.html", "results.json", "results.csv"] {
+        assert!(!output.join(asset).exists(), "{asset} was written");
+    }
+}
+
 #[test]
 fn site_uses_registry_identity_and_excludes_unresolved_rows_from_grouping() {
     let directory = tempfile::tempdir().expect("temp directory");

From 0a48e31f6423edf80a4dc4b3b295647a6731f8c6 Mon Sep 17 00:00:00 2001
From: Yash Raj Pandey <yashpn62@gmail.com>
Date: Mon, 12 Oct 2026 15:43:20 -0400
Subject: [PATCH 4/5] Keep run output inside the schema that every reader
 checks

An endpoint can answer with a status from 600 to 999. The runner
recorded that status in failure.http_status, but the v3 schema allows
100 to 599 only. Since the readers check the schema, run wrote a file
that validate, site, annotate and rescore then refused.

RFC 9110 defines status codes 100 to 599 only. The runner now records
http_status only in that range. The failure code stays http_error, and
the evidence keeps the raw status.

Both result writers now check the serialized document the way the
readers do, before they write it. A writer and the schema can no longer
drift apart without a loud error.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 crates/wic-core/src/result.rs      | 29 +++++++++--
 crates/wic-core/src/runner.rs      |  4 +-
 crates/willitcall/src/main.rs      | 28 +++++++++++
 crates/willitcall/tests/run_cli.rs | 81 ++++++++++++++++++++++++++++++
 4 files changed, 137 insertions(+), 5 deletions(-)

diff --git a/crates/wic-core/src/result.rs b/crates/wic-core/src/result.rs
index 67e7966..7ab9a57 100644
--- a/crates/wic-core/src/result.rs
+++ b/crates/wic-core/src/result.rs
@@ -836,10 +836,12 @@ fn validate_totals(
 }
 
 pub fn write_result_atomic(path: &Path, result: &RunResult) -> io::Result<()> {
-    atomic_write_with(path, |file| {
-        serde_json::to_writer_pretty(&mut *file, result).map_err(io::Error::other)?;
-        file.write_all(b"\n")
-    })
+    let mut bytes = serde_json::to_vec_pretty(result).map_err(io::Error::other)?;
+    bytes.push(b'\n');
+    // Check the document as every reader does, so no file is written that
+    // validate, site or annotate then refuses.
+    parse_and_validate_result(&bytes).map_err(io::Error::other)?;
+    atomic_write_with(path, |file| file.write_all(&bytes))
 }
 
 pub(crate) fn write_transcript_atomic(path: &Path, transcript: &Transcript) -> io::Result<Vec<u8>> {
@@ -1085,6 +1087,25 @@ mod tests {
         assert_eq!(super::exit_code_for_totals(&totals), 4);
     }
 
+    #[test]
+    fn atomic_write_refuses_a_document_its_schema_rejects() {
+        let directory = tempfile::tempdir().expect("temp directory");
+        let destination = directory.path().join("result.json");
+        let mut result = sample_result();
+        result.scenarios[0].failure_class = Some("bogus".to_owned());
+
+        let error = write_result_atomic(&destination, &result).expect_err("schema must refuse");
+
+        assert!(
+            error
+                .to_string()
+                .starts_with("invalid result document: at '/scenarios/0/failure_class': "),
+            "{error}"
+        );
+        assert!(!destination.exists());
+        assert_eq!(fs::read_dir(directory.path()).expect("list").count(), 0);
+    }
+
     #[test]
     fn atomic_write_publishes_a_complete_result() {
         let directory = tempfile::tempdir().expect("temp directory");
diff --git a/crates/wic-core/src/runner.rs b/crates/wic-core/src/runner.rs
index 24d3552..c1db14c 100644
--- a/crates/wic-core/src/runner.rs
+++ b/crates/wic-core/src/runner.rs
@@ -741,7 +741,9 @@ fn scenario_failure(
     ScenarioFailure {
         stage: stage.to_owned(),
         code: code.to_owned(),
-        http_status,
+        // RFC 9110 defines status codes 100 to 599 only, and the result schema
+        // records no other value. The evidence keeps the raw status.
+        http_status: http_status.filter(|status| (100..=599).contains(status)),
         failed_turn_index: u32::try_from(turn_index + 1).ok(),
     }
 }
diff --git a/crates/willitcall/src/main.rs b/crates/willitcall/src/main.rs
index 1bf7574..5a5c0f3 100644
--- a/crates/willitcall/src/main.rs
+++ b/crates/willitcall/src/main.rs
@@ -310,6 +310,9 @@ fn read_result(path: &Path) -> Result<EditableResult, ExecuteError> {
 fn write_v3_result_atomic(path: &Path, result: &RunResultV3) -> std::io::Result<Vec<u8>> {
     let mut bytes = serde_json::to_vec_pretty(result).map_err(std::io::Error::other)?;
     bytes.push(b'\n');
+    // Check the document as every reader does, so no file is written that
+    // validate, site or annotate then refuses.
+    parse_and_validate_measurement(&bytes).map_err(std::io::Error::other)?;
     let parent = match path.parent() {
         Some(parent) if !parent.as_os_str().is_empty() => parent,
         _ => Path::new("."),
@@ -832,6 +835,31 @@ mod tests {
         execute_with_known_servers, resolve_endpoint, Cli, Command, ExecuteError, ServerPreset,
     };
 
+    #[test]
+    fn v3_write_refuses_a_document_its_schema_rejects() {
+        let mut result: wic_core::result::RunResultV3 = serde_json::from_str(include_str!(
+            "../tests/fixtures/site-contract-results/03-v3-declared.json"
+        ))
+        .expect("v3 fixture");
+        result.scenarios[0].failure_class = Some("bogus".to_owned());
+        let directory = tempfile::tempdir().expect("temp directory");
+        let destination = directory.path().join("result.json");
+
+        let error =
+            super::write_v3_result_atomic(&destination, &result).expect_err("schema must refuse");
+
+        assert!(
+            error
+                .to_string()
+                .starts_with("invalid result document: at '/scenarios/0/failure_class': "),
+            "{error}"
+        );
+        assert_eq!(
+            std::fs::read_dir(directory.path()).expect("list").count(),
+            0
+        );
+    }
+
     #[test]
     fn run_subcommand_parses_m0_arguments() {
         let cli = Cli::try_parse_from([
diff --git a/crates/willitcall/tests/run_cli.rs b/crates/willitcall/tests/run_cli.rs
index 76950d0..6607de5 100644
--- a/crates/willitcall/tests/run_cli.rs
+++ b/crates/willitcall/tests/run_cli.rs
@@ -1204,6 +1204,87 @@ async fn validate_rejects_a_corrupt_document_with_a_nonzero_exit() {
     );
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn a_status_outside_rfc_9110_still_writes_a_result_validate_accepts() {
+    let server = MockServer::start_scripted(
+        "fixture-model",
+        vec![ScriptedResponse::Status(600, "not http".to_owned())],
+    )
+    .await;
+    let directory = tempfile::tempdir().expect("temp directory");
+    let scenario_path = directory.path().join("scenarios");
+    fs::create_dir(&scenario_path).expect("scenario directory");
+    fs::write(
+        scenario_path.join("status-600.toml"),
+        r#"
+id = "status-600"
+category = "negative_trap"
+description = "Produce no tool call."
+rationale = "This fixture asserts how a status outside 100 to 599 is recorded."
+
+[[tools]]
+name = "get_weather"
+description = "Get weather."
+
+[tools.parameters]
+type = "object"
+
+[tool_choice]
+mode = "none"
+
+[[turns]]
+[[turns.messages]]
+role = "user"
+content = "Reply ready."
+"#,
+    )
+    .expect("write scenario");
+    let result_path = directory.path().join("result.json");
+
+    let run = run_binary(vec![
+        "run".to_owned(),
+        "--endpoint".to_owned(),
+        server.endpoint(),
+        "--model".to_owned(),
+        "fixture-model".to_owned(),
+        "--force".to_owned(),
+        "--scenarios".to_owned(),
+        scenario_path.display().to_string(),
+        "--out".to_owned(),
+        result_path.display().to_string(),
+    ])
+    .await;
+
+    assert_eq!(
+        run.status.code(),
+        Some(4),
+        "{}",
+        String::from_utf8_lossy(&run.stderr)
+    );
+    let result: Value =
+        serde_json::from_slice(&fs::read(&result_path).expect("result bytes")).expect("json");
+    assert_eq!(
+        result["scenarios"][0]["failure"],
+        json!({
+            "stage": "request",
+            "code": "http_error",
+            "http_status": null,
+            "failed_turn_index": 1
+        })
+    );
+    let validate = run_binary(vec![
+        "validate".to_owned(),
+        result_path.display().to_string(),
+    ])
+    .await;
+    assert_eq!(
+        validate.status.code(),
+        Some(0),
+        "{}",
+        String::from_utf8_lossy(&validate.stderr)
+    );
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn validate_rejects_what_the_published_schema_rejects() {
     type Mutation = fn(&mut Value);

From b6ebce79d5d669c0f5b5f06bc95c0266b25c1863 Mon Sep 17 00:00:00 2001
From: Yash Raj Pandey <yashpn62@gmail.com>
Date: Mon, 12 Oct 2026 15:43:20 -0400
Subject: [PATCH 5/5] Accept a fragment after a repository path in
 cause.reference

The reference check applied the path segment rule to the whole value.
It refused a case-study link with a section anchor, such as
docs/case-studies/x.md#section, and a line anchor such as
runner.rs#L770. The base accepted both and rendered working links.

The check now splits off one '#' fragment. The path part keeps the
segment rule. The fragment allows ASCII letters, digits, '.', '_' and
'-' only, so it cannot carry a local path.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 crates/wic-core/src/result.rs | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/crates/wic-core/src/result.rs b/crates/wic-core/src/result.rs
index 7ab9a57..f92f98d 100644
--- a/crates/wic-core/src/result.rs
+++ b/crates/wic-core/src/result.rs
@@ -774,9 +774,15 @@ fn is_safe_evidence_path(path: &str) -> bool {
 }
 
 fn is_safe_reference(reference: &str) -> bool {
-    reference.starts_with("https://")
-        || reference.starts_with("http://")
-        || is_safe_relative_path(reference)
+    if reference.starts_with("https://") || reference.starts_with("http://") {
+        return true;
+    }
+    // A fragment names a section or a line, so it may follow a repository path.
+    let (path, fragment) = reference.split_once('#').unwrap_or((reference, ""));
+    is_safe_relative_path(path)
+        && fragment
+            .bytes()
+            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
 }
 
 // Platform independent on purpose: a result written on Windows is checked on
@@ -1660,6 +1666,9 @@ mod tests {
             "file:///Users/alice/notes.md",
             "../notes.md",
             "docs/../../notes.md",
+            "/Users/alice/notes.md#intro",
+            "docs/notes.md#/Users/alice/notes.md",
+            "docs/notes.md#a#b",
         ];
         for reference in unsafe_references {
             let mut result = sample_result();
@@ -1685,6 +1694,11 @@ mod tests {
                 None,
                 Some("docs/case-studies/2026-07-19-ollama-drops-valid-tool-calls.md"),
             ),
+            (
+                None,
+                Some("docs/case-studies/2026-07-19-ollama-drops-valid-tool-calls.md#what-remains-a-genuine-ollama-defect"),
+            ),
+            (None, Some("crates/wic-core/src/runner.rs#L770")),
         ] {
             let mut result = sample_result();
             result.scenarios[0].evidence_path = evidence_path.map(str::to_owned);