From 6370bdeb97907d5af801328df4752bbb3dbd50ba Mon Sep 17 00:00:00 2001
From: Yoshitaka Inoue <8393063+inoue0426@users.noreply.github.com>
Date: Mon, 12 Oct 2026 00:29:26 -0400
Subject: [PATCH 1/9] Benchmark claim retrieval and abstention on 100
 source-derived questions

---
 .github/workflows/ci.yml                      |   1 +
 .../workflows/claim-abstention-benchmark.yml  |  79 +++++++
 README.md                                     |   2 +
 docs/CLAIM_BENCHMARK.md                       |  69 ++++++
 docs/CLAIM_EVIDENCE.md                        |   6 +
 scripts/benchmark_claim_abstention.cjs        | 207 ++++++++++++++++++
 tests/test_claim_benchmark.cjs                |  61 ++++++
 7 files changed, 425 insertions(+)
 create mode 100644 .github/workflows/claim-abstention-benchmark.yml
 create mode 100644 docs/CLAIM_BENCHMARK.md
 create mode 100644 scripts/benchmark_claim_abstention.cjs
 create mode 100644 tests/test_claim_benchmark.cjs

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ccec423..0af52ce 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -18,6 +18,7 @@ jobs:
       - run: node tests/test_variant_comparison.cjs
       - run: node tests/test_evidence_tree.cjs
       - run: node tests/test_claim_evidence.cjs
+      - run: node tests/test_claim_benchmark.cjs
       - run: node tests/test_ask_evidence.cjs
       - run: node tests/test_cloudflare_ask.cjs
       - run: node tests/test_variant_question_evaluation.cjs
diff --git a/.github/workflows/claim-abstention-benchmark.yml b/.github/workflows/claim-abstention-benchmark.yml
new file mode 100644
index 0000000..611cd08
--- /dev/null
+++ b/.github/workflows/claim-abstention-benchmark.yml
@@ -0,0 +1,79 @@
+name: Claim abstention benchmark
+
+on:
+  pull_request:
+    paths:
+      - 'scripts/benchmark_claim_abstention.cjs'
+      - 'tests/test_claim_benchmark.cjs'
+      - 'web/claim-evidence.js'
+      - '.github/workflows/claim-abstention-benchmark.yml'
+  push:
+    branches: [main]
+    paths:
+      - 'scripts/benchmark_claim_abstention.cjs'
+      - 'web/claim-evidence.js'
+      - '.github/workflows/claim-abstention-benchmark.yml'
+  workflow_dispatch:
+  workflow_run:
+    workflows: ['Refresh data']
+    types: [completed]
+    branches: [main]
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  benchmark:
+    if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+      - name: Download the published-snapshot refresh artifact
+        id: refresh
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          run_id=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow refresh-data.yml \
+            --status success --limit 1 --json databaseId --jq '.[0].databaseId')
+          if [ -z "$run_id" ] || [ "$run_id" = "null" ]; then
+            echo '::error::No successful refresh with a search-index artifact exists.'
+            exit 1
+          fi
+          echo "run_id=$run_id" >> "$GITHUB_OUTPUT"
+      - uses: actions/download-artifact@v4
+        with:
+          name: search-index
+          path: _data-refresh
+          run-id: ${{ steps.refresh.outputs.run_id }}
+          github-token: ${{ github.token }}
+      - name: Run 100 source-consistency questions and three methods
+        run: |
+          node tests/test_claim_benchmark.cjs
+          node scripts/benchmark_claim_abstention.cjs \
+            --snapshot _data-refresh --questions 100 --out _claim-benchmark
+      - name: Summarize method and limitations
+        run: |
+          node -e '
+            const fs = require("node:fs");
+            const r = JSON.parse(fs.readFileSync("_claim-benchmark/report.json", "utf8"));
+            if (r.question_count !== 100) process.exit(1);
+            const pct = (x) => x == null ? "n/a" : (100 * x).toFixed(1) + "%";
+            const lines = ["## 100 source-derived claim questions", "",
+              "**Not independent biomedical gold or an LLM experiment.** Labels are derived from accepted CIViC fields in this same frozen snapshot; the result tests source consistency only.", "",
+              "Snapshot hashes: `" + JSON.stringify(r.snapshot_sha256) + "`", "",
+              "Question types: `" + JSON.stringify(r.categories) + "`; truncated: " + r.truncated_count, "",
+              "| Method | Answered | Answer rate on source-answerable | False answers on source-unanswerable | Error among answered |",
+              "| --- | ---: | ---: | ---: | ---: |"];
+            for (const [key, m] of Object.entries(r.metrics)) lines.push(`| ${key} | ${m.answered_count}/${m.evaluated_count} | ${pct(m.answer_rate_on_answerable)} | ${pct(m.false_answer_rate_on_unanswerable)} | ${pct(m.selective_error_among_answers)} |`);
+            fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join("\n") + "\n");
+          '
+      - uses: actions/upload-artifact@v4
+        with:
+          name: claim-abstention-benchmark-report
+          path: _claim-benchmark/report.json
+          retention-days: 30
diff --git a/README.md b/README.md
index 796c2cf..8ae653e 100644
--- a/README.md
+++ b/README.md
@@ -61,6 +61,8 @@ The [variant-conditioned retrieval experiment](docs/VARIANT_RETRIEVAL_EXPERIMENT
 
 The [variant claim check](docs/CLAIM_EVIDENCE.md) additionally filters those records by exact therapy and reported sensitivity or resistance. It keeps supporting, opposing, disputing and insufficient-context records separate in the browser, local API and MCP. This research-only rule set does not validate LLM prose or establish treatment efficacy.
 
+The [100-question CI benchmark](docs/CLAIM_BENCHMARK.md) runs three retrieval strategies on the same frozen Pages artifact and publishes a detailed report. Its automatically generated labels test source consistency, **not** independently established biomedical accuracy.
+
 **Explore by cancer name** groups disease entities currently in the public index using explicit matches on their names. Open a subtype to inspect its direct relations and, when available, accepted CIViC molecular-profile evidence. The groups are a browsing shortcut, not a complete ontology or a claim that all matching diseases are equivalent.
 
 CIViC records retain their molecular profile, named variants, HGVS expression when available, evidence type, level, trust rating, significance, review status, source record, and support/contradiction direction. A profile can contain multiple variants; its clinical evidence does not necessarily apply to each member. Filter to accepted level A/B records rated at least 4/5 by CIViC to focus on stronger study-level evidence; this does not guarantee a positive result or clinical actionability. OncoKB and COSMIC are outbound lookups only; their content is not imported or graded here, and COSMIC may require login.
diff --git a/docs/CLAIM_BENCHMARK.md b/docs/CLAIM_BENCHMARK.md
new file mode 100644
index 0000000..bf61811
--- /dev/null
+++ b/docs/CLAIM_BENCHMARK.md
@@ -0,0 +1,69 @@
+# Controlled 100-question claim and abstention benchmark
+
+## Purpose and scope
+
+GitHub Actions runs `scripts/benchmark_claim_abstention.cjs` against the latest
+successful `Refresh data` **search-index artifact**, the same source used by
+Pages. This is a **source-consistency stress test** for the question "Do
+accepted CIViC records report sensitivity or resistance for gene–variant,
+disease and therapy?" It is **not** a test of clinical truth, independent
+expert adjudication, recommendation quality, or LLM-generated answer quality.
+
+The script deterministically constructs exactly **100 distinct questions**
+from accepted predictive CIViC records with exported variant membership. It
+adds counterfactual conditions (another variant, therapy, disease, opposite
+effect) and, where available, complex molecular profiles. It checks every
+counterfactual against all eligible source records before assigning its
+source-derived label, so a swapped condition with genuine matching evidence
+is not mistakenly treated as empty. Target quotas are 25 direct, 20 other
+variant, 20 other therapy, 15 other cancer, 10 opposite effect and 10 complex
+profile questions. If a category has insufficient records, other categories
+fill the set; the **actual** mixture appears in the report. If fewer than 100
+unique questions exist, CI fails rather than duplicating questions.
+
+The source reference enumerates raw accepted predictive CIViC records by exact
+gene, single-variant membership, disease and therapy; a record has a matching
+reported effect only when the direction is `SUPPORTS`. A reference question
+is marked source-answerable if it has matching support and no supporting
+report for the opposite effect or `DOES_NOT_SUPPORT` report of that effect.
+This label is derived from the **same CIViC snapshot** as all methods and
+is not an independent biomedical gold standard. Conflict and missing evidence
+mean abstain **within this snapshot**, not absence of clinical benefit.
+
+## Strategies
+
+| Method | Answer condition | Deliberate limitation |
+| --- | --- | --- |
+| `context_blind` | Any accepted CIViC record for the gene, drug and disease | Ignores variant, direction and significance |
+| `variant_only` | Any accepted CIViC record for the gene, variant/profile, drug and disease | Ignores predictive significance, direction and complex membership |
+| `claim_scoped` | The structured claim check returns `supporting_records` | Requires exact reported effect and single-variant profile; abstains on opposing/disputing reports and incomplete input |
+
+The baselines are **ablations of OncoGraph's own retrieval**, not standalone
+CIViC MCP, an external clinical service or an LLM. The source reference is
+field-defined, so the rules of the proposed method align with it by design:
+high scores only show consistency with the frozen source fields. In particular,
+they cannot demonstrate a novel clinical conclusion or a performance gain on
+an independently labeled question set.
+
+Metrics on the 100 questions: answer rate among source-answerable items;
+false-answer rate among source-unanswerable items; answered count/coverage;
+and error among answered. A tree with more than 200 matching records is flagged
+and excluded from metrics for **all methods**, while remaining in the detailed
+question list. There is no claim that a zero error rate generalizes outside
+this bounded snapshot.
+
+## Reproduce and inspect
+
+```sh
+node tests/test_claim_benchmark.cjs
+node scripts/benchmark_claim_abstention.cjs --snapshot web/data --questions 100 --out claim-benchmark
+```
+
+`report.json` includes all 100 query texts, question types, source-derived
+labels, EIDs per method, coverage, abstention counts and SHA-256 hashes of the
+two input JSON files. CI publishes a summary and the report as an Actions
+artifact named `claim-abstention-benchmark-report`; the benchmark workflow
+also runs on pull requests, manually, and after successful data refreshes.
+To evaluate scientific novelty, independently review a held-out set of
+question/claim/source tuples, evaluate claim-level citations and free-text
+answers, and compare with appropriately configured CIViC MCP and LLM baselines.
diff --git a/docs/CLAIM_EVIDENCE.md b/docs/CLAIM_EVIDENCE.md
index d17700e..6d81eb3 100644
--- a/docs/CLAIM_EVIDENCE.md
+++ b/docs/CLAIM_EVIDENCE.md
@@ -53,6 +53,12 @@ produce `context_only` for profile-name matches.
 
 ## Evaluation before publication claims
 
+An automated [100-question benchmark](CLAIM_BENCHMARK.md) runs in GitHub
+Actions after a data refresh or on a change to the benchmark. It compares
+context-blind, variant-only and claim-scoped retrieval on the **same** frozen
+snapshot. Its source-derived labels test internal consistency, not medical
+accuracy, model-written answers or independent expert agreement.
+
 This is a **rule-based baseline**, not a demonstrated improvement or a
 validated algorithm. The current [variant retrieval experiment](VARIANT_RETRIEVAL_EXPERIMENT.md)
 evaluates Evidence Item retrieval, not this claim classifier or LLM responses.
diff --git a/scripts/benchmark_claim_abstention.cjs b/scripts/benchmark_claim_abstention.cjs
new file mode 100644
index 0000000..9c77058
--- /dev/null
+++ b/scripts/benchmark_claim_abstention.cjs
@@ -0,0 +1,207 @@
+#!/usr/bin/env node
+// A 100-question source-consistency stress test, NOT independent clinical gold.
+const fs = require("node:fs");
+const path = require("node:path");
+const crypto = require("node:crypto");
+const { buildEvidenceTree } = require("../web/evidence-tree.js");
+const { evaluateEvidenceClaim } = require("../web/claim-evidence.js");
+const { contextBlindIds } = require("./evaluate_variant_questions.cjs");
+
+const normal = (value) => String(value || "").trim().toLowerCase();
+function context(row) {
+  if (row.context && typeof row.context === "object") return row.context;
+  try { return JSON.parse(row.context || "{}"); } catch { return {}; }
+}
+function memberName(member, gene) {
+  const value = String(member).trim();
+  return normal(value).startsWith(`${normal(gene)} `) ? value.slice(gene.length + 1) : value;
+}
+function key(q) {
+  return [q.gene, q.variant, q.disease, q.therapy, q.effect].map(normal).join("|");
+}
+
+// Independently enumerate upstream-shaped CIViC records (before calling either
+// retrieval method). This reference is still source-derived, not expert gold.
+function readSourceRecords(entities, relations) {
+  const byId = new Map(entities.map((entity) => [String(entity.id), entity]));
+  const rows = [];
+  for (const relation of relations) {
+    if (relation.predicate !== "clinically_evidenced_for") continue;
+    const drug = byId.get(String(relation.subject_id));
+    const cancer = byId.get(String(relation.object_id));
+    if (normal(drug?.type) !== "drug" || normal(cancer?.type) !== "disease") continue;
+    for (const item of relation.evidence || []) {
+      const ctx = context(item);
+      if (normal(item.source) !== "civic" || ctx.civic_review_status !== "ACCEPTED" ||
+          normal(ctx.civic_evidence_type) !== "predictive" ||
+          !["sensitivity", "resistance"].includes(normal(ctx.significance)) ||
+          !ctx.civic_evidence_id || !ctx.gene_name || !Array.isArray(ctx.profile_variants)) continue;
+      rows.push({ id: String(ctx.civic_evidence_id), gene: String(ctx.gene_name),
+        members: ctx.profile_variants.map((member) => memberName(member, ctx.gene_name)),
+        disease: cancer.name, therapy: drug.name, effect: normal(ctx.significance),
+        direction: normal(item.claim_state) });
+    }
+  }
+  return { rows, byId };
+}
+
+function referenceIndex(records) {
+  const index = new Map();
+  for (const row of records) {
+    if (row.members.length !== 1) continue;
+    const base = [row.gene, row.members[0], row.disease, row.therapy].map(normal).join("|");
+    if (!index.has(base)) index.set(base, []);
+    index.get(base).push(row);
+  }
+  return index;
+}
+function sourceReference(q, recordsOrIndex) {
+  const supporting = new Set(), opposing = new Set(), disputed = new Set();
+  const index = recordsOrIndex instanceof Map ? recordsOrIndex : referenceIndex(recordsOrIndex);
+  const base = [q.gene, q.variant, q.disease, q.therapy].map(normal).join("|");
+  for (const row of index.get(base) || []) {
+    if (row.direction === "supports" && row.effect === q.effect) supporting.add(row.id);
+    else if (row.direction === "supports") opposing.add(row.id);
+    else if (row.direction === "contradicts" && row.effect === q.effect) disputed.add(row.id);
+  }
+  return { answerable: supporting.size > 0 && !opposing.size && !disputed.size,
+    supporting_eids: [...supporting].sort(), opposing_count: opposing.size, disputed_count: disputed.size };
+}
+
+function makeQuestions(records, count = 100) {
+  if (!Number.isInteger(count) || count < 1 || count > 1000) throw Error("Invalid question count");
+  const ordered = records.filter((row) => row.members.length === 1 && row.direction === "supports")
+    .sort((a, b) => [a.gene, a.disease, a.therapy, a.members[0], a.effect, a.id].join("|")
+      .localeCompare([b.gene, b.disease, b.therapy, b.members[0], b.effect, b.id].join("|")));
+  if (!ordered.length) throw Error("No eligible accepted CIViC predictive records in snapshot");
+  const genes = new Map();
+  for (const row of ordered) {
+    if (!genes.has(row.gene)) genes.set(row.gene, []);
+    genes.get(row.gene).push(row);
+  }
+  const seed = [];
+  while (seed.length < 300 && [...genes.values()].some((list) => list.length)) {
+    for (const list of genes.values()) if (list.length && seed.length < 300) seed.push(list.shift());
+  }
+  const index = referenceIndex(records);
+  const variantsByGene = new Map();
+  for (const row of records) for (const member of row.members) {
+    if (!variantsByGene.has(row.gene)) variantsByGene.set(row.gene, new Set());
+    variantsByGene.get(row.gene).add(member);
+  }
+  const therapies = [...new Set(records.map((row) => row.therapy))].sort();
+  const cancers = [...new Set(records.map((row) => row.disease))].sort();
+  const pools = Object.fromEntries(["direct", "other_variant", "other_therapy", "other_cancer",
+    "opposite_effect", "complex_profile"].map((name) => [name, new Map()]));
+  const add = (category, q, expected) => {
+    if (pools[category].size >= 250) return;
+    const reference = sourceReference(q, index);
+    if (reference.answerable !== expected) return;
+    const id = key(q);
+    if (!pools[category].has(id)) pools[category].set(id, { ...q, category, reference });
+  };
+  // These are perturbations of real source rows. Negatives are checked
+  // against the full available source record set, never presumed absent.
+  for (const row of seed) {
+    const base = { gene: row.gene, variant: row.members[0], disease: row.disease,
+      therapy: row.therapy, effect: row.effect };
+    add("direct", base, true);
+    add("opposite_effect", { ...base, effect: row.effect === "sensitivity" ? "resistance" : "sensitivity" }, false);
+    for (const variant of [...(variantsByGene.get(row.gene) || [])].slice(0, 15)) {
+      if (normal(variant) !== normal(base.variant)) add("other_variant", { ...base, variant }, false);
+    }
+    for (const therapy of therapies.slice(0, 20)) {
+      if (normal(therapy) !== normal(base.therapy)) add("other_therapy", { ...base, therapy }, false);
+    }
+    for (const disease of cancers.slice(0, 20)) {
+      if (normal(disease) !== normal(base.disease)) add("other_cancer", { ...base, disease }, false);
+    }
+  }
+  for (const row of records.filter((item) => item.members.length > 1)) {
+    for (const variant of row.members) {
+      add("complex_profile", { gene: row.gene, variant, disease: row.disease,
+        therapy: row.therapy, effect: row.effect }, false);
+    }
+  }
+  const quotas = { direct: 25, other_variant: 20, other_therapy: 20,
+    other_cancer: 15, opposite_effect: 10, complex_profile: 10 };
+  const selected = new Map();
+  for (const [category, size] of Object.entries(quotas)) {
+    for (const [id, q] of [...pools[category]].slice(0, size)) if (!selected.has(id)) selected.set(id, q);
+  }
+  // Rare source distributions can lack a category. Fill the remainder
+  // deterministically and report actual category counts instead of faking it.
+  for (const category of Object.keys(pools)) {
+    for (const [id, q] of pools[category]) {
+      if (selected.size >= count) break;
+      if (!selected.has(id)) selected.set(id, q);
+    }
+  }
+  if (selected.size < count) throw Error(`Only ${selected.size} unique eligible questions; need ${count}`);
+  return [...selected.values()].slice(0, count).map((q, i) => ({ id: `claim-${String(i + 1).padStart(3, "0")}`,
+    question: `In ${q.disease}, do accepted CIViC records report ${q.effect} for ${q.gene} ${q.variant} with ${q.therapy}?`, ...q }));
+}
+
+function metrics(results, strategy) {
+  const valid = results.filter((row) => !row.tree_truncated);
+  const positive = valid.filter((row) => row.reference.answerable);
+  const negative = valid.filter((row) => !row.reference.answerable);
+  const saidYes = valid.filter((row) => row.predictions[strategy].answers);
+  const trueYes = positive.filter((row) => row.predictions[strategy].answers);
+  const falseYes = negative.filter((row) => row.predictions[strategy].answers);
+  return { evaluated_count: valid.length, source_answerable_count: positive.length,
+    source_unanswerable_count: negative.length, answered_count: saidYes.length,
+    answer_coverage: valid.length ? saidYes.length / valid.length : null,
+    answer_rate_on_answerable: positive.length ? trueYes.length / positive.length : null,
+    false_answer_rate_on_unanswerable: negative.length ? falseYes.length / negative.length : null,
+    selective_error_among_answers: saidYes.length ? falseYes.length / saidYes.length : null };
+}
+
+function evaluate(entities, relations, count = 100) {
+  const { rows: source, byId } = readSourceRecords(entities, relations);
+  const questions = makeQuestions(source, count);
+  const results = questions.map((q) => {
+    const tree = buildEvidenceTree(entities, relations, { gene: q.gene,
+      variant: q.variant, disease: q.disease, limit: 200 });
+    const claim = evaluateEvidenceClaim(tree, { therapy: q.therapy, effect: q.effect, limit: 200 });
+    const blindIds = contextBlindIds(q, relations, byId);
+    const scopedIds = tree.evidence.filter((item) => item.therapies.some((drug) =>
+      normal(drug.name) === normal(q.therapy))).map((item) => item.evidence_id);
+    return { id: q.id, category: q.category, question: q.question,
+      query: { gene: q.gene, variant: q.variant, disease: q.disease,
+        therapy: q.therapy, effect: q.effect },
+      reference: q.reference, tree_truncated: tree.truncated,
+      predictions: { context_blind: { answers: blindIds.length > 0, evidence_ids: blindIds },
+        variant_only: { answers: scopedIds.length > 0 && !tree.truncated, evidence_ids: scopedIds },
+        claim_scoped: { answers: claim.status === "supporting_records", status: claim.status,
+          evidence_ids: claim.evidence.supporting.map((item) => item.evidence_id) } } };
+  });
+  return { methodology: "source-derived controlled perturbation; not independent clinical gold or LLM evaluation",
+    question_count: results.length,
+    categories: Object.fromEntries(Object.keys({ direct: 1, other_variant: 1, other_therapy: 1,
+      other_cancer: 1, opposite_effect: 1, complex_profile: 1 }).map((category) =>
+      [category, results.filter((row) => row.category === category).length])),
+    truncated_count: results.filter((row) => row.tree_truncated).length,
+    metrics: Object.fromEntries(["context_blind", "variant_only", "claim_scoped"].map((strategy) =>
+      [strategy, metrics(results, strategy)])), questions: results };
+}
+
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const value = (key, fallback) => args.includes(key) ? args[args.indexOf(key) + 1] : fallback;
+  const dir = value("--snapshot", "web/data");
+  const out = value("--out", "claim-benchmark");
+  const count = Number(value("--questions", "100"));
+  const names = ["search-index.json", "relations.json"];
+  const raw = names.map((name) => fs.readFileSync(path.join(dir, name)));
+  const report = evaluate(...raw.map((item) => JSON.parse(item.toString("utf8"))), count);
+  report.snapshot_sha256 = Object.fromEntries(names.map((name, i) =>
+    [name, crypto.createHash("sha256").update(raw[i]).digest("hex")]));
+  fs.mkdirSync(out, { recursive: true });
+  fs.writeFileSync(path.join(out, "report.json"), JSON.stringify(report, null, 2) + "\n");
+  process.stdout.write(JSON.stringify({ question_count: report.question_count,
+    categories: report.categories, truncated_count: report.truncated_count,
+    metrics: report.metrics, snapshot_sha256: report.snapshot_sha256 }, null, 2) + "\n");
+}
+
+module.exports = { evaluate, makeQuestions, readSourceRecords, sourceReference };
diff --git a/tests/test_claim_benchmark.cjs b/tests/test_claim_benchmark.cjs
new file mode 100644
index 0000000..2b59828
--- /dev/null
+++ b/tests/test_claim_benchmark.cjs
@@ -0,0 +1,61 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const { evaluate, makeQuestions, readSourceRecords } = require("../scripts/benchmark_claim_abstention.cjs");
+
+function fixture() {
+  const entities = [];
+  const relations = [];
+  let eid = 1;
+  for (let gene = 0; gene < 3; gene++) {
+    for (let variant = 0; variant < 6; variant++) {
+      for (let drug = 0; drug < 3; drug++) {
+        const drugId = `drug-${drug}`;
+        const cancerId = `cancer-${gene}`;
+        if (!entities.some((item) => item.id === drugId)) entities.push({ id: drugId, type: "drug", name: `Drug ${drug}` });
+        if (!entities.some((item) => item.id === cancerId)) entities.push({ id: cancerId,
+          type: "disease", name: `Cancer ${gene}`, canonical_id: `doid:${gene}` });
+        const member = `GENE${gene} V${variant}`;
+        relations.push({ subject_id: drugId, object_id: cancerId, predicate: "clinically_evidenced_for",
+          evidence: [{ source: "civic", claim_state: "SUPPORTS", source_id: `EID${eid}`,
+            context: { civic_review_status: "ACCEPTED", civic_evidence_id: eid,
+              molecular_profile_id: eid, molecular_profile: member, profile_variants: [member],
+              gene_name: `GENE${gene}`, civic_evidence_type: "PREDICTIVE",
+              significance: variant % 2 ? "RESISTANCE" : "SENSITIVITY" } }] });
+        eid++;
+      }
+    }
+  }
+  entities.push({ id: "complex-drug", type: "drug", name: "Combo-only" });
+  for (let i = 0; i < 12; i++) {
+    const gene = i % 3;
+    relations.push({ subject_id: "complex-drug", object_id: `cancer-${gene}`,
+      predicate: "clinically_evidenced_for", evidence: [{ source: "civic", claim_state: "SUPPORTS",
+        context: { civic_review_status: "ACCEPTED", civic_evidence_id: eid,
+          molecular_profile_id: eid, molecular_profile: `GENE${gene} V${i} + V${i + 1}`,
+          profile_variants: [`GENE${gene} V${i}`, `GENE${gene} V${i + 1}`],
+          gene_name: `GENE${gene}`, civic_evidence_type: "PREDICTIVE", significance: "SENSITIVITY" } }] });
+    eid++;
+  }
+  return { entities, relations };
+}
+
+test("100 unique snapshot-derived questions compare two ablations and safe abstention", () => {
+  const { entities, relations } = fixture();
+  const source = readSourceRecords(entities, relations).rows;
+  assert.equal(makeQuestions(source).length, 100);
+  const report = evaluate(entities, relations);
+  assert.equal(new Set(report.questions.map((row) => JSON.stringify(row.query))).size, 100);
+  assert(report.categories.direct > 0);
+  assert(report.categories.other_variant > 0);
+  assert(report.categories.complex_profile > 0);
+  assert(report.metrics.context_blind.false_answer_rate_on_unanswerable > 0);
+  assert(report.metrics.variant_only.false_answer_rate_on_unanswerable > 0);
+  assert.equal(report.metrics.claim_scoped.false_answer_rate_on_unanswerable, 0);
+  assert.equal(report.metrics.claim_scoped.answer_rate_on_answerable, 1);
+  assert.equal(report.truncated_count, 0);
+});
+
+test("fails rather than reporting 100 questions if there are too few unique source perturbations", () => {
+  const { entities, relations } = fixture();
+  assert.throws(() => evaluate(entities, relations, 1000), /unique eligible questions/);
+});

From 733102a28ecf18995ccbeae51a9c0b95a613ab03 Mon Sep 17 00:00:00 2001
From: Yoshitaka Inoue <8393063+inoue0426@users.noreply.github.com>
Date: Mon, 12 Oct 2026 00:31:42 -0400
Subject: [PATCH 2/9] Expose CI benchmark failures and report artifacts

---
 .../workflows/claim-abstention-benchmark.yml  | 23 ++++++++++++++++---
 docs/CLAIM_BENCHMARK.md                       |  3 ++-
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/claim-abstention-benchmark.yml b/.github/workflows/claim-abstention-benchmark.yml
index 611cd08..87e1730 100644
--- a/.github/workflows/claim-abstention-benchmark.yml
+++ b/.github/workflows/claim-abstention-benchmark.yml
@@ -55,7 +55,21 @@ jobs:
         run: |
           node tests/test_claim_benchmark.cjs
           node scripts/benchmark_claim_abstention.cjs \
-            --snapshot _data-refresh --questions 100 --out _claim-benchmark
+            --snapshot _data-refresh --questions 100 --out _claim-benchmark \
+            2>&1 | tee _benchmark-console.txt
+      - name: Label the result artifact for inspection
+        id: label
+        if: always()
+        run: |
+          node -e '
+            const fs = require("node:fs");
+            let label = "report";
+            if (!fs.existsSync("_claim-benchmark/report.json")) {
+              const log = fs.existsSync("_benchmark-console.txt") ? fs.readFileSync("_benchmark-console.txt", "utf8") : "Missing benchmark output";
+              label = "error-" + (log.match(/(?:Error|TypeError): ([^\n]+)/)?.[1] || "unknown").replace(/[^a-zA-Z0-9]+/g, "-").slice(0, 80);
+            }
+            fs.appendFileSync(process.env.GITHUB_OUTPUT, `label=${label}\n`);
+          '
       - name: Summarize method and limitations
         run: |
           node -e '
@@ -73,7 +87,10 @@ jobs:
             fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join("\n") + "\n");
           '
       - uses: actions/upload-artifact@v4
+        if: always()
         with:
-          name: claim-abstention-benchmark-report
-          path: _claim-benchmark/report.json
+          name: claim-abstention-benchmark-${{ steps.label.outputs.label }}
+          path: |
+            _claim-benchmark/report.json
+            _benchmark-console.txt
           retention-days: 30
diff --git a/docs/CLAIM_BENCHMARK.md b/docs/CLAIM_BENCHMARK.md
index bf61811..d6c455d 100644
--- a/docs/CLAIM_BENCHMARK.md
+++ b/docs/CLAIM_BENCHMARK.md
@@ -62,7 +62,8 @@ node scripts/benchmark_claim_abstention.cjs --snapshot web/data --questions 100
 `report.json` includes all 100 query texts, question types, source-derived
 labels, EIDs per method, coverage, abstention counts and SHA-256 hashes of the
 two input JSON files. CI publishes a summary and the report as an Actions
-artifact named `claim-abstention-benchmark-report`; the benchmark workflow
+artifact named `claim-abstention-benchmark-report`; failed runs instead put a
+brief error hint in the artifact name and upload the console log. The workflow
 also runs on pull requests, manually, and after successful data refreshes.
 To evaluate scientific novelty, independently review a held-out set of
 question/claim/source tuples, evaluate claim-level citations and free-text

From 88aa2b725846c0f4557485377a57265141f58c29 Mon Sep 17 00:00:00 2001
From: Yoshitaka Inoue <8393063+inoue0426@users.noreply.github.com>
Date: Mon, 12 Oct 2026 00:33:00 -0400
Subject: [PATCH 3/9] Report public snapshot coverage before claim benchmark

---
 scripts/benchmark_claim_abstention.cjs | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/scripts/benchmark_claim_abstention.cjs b/scripts/benchmark_claim_abstention.cjs
index 9c77058..47fe6d2 100644
--- a/scripts/benchmark_claim_abstention.cjs
+++ b/scripts/benchmark_claim_abstention.cjs
@@ -25,24 +25,34 @@ function key(q) {
 function readSourceRecords(entities, relations) {
   const byId = new Map(entities.map((entity) => [String(entity.id), entity]));
   const rows = [];
+  const inspected = { clinical_relations: 0, drug_disease_relations: 0, civic_items: 0,
+    accepted_items: 0, predictive_items: 0, valid_effect_items: 0, variant_member_items: 0 };
   for (const relation of relations) {
     if (relation.predicate !== "clinically_evidenced_for") continue;
+    inspected.clinical_relations++;
     const drug = byId.get(String(relation.subject_id));
     const cancer = byId.get(String(relation.object_id));
     if (normal(drug?.type) !== "drug" || normal(cancer?.type) !== "disease") continue;
+    inspected.drug_disease_relations++;
     for (const item of relation.evidence || []) {
       const ctx = context(item);
-      if (normal(item.source) !== "civic" || ctx.civic_review_status !== "ACCEPTED" ||
-          normal(ctx.civic_evidence_type) !== "predictive" ||
-          !["sensitivity", "resistance"].includes(normal(ctx.significance)) ||
-          !ctx.civic_evidence_id || !ctx.gene_name || !Array.isArray(ctx.profile_variants)) continue;
+      if (normal(item.source) !== "civic") continue;
+      inspected.civic_items++;
+      if (ctx.civic_review_status !== "ACCEPTED") continue;
+      inspected.accepted_items++;
+      if (normal(ctx.civic_evidence_type) !== "predictive") continue;
+      inspected.predictive_items++;
+      if (!["sensitivity", "resistance"].includes(normal(ctx.significance))) continue;
+      inspected.valid_effect_items++;
+      if (!ctx.civic_evidence_id || !ctx.gene_name || !Array.isArray(ctx.profile_variants)) continue;
+      inspected.variant_member_items++;
       rows.push({ id: String(ctx.civic_evidence_id), gene: String(ctx.gene_name),
         members: ctx.profile_variants.map((member) => memberName(member, ctx.gene_name)),
         disease: cancer.name, therapy: drug.name, effect: normal(ctx.significance),
         direction: normal(item.claim_state) });
     }
   }
-  return { rows, byId };
+  return { rows, byId, inspected };
 }
 
 function referenceIndex(records) {
@@ -158,7 +168,8 @@ function metrics(results, strategy) {
 }
 
 function evaluate(entities, relations, count = 100) {
-  const { rows: source, byId } = readSourceRecords(entities, relations);
+  const { rows: source, byId, inspected } = readSourceRecords(entities, relations);
+  if (!source.length) throw Error(`No eligible source records: ${JSON.stringify(inspected)}`);
   const questions = makeQuestions(source, count);
   const results = questions.map((q) => {
     const tree = buildEvidenceTree(entities, relations, { gene: q.gene,

From 6e3d8fdd975b086963ff12f21b4ac6089f8fbf07 Mon Sep 17 00:00:00 2001
From: Yoshitaka Inoue <8393063+inoue0426@users.noreply.github.com>
Date: Mon, 12 Oct 2026 00:33:53 -0400
Subject: [PATCH 4/9] Clarify CIViC eligibility diagnostics

---
 scripts/benchmark_claim_abstention.cjs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/benchmark_claim_abstention.cjs b/scripts/benchmark_claim_abstention.cjs
index 47fe6d2..8e72975 100644
--- a/scripts/benchmark_claim_abstention.cjs
+++ b/scripts/benchmark_claim_abstention.cjs
@@ -169,7 +169,7 @@ function metrics(results, strategy) {
 
 function evaluate(entities, relations, count = 100) {
   const { rows: source, byId, inspected } = readSourceRecords(entities, relations);
-  if (!source.length) throw Error(`No eligible source records: ${JSON.stringify(inspected)}`);
+  if (!source.length) throw Error(`No eligible source records: civic=${inspected.civic_items} accepted=${inspected.accepted_items} predictive=${inspected.predictive_items} effect=${inspected.valid_effect_items} members=${inspected.variant_member_items} clinical=${inspected.clinical_relations}`);
   const questions = makeQuestions(source, count);
   const results = questions.map((q) => {
     const tree = buildEvidenceTree(entities, relations, { gene: q.gene,

From 3d05ad60f858ae993dde0894a34458f84abae26c Mon Sep 17 00:00:00 2001
From: Yoshitaka Inoue <8393063+inoue0426@users.noreply.github.com>
Date: Mon, 12 Oct 2026 00:35:18 -0400
Subject: [PATCH 5/9] Compact real-data eligibility summary

---
 scripts/benchmark_claim_abstention.cjs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/benchmark_claim_abstention.cjs b/scripts/benchmark_claim_abstention.cjs
index 8e72975..2be7546 100644
--- a/scripts/benchmark_claim_abstention.cjs
+++ b/scripts/benchmark_claim_abstention.cjs
@@ -169,7 +169,7 @@ function metrics(results, strategy) {
 
 function evaluate(entities, relations, count = 100) {
   const { rows: source, byId, inspected } = readSourceRecords(entities, relations);
-  if (!source.length) throw Error(`No eligible source records: civic=${inspected.civic_items} accepted=${inspected.accepted_items} predictive=${inspected.predictive_items} effect=${inspected.valid_effect_items} members=${inspected.variant_member_items} clinical=${inspected.clinical_relations}`);
+  if (!source.length) throw Error(`Missing eligible CIViC: c${inspected.civic_items}a${inspected.accepted_items}p${inspected.predictive_items}e${inspected.valid_effect_items}m${inspected.variant_member_items}`);
   const questions = makeQuestions(source, count);
   const results = questions.map((q) => {
     const tree = buildEvidenceTree(entities, relations, { gene: q.gene,

From 1e5e11a180977d2a991376fd54f1d72f2148d366 Mon Sep 17 00:00:00 2001
From: Yoshitaka Inoue <8393063+inoue0426@users.noreply.github.com>
Date: Mon, 12 Oct 2026 00:39:23 -0400
Subject: [PATCH 6/9] Evaluate legacy snapshots at explicit whole-profile scope

---
 .../workflows/claim-abstention-benchmark.yml  |  2 +
 README.md                                     |  2 +-
 docs/CLAIM_BENCHMARK.md                       | 19 +++++--
 docs/CLAIM_EVIDENCE.md                        | 14 ++++--
 docs/MCP.md                                   |  2 +-
 scripts/benchmark_claim_abstention.cjs        | 50 ++++++++++++-------
 src/oncograph/claim_evidence.py               | 15 +++---
 src/oncograph/main.py                         |  5 +-
 src/oncograph/mcp/tools.py                    |  5 +-
 tests/test_claim_benchmark.cjs                | 14 ++++++
 tests/test_claim_evidence.cjs                 | 10 ++++
 tests/test_claim_evidence.py                  | 13 +++++
 web/claim-evidence.js                         |  9 ++--
 web/evidence-tree.js                          | 14 +++++-
 web/index.html                                |  2 +-
 15 files changed, 131 insertions(+), 45 deletions(-)

diff --git a/.github/workflows/claim-abstention-benchmark.yml b/.github/workflows/claim-abstention-benchmark.yml
index 87e1730..e01a58c 100644
--- a/.github/workflows/claim-abstention-benchmark.yml
+++ b/.github/workflows/claim-abstention-benchmark.yml
@@ -79,6 +79,8 @@ jobs:
             const pct = (x) => x == null ? "n/a" : (100 * x).toFixed(1) + "%";
             const lines = ["## 100 source-derived claim questions", "",
               "**Not independent biomedical gold or an LLM experiment.** Labels are derived from accepted CIViC fields in this same frozen snapshot; the result tests source consistency only.", "",
+              "Question scope: **" + r.scope + "**. " + r.scope_note, "",
+              "Source coverage: `" + JSON.stringify(r.source_coverage) + "`", "",
               "Snapshot hashes: `" + JSON.stringify(r.snapshot_sha256) + "`", "",
               "Question types: `" + JSON.stringify(r.categories) + "`; truncated: " + r.truncated_count, "",
               "| Method | Answered | Answer rate on source-answerable | False answers on source-unanswerable | Error among answered |",
diff --git a/README.md b/README.md
index 8ae653e..ec361f7 100644
--- a/README.md
+++ b/README.md
@@ -61,7 +61,7 @@ The [variant-conditioned retrieval experiment](docs/VARIANT_RETRIEVAL_EXPERIMENT
 
 The [variant claim check](docs/CLAIM_EVIDENCE.md) additionally filters those records by exact therapy and reported sensitivity or resistance. It keeps supporting, opposing, disputing and insufficient-context records separate in the browser, local API and MCP. This research-only rule set does not validate LLM prose or establish treatment efficacy.
 
-The [100-question CI benchmark](docs/CLAIM_BENCHMARK.md) runs three retrieval strategies on the same frozen Pages artifact and publishes a detailed report. Its automatically generated labels test source consistency, **not** independently established biomedical accuracy.
+The [100-question CI benchmark](docs/CLAIM_BENCHMARK.md) runs three retrieval strategies on the same frozen Pages artifact and publishes a detailed report. A snapshot lacking variant-member exports is evaluated at **whole-profile scope only**, not as a single-variant study. Its automatically generated labels test source consistency, **not** independently established biomedical accuracy.
 
 **Explore by cancer name** groups disease entities currently in the public index using explicit matches on their names. Open a subtype to inspect its direct relations and, when available, accepted CIViC molecular-profile evidence. The groups are a browsing shortcut, not a complete ontology or a claim that all matching diseases are equivalent.
 
diff --git a/docs/CLAIM_BENCHMARK.md b/docs/CLAIM_BENCHMARK.md
index d6c455d..0a7fe0f 100644
--- a/docs/CLAIM_BENCHMARK.md
+++ b/docs/CLAIM_BENCHMARK.md
@@ -10,19 +10,24 @@ disease and therapy?" It is **not** a test of clinical truth, independent
 expert adjudication, recommendation quality, or LLM-generated answer quality.
 
 The script deterministically constructs exactly **100 distinct questions**
-from accepted predictive CIViC records with exported variant membership. It
+from accepted predictive CIViC records. When the snapshot exports variant
+membership, questions target **one recorded variant**. When membership is
+unavailable, questions target the **exact complete molecular profile name**
+instead; the program must not treat that name as proof about a single variant.
+The mode and membership count are recorded in the report. It
 adds counterfactual conditions (another variant, therapy, disease, opposite
 effect) and, where available, complex molecular profiles. It checks every
 counterfactual against all eligible source records before assigning its
 source-derived label, so a swapped condition with genuine matching evidence
 is not mistakenly treated as empty. Target quotas are 25 direct, 20 other
 variant, 20 other therapy, 15 other cancer, 10 opposite effect and 10 complex
-profile questions. If a category has insufficient records, other categories
+profile questions **when membership is available**. If a category has insufficient records, other categories
 fill the set; the **actual** mixture appears in the report. If fewer than 100
 unique questions exist, CI fails rather than duplicating questions.
 
 The source reference enumerates raw accepted predictive CIViC records by exact
-gene, single-variant membership, disease and therapy; a record has a matching
+gene, disease, therapy and either verified single-variant membership or complete
+profile name, depending on the available snapshot; a record has a matching
 reported effect only when the direction is `SUPPORTS`. A reference question
 is marked source-answerable if it has matching support and no supporting
 report for the opposite effect or `DOES_NOT_SUPPORT` report of that effect.
@@ -36,7 +41,7 @@ mean abstain **within this snapshot**, not absence of clinical benefit.
 | --- | --- | --- |
 | `context_blind` | Any accepted CIViC record for the gene, drug and disease | Ignores variant, direction and significance |
 | `variant_only` | Any accepted CIViC record for the gene, variant/profile, drug and disease | Ignores predictive significance, direction and complex membership |
-| `claim_scoped` | The structured claim check returns `supporting_records` | Requires exact reported effect and single-variant profile; abstains on opposing/disputing reports and incomplete input |
+| `claim_scoped` | The structured claim check returns `supporting_records` | Requires exact reported effect and the mode's matching scope; abstains on opposing/disputing reports and incomplete input |
 
 The baselines are **ablations of OncoGraph's own retrieval**, not standalone
 CIViC MCP, an external clinical service or an LLM. The source reference is
@@ -45,6 +50,12 @@ high scores only show consistency with the frozen source fields. In particular,
 they cannot demonstrate a novel clinical conclusion or a performance gain on
 an independently labeled question set.
 
+**Profile-mode results cannot establish variant-specific accuracy.** The
+profile-name lookup is permitted only when the question explicitly asks about
+the whole profile. For a single-variant question without membership, the claim
+checker continues to abstain. The workflow cannot silently substitute one
+meaning for the other.
+
 Metrics on the 100 questions: answer rate among source-answerable items;
 false-answer rate among source-unanswerable items; answered count/coverage;
 and error among answered. A tree with more than 200 matching records is flagged
diff --git a/docs/CLAIM_EVIDENCE.md b/docs/CLAIM_EVIDENCE.md
index 6d81eb3..b30bd2a 100644
--- a/docs/CLAIM_EVIDENCE.md
+++ b/docs/CLAIM_EVIDENCE.md
@@ -8,15 +8,18 @@ or patient suitability. It does not yet verify free-text LLM answers.
 
 Open **More tools → Review variant evidence → Check a variant claim** and enter
 an exact gene, variant, disease, therapy, and whether to check a report of
-`sensitivity` or `resistance`. Download the structured JSON and follow the
+`sensitivity` or `resistance`. If using the whole molecular profile name,
+explicitly choose **profile** scope; `variant` remains the conservative default.
+An exact whole-profile match does not prove a finding for an individual member
+variant. Download the structured JSON and follow the
 CIViC source URLs to verify the records. Therapy may be an exact name or NCIt
 ID **as stored in this snapshot**. No cross-source alias mapping is performed.
 
 Against a locally imported database, the same algorithm is exposed through:
 
 ```text
-GET /claim-evidence?gene=EGFR&variant=T790M&disease=<exact name or DOID>&therapy=<exact name or NCIt ID>&effect=resistance
-MCP check_variant_claim(gene, variant, disease, therapy, effect, limit=20)
+GET /claim-evidence?gene=EGFR&variant=T790M&disease=<exact name or DOID>&therapy=<exact name or NCIt ID>&effect=resistance&scope=variant
+MCP check_variant_claim(gene, variant, disease, therapy, effect, limit=20, scope="variant")
 ```
 
 The browser reads GitHub Pages JSON. The Python API and local MCP read their
@@ -28,7 +31,10 @@ First use `buildEvidenceTree` / `get_evidence_tree` with a 200-Evidence-Item
 cap. This already requires the exact gene and disease, a matching variant
 member (or flagged legacy profile-name match), accepted CIViC status, and
 preserves the original evidence ID and claim direction. Then require a therapy
-matched by exact stored name or ID and `PREDICTIVE` evidence. Other relation
+matched by exact stored name or ID and `PREDICTIVE` evidence. With explicit
+`scope=profile`, an exact whole-profile-name match can support a claim about
+that **whole profile** even when member variants were not exported; it never
+establishes a claim about an individual variant. Other relation
 types, including drug `targets` gene, trial registrations, pathway membership,
 and target–disease associations, **cannot** support a variant–therapy effect.
 
diff --git a/docs/MCP.md b/docs/MCP.md
index fc0d460..1c74d28 100644
--- a/docs/MCP.md
+++ b/docs/MCP.md
@@ -2,7 +2,7 @@
 
 The local stdio MCP server also exposes `get_evidence_tree(gene, variant, disease, jurisdiction="US", limit=50)`. It returns accepted CIViC evidence with source links, preserves contradictory claims, and marks US regulatory labels and CIViC Assertions as not integrated. See [Evidence Tree v1](EVIDENCE_TREE.md) for exact inputs and response fields. GitHub Pages does not serve MCP.
 
-`check_variant_claim(gene, variant, disease, therapy, effect, limit=20)` classifies accepted CIViC predictive items for exact stored therapy and a reported `sensitivity` or `resistance` effect. It returns separate supporting, opposing, disputing and context-only records and abstains on incomplete profile membership or truncated input. See [claim check contract](CLAIM_EVIDENCE.md). It never converts a drug–target–disease path into a therapeutic claim.
+`check_variant_claim(gene, variant, disease, therapy, effect, limit=20, scope="variant")` classifies accepted CIViC predictive items for exact stored therapy and a reported `sensitivity` or `resistance` effect. It returns separate supporting, opposing, disputing and context-only records and abstains on incomplete variant membership or truncated input. Explicit `scope="profile"` checks the exact complete molecular profile name, never a single variant. See [claim check contract](CLAIM_EVIDENCE.md). It never converts a drug–target–disease path into a therapeutic claim.
 
 **Research use only.** OncoGraph's MCP server exposes structured oncology
 knowledge-graph data -- entities, relations, and their evidence -- to MCP
diff --git a/scripts/benchmark_claim_abstention.cjs b/scripts/benchmark_claim_abstention.cjs
index 2be7546..04317d1 100644
--- a/scripts/benchmark_claim_abstention.cjs
+++ b/scripts/benchmark_claim_abstention.cjs
@@ -44,10 +44,12 @@ function readSourceRecords(entities, relations) {
       inspected.predictive_items++;
       if (!["sensitivity", "resistance"].includes(normal(ctx.significance))) continue;
       inspected.valid_effect_items++;
-      if (!ctx.civic_evidence_id || !ctx.gene_name || !Array.isArray(ctx.profile_variants)) continue;
-      inspected.variant_member_items++;
+      if (!ctx.civic_evidence_id || !ctx.gene_name || !ctx.molecular_profile) continue;
+      if (Array.isArray(ctx.profile_variants) && ctx.profile_variants.length) inspected.variant_member_items++;
       rows.push({ id: String(ctx.civic_evidence_id), gene: String(ctx.gene_name),
-        members: ctx.profile_variants.map((member) => memberName(member, ctx.gene_name)),
+        profile_name: String(ctx.molecular_profile),
+        members: Array.isArray(ctx.profile_variants) ? ctx.profile_variants.map((member) =>
+          memberName(member, ctx.gene_name)) : [],
         disease: cancer.name, therapy: drug.name, effect: normal(ctx.significance),
         direction: normal(item.claim_state) });
     }
@@ -55,11 +57,12 @@ function readSourceRecords(entities, relations) {
   return { rows, byId, inspected };
 }
 
-function referenceIndex(records) {
+function referenceIndex(records, scope = "variant") {
   const index = new Map();
   for (const row of records) {
-    if (row.members.length !== 1) continue;
-    const base = [row.gene, row.members[0], row.disease, row.therapy].map(normal).join("|");
+    if (scope === "variant" && row.members.length !== 1) continue;
+    const name = scope === "profile" ? row.profile_name : row.members[0];
+    const base = [row.gene, name, row.disease, row.therapy].map(normal).join("|");
     if (!index.has(base)) index.set(base, []);
     index.get(base).push(row);
   }
@@ -67,7 +70,7 @@ function referenceIndex(records) {
 }
 function sourceReference(q, recordsOrIndex) {
   const supporting = new Set(), opposing = new Set(), disputed = new Set();
-  const index = recordsOrIndex instanceof Map ? recordsOrIndex : referenceIndex(recordsOrIndex);
+  const index = recordsOrIndex instanceof Map ? recordsOrIndex : referenceIndex(recordsOrIndex, q.scope);
   const base = [q.gene, q.variant, q.disease, q.therapy].map(normal).join("|");
   for (const row of index.get(base) || []) {
     if (row.direction === "supports" && row.effect === q.effect) supporting.add(row.id);
@@ -78,9 +81,11 @@ function sourceReference(q, recordsOrIndex) {
     supporting_eids: [...supporting].sort(), opposing_count: opposing.size, disputed_count: disputed.size };
 }
 
-function makeQuestions(records, count = 100) {
+function makeQuestions(records, count = 100, scope = "variant") {
   if (!Number.isInteger(count) || count < 1 || count > 1000) throw Error("Invalid question count");
-  const ordered = records.filter((row) => row.members.length === 1 && row.direction === "supports")
+  if (!["variant", "profile"].includes(scope)) throw Error("Invalid scope");
+  const ordered = records.filter((row) => (scope === "profile" || row.members.length === 1) &&
+    row.direction === "supports")
     .sort((a, b) => [a.gene, a.disease, a.therapy, a.members[0], a.effect, a.id].join("|")
       .localeCompare([b.gene, b.disease, b.therapy, b.members[0], b.effect, b.id].join("|")));
   if (!ordered.length) throw Error("No eligible accepted CIViC predictive records in snapshot");
@@ -93,9 +98,9 @@ function makeQuestions(records, count = 100) {
   while (seed.length < 300 && [...genes.values()].some((list) => list.length)) {
     for (const list of genes.values()) if (list.length && seed.length < 300) seed.push(list.shift());
   }
-  const index = referenceIndex(records);
+  const index = referenceIndex(records, scope);
   const variantsByGene = new Map();
-  for (const row of records) for (const member of row.members) {
+  for (const row of records) for (const member of scope === "profile" ? [row.profile_name] : row.members) {
     if (!variantsByGene.has(row.gene)) variantsByGene.set(row.gene, new Set());
     variantsByGene.get(row.gene).add(member);
   }
@@ -113,8 +118,8 @@ function makeQuestions(records, count = 100) {
   // These are perturbations of real source rows. Negatives are checked
   // against the full available source record set, never presumed absent.
   for (const row of seed) {
-    const base = { gene: row.gene, variant: row.members[0], disease: row.disease,
-      therapy: row.therapy, effect: row.effect };
+    const base = { gene: row.gene, variant: scope === "profile" ? row.profile_name : row.members[0],
+      disease: row.disease, therapy: row.therapy, effect: row.effect, scope };
     add("direct", base, true);
     add("opposite_effect", { ...base, effect: row.effect === "sensitivity" ? "resistance" : "sensitivity" }, false);
     for (const variant of [...(variantsByGene.get(row.gene) || [])].slice(0, 15)) {
@@ -127,10 +132,10 @@ function makeQuestions(records, count = 100) {
       if (normal(disease) !== normal(base.disease)) add("other_cancer", { ...base, disease }, false);
     }
   }
-  for (const row of records.filter((item) => item.members.length > 1)) {
+  for (const row of scope === "variant" ? records.filter((item) => item.members.length > 1) : []) {
     for (const variant of row.members) {
       add("complex_profile", { gene: row.gene, variant, disease: row.disease,
-        therapy: row.therapy, effect: row.effect }, false);
+        therapy: row.therapy, effect: row.effect, scope }, false);
     }
   }
   const quotas = { direct: 25, other_variant: 20, other_therapy: 20,
@@ -149,7 +154,7 @@ function makeQuestions(records, count = 100) {
   }
   if (selected.size < count) throw Error(`Only ${selected.size} unique eligible questions; need ${count}`);
   return [...selected.values()].slice(0, count).map((q, i) => ({ id: `claim-${String(i + 1).padStart(3, "0")}`,
-    question: `In ${q.disease}, do accepted CIViC records report ${q.effect} for ${q.gene} ${q.variant} with ${q.therapy}?`, ...q }));
+    question: `In ${q.disease}, do accepted CIViC records report ${q.effect} for ${scope === "profile" ? "the complete molecular profile" : "the single variant"} ${q.gene} ${q.variant} with ${q.therapy}?`, ...q }));
 }
 
 function metrics(results, strategy) {
@@ -170,17 +175,19 @@ function metrics(results, strategy) {
 function evaluate(entities, relations, count = 100) {
   const { rows: source, byId, inspected } = readSourceRecords(entities, relations);
   if (!source.length) throw Error(`Missing eligible CIViC: c${inspected.civic_items}a${inspected.accepted_items}p${inspected.predictive_items}e${inspected.valid_effect_items}m${inspected.variant_member_items}`);
-  const questions = makeQuestions(source, count);
+  const scope = inspected.variant_member_items ? "variant" : "profile";
+  const questions = makeQuestions(source, count, scope);
   const results = questions.map((q) => {
     const tree = buildEvidenceTree(entities, relations, { gene: q.gene,
       variant: q.variant, disease: q.disease, limit: 200 });
-    const claim = evaluateEvidenceClaim(tree, { therapy: q.therapy, effect: q.effect, limit: 200 });
+    const claim = evaluateEvidenceClaim(tree, { therapy: q.therapy, effect: q.effect,
+      scope, limit: 200 });
     const blindIds = contextBlindIds(q, relations, byId);
     const scopedIds = tree.evidence.filter((item) => item.therapies.some((drug) =>
       normal(drug.name) === normal(q.therapy))).map((item) => item.evidence_id);
     return { id: q.id, category: q.category, question: q.question,
       query: { gene: q.gene, variant: q.variant, disease: q.disease,
-        therapy: q.therapy, effect: q.effect },
+        therapy: q.therapy, effect: q.effect, scope },
       reference: q.reference, tree_truncated: tree.truncated,
       predictions: { context_blind: { answers: blindIds.length > 0, evidence_ids: blindIds },
         variant_only: { answers: scopedIds.length > 0 && !tree.truncated, evidence_ids: scopedIds },
@@ -188,6 +195,10 @@ function evaluate(entities, relations, count = 100) {
           evidence_ids: claim.evidence.supporting.map((item) => item.evidence_id) } } };
   });
   return { methodology: "source-derived controlled perturbation; not independent clinical gold or LLM evaluation",
+    scope, scope_note: scope === "profile"
+      ? "No exported variant membership: all 100 questions concern exact whole molecular profiles, not single variants."
+      : "Single-variant membership exported; whole-profile-name-only records are never treated as single-variant proof.",
+    source_coverage: inspected,
     question_count: results.length,
     categories: Object.fromEntries(Object.keys({ direct: 1, other_variant: 1, other_therapy: 1,
       other_cancer: 1, opposite_effect: 1, complex_profile: 1 }).map((category) =>
@@ -211,6 +222,7 @@ if (require.main === module) {
   fs.mkdirSync(out, { recursive: true });
   fs.writeFileSync(path.join(out, "report.json"), JSON.stringify(report, null, 2) + "\n");
   process.stdout.write(JSON.stringify({ question_count: report.question_count,
+    scope: report.scope, source_coverage: report.source_coverage,
     categories: report.categories, truncated_count: report.truncated_count,
     metrics: report.metrics, snapshot_sha256: report.snapshot_sha256 }, null, 2) + "\n");
 }
diff --git a/src/oncograph/claim_evidence.py b/src/oncograph/claim_evidence.py
index 1e72c02..05693cb 100644
--- a/src/oncograph/claim_evidence.py
+++ b/src/oncograph/claim_evidence.py
@@ -10,7 +10,7 @@
 
 
 def evaluate_evidence_claim(tree: dict[str, Any], *, therapy: str, effect: str,
-                            limit: int = 20) -> dict[str, Any]:
+                            scope: str = "variant", limit: int = 20) -> dict[str, Any]:
     """Classify accepted records without promoting graph paths to clinical claims.
 
     Only predictive evidence with an exact named therapy and a single-variant
@@ -18,7 +18,7 @@ def evaluate_evidence_claim(tree: dict[str, Any], *, therapy: str, effect: str,
     An opposing significance is context, not necessarily a logical negation.
     """
     if (not isinstance(therapy, str) or not therapy.strip()
-            or effect not in {"sensitivity", "resistance"}
+            or effect not in {"sensitivity", "resistance"} or scope not in {"variant", "profile"}
             or type(limit) is not int or not 1 <= limit <= 200):
         raise ValueError("Invalid claim query")
     if not isinstance(tree, dict) or not isinstance(tree.get("evidence"), list) or "query" not in tree:
@@ -44,7 +44,9 @@ def normal(value: Any) -> str:
             excluded["unrelated_effect"] += 1
             continue
         profile = item.get("profile") or {}
-        if profile.get("match_type") != "variant_member" or profile.get("complex_profile"):
+        exact_profile = (scope == "profile" and profile.get("match_type") == "profile_only"
+                         and normal(profile.get("name")) == normal(tree["query"].get("variant")))
+        if not exact_profile and (profile.get("match_type") != "variant_member" or profile.get("complex_profile")):
             groups["context_only"].append({**item, "reason":
                 "Variant-specific effect unverified for this molecular profile"})
             excluded["incomplete_profile"] += 1
@@ -85,7 +87,7 @@ def normal(value: Any) -> str:
     else:
         status = "not_found_in_snapshot"
     return {"schema_version": "1.0", "query": {**tree["query"], "therapy": therapy.strip(),
-                                                "effect": effect},
+                                                "effect": effect, "scope": scope},
             "status": status, "counts": totals, "excluded": excluded, "incomplete": incomplete,
             "evidence": selected,
             "limitations": ["This checks a bounded CIViC snapshot, not treatment efficacy, FDA status or patient suitability.",
@@ -94,8 +96,9 @@ def normal(value: Any) -> str:
 
 
 def get_claim_evidence(session: Session, *, gene: str, variant: str, disease: str,
-                       therapy: str, effect: str, limit: int = 20) -> dict[str, Any]:
+                       therapy: str, effect: str, scope: str = "variant",
+                       limit: int = 20) -> dict[str, Any]:
     """Use the same capped Evidence Tree as the browser; flag incomplete input."""
     return evaluate_evidence_claim(get_evidence_tree(session, gene=gene, variant=variant,
                                                      disease=disease, limit=200),
-                                   therapy=therapy, effect=effect, limit=limit)
+                                   therapy=therapy, effect=effect, scope=scope, limit=limit)
diff --git a/src/oncograph/main.py b/src/oncograph/main.py
index 65669af..120b390 100644
--- a/src/oncograph/main.py
+++ b/src/oncograph/main.py
@@ -47,11 +47,12 @@ def evidence_tree(session: SessionDep, gene: str, variant: str, disease: str,
 
 @app.get("/claim-evidence")
 def claim_evidence(session: SessionDep, gene: str, variant: str, disease: str,
-                   therapy: str, effect: str, limit: Annotated[int, Query(ge=1, le=200)] = 20):
+                   therapy: str, effect: str, scope: str = "variant",
+                   limit: Annotated[int, Query(ge=1, le=200)] = 20):
     """Classify scoped CIViC records; never assert patient benefit or FDA approval."""
     try:
         return check_claim_from_db(session, gene=gene, variant=variant, disease=disease,
-                                   therapy=therapy, effect=effect, limit=limit)
+                                   therapy=therapy, effect=effect, scope=scope, limit=limit)
     except ValueError as exc:
         raise HTTPException(422, str(exc)) from exc
 
diff --git a/src/oncograph/mcp/tools.py b/src/oncograph/mcp/tools.py
index 5898a09..c9b24bf 100644
--- a/src/oncograph/mcp/tools.py
+++ b/src/oncograph/mcp/tools.py
@@ -73,12 +73,13 @@ def get_evidence_tree(gene: str, variant: str, disease: str, jurisdiction: str =
 
 
 def check_variant_claim(gene: str, variant: str, disease: str, therapy: str,
-                        effect: str, limit: int = 20) -> dict[str, Any]:
+                        effect: str, limit: int = 20, scope: str = "variant") -> dict[str, Any]:
     """Separate supporting, opposing and inconclusive CIViC records for one research claim."""
     try:
         with session_scope() as session:
             return _check_claim_from_db(session, gene=gene, variant=variant,
-                                        disease=disease, therapy=therapy, effect=effect, limit=limit)
+                                        disease=disease, therapy=therapy, effect=effect,
+                                        scope=scope, limit=limit)
     except ValueError as exc:
         raise OncoGraphMCPError(str(exc)) from exc
 
diff --git a/tests/test_claim_benchmark.cjs b/tests/test_claim_benchmark.cjs
index 2b59828..44b7e7b 100644
--- a/tests/test_claim_benchmark.cjs
+++ b/tests/test_claim_benchmark.cjs
@@ -59,3 +59,17 @@ test("fails rather than reporting 100 questions if there are too few unique sour
   const { entities, relations } = fixture();
   assert.throws(() => evaluate(entities, relations, 1000), /unique eligible questions/);
 });
+
+test("legacy snapshot runs 100 profile-level questions without inventing variant membership", () => {
+  const { entities, relations } = fixture();
+  for (const relation of relations) for (const item of relation.evidence) {
+    delete item.context.profile_variants;
+  }
+  const report = evaluate(entities, relations);
+  assert.equal(report.scope, "profile");
+  assert.equal(report.source_coverage.variant_member_items, 0);
+  assert.equal(report.question_count, 100);
+  assert(report.questions.every((row) => row.query.scope === "profile"));
+  assert.equal(report.metrics.claim_scoped.false_answer_rate_on_unanswerable, 0);
+  assert.equal(report.metrics.claim_scoped.answer_rate_on_answerable, 1);
+});
diff --git a/tests/test_claim_evidence.cjs b/tests/test_claim_evidence.cjs
index 21b121a..9b8854a 100644
--- a/tests/test_claim_evidence.cjs
+++ b/tests/test_claim_evidence.cjs
@@ -58,3 +58,13 @@ test("truncation and display caps force incomplete review; invalid claims are re
   assert.throws(() => evaluateEvidenceClaim(tree, { therapy: "", effect: "sensitivity" }));
   assert.throws(() => evaluateEvidenceClaim(tree, { therapy: "Osimertinib", effect: "response" }));
 });
+
+test("legacy snapshot can answer exact whole-profile question while single variant still abstains", () => {
+  const tree = buildEvidenceTree(entities, [relation("t", 9, "SENSITIVITY", "SUPPORTS", [])],
+    { gene: "EGFR", variant: "EGFR T790M", disease: "Lung cancer" });
+  assert.equal(evaluateEvidenceClaim(tree, { therapy: "Osimertinib", effect: "sensitivity" }).status,
+    "insufficient_variant_context");
+  const profile = evaluateEvidenceClaim(tree, { therapy: "Osimertinib", effect: "sensitivity", scope: "profile" });
+  assert.equal(profile.status, "supporting_records");
+  assert.equal(profile.query.scope, "profile");
+});
diff --git a/tests/test_claim_evidence.py b/tests/test_claim_evidence.py
index 0133baf..875b50f 100644
--- a/tests/test_claim_evidence.py
+++ b/tests/test_claim_evidence.py
@@ -38,3 +38,16 @@ def item(eid, significance, members, state="SUPPORTS"):
     assert evaluate_evidence_claim({**tree, "truncated": True}, therapy="ncit:1", effect="sensitivity")["status"] == "incomplete_review"
     with pytest.raises(ValueError):
         evaluate_evidence_claim(tree, therapy="ncit:1", effect="efficacy")
+
+
+def test_whole_profile_is_distinct_from_unverified_single_variant():
+    tree = {"query": {"gene": "EGFR", "variant": "EGFR T790M", "disease": "Lung cancer"},
+            "truncated": False, "evidence": [
+                {"evidence_id": "9", "profile": {"name": "EGFR T790M", "match_type": "profile_only",
+                                                  "complex_profile": False},
+                 "evidence_type": "PREDICTIVE", "significance": "SENSITIVITY",
+                 "claim_state": "supports", "therapies": [{"name": "Osimertinib"}]},
+            ]}
+    assert evaluate_evidence_claim(tree, therapy="Osimertinib", effect="sensitivity")["status"] == "insufficient_variant_context"
+    assert evaluate_evidence_claim(tree, therapy="Osimertinib", effect="sensitivity",
+                                   scope="profile")["status"] == "supporting_records"
diff --git a/web/claim-evidence.js b/web/claim-evidence.js
index 69b1171..c12806a 100644
--- a/web/claim-evidence.js
+++ b/web/claim-evidence.js
@@ -1,7 +1,8 @@
 // A conservative, deterministic classifier of *retrieved CIViC records*.
 // Statuses describe the snapshot only; none are treatment recommendations.
-function evaluateEvidenceClaim(tree, { therapy, effect, limit = 20 }) {
+function evaluateEvidenceClaim(tree, { therapy, effect, scope = "variant", limit = 20 }) {
   if (typeof therapy !== "string" || !therapy.trim() || !["sensitivity", "resistance"].includes(effect)
+      || !["variant", "profile"].includes(scope)
       || !Number.isInteger(limit) || limit < 1 || limit > 200) throw new Error("Invalid claim query");
   if (!tree || !Array.isArray(tree.evidence) || !tree.query) throw new Error("Invalid evidence tree");
   const normal = (value) => String(value || "").trim().toLowerCase();
@@ -16,7 +17,9 @@ function evaluateEvidenceClaim(tree, { therapy, effect, limit = 20 }) {
     if (!["sensitivity", "resistance"].includes(significance)) { counts.unrelated_effect++; continue; }
     // An item with incomplete membership or a multi-variant profile cannot
     // prove that this single variant has the reported effect.
-    if (item.profile?.match_type !== "variant_member" || item.profile?.complex_profile) {
+    const exactProfile = scope === "profile" && item.profile?.match_type === "profile_only" &&
+      normal(item.profile.name) === normal(tree.query.variant);
+    if (!exactProfile && (item.profile?.match_type !== "variant_member" || item.profile?.complex_profile)) {
       groups.context_only.push({ ...item, reason: "Variant-specific effect unverified for this molecular profile" });
       counts.incomplete_profile++; continue;
     }
@@ -45,7 +48,7 @@ function evaluateEvidenceClaim(tree, { therapy, effect, limit = 20 }) {
   else if (totals.supporting) status = "supporting_records";
   else if (totals.opposing || totals.disputed) status = "opposing_or_disputed_records";
   else if (totals.context_only) status = "insufficient_variant_context";
-  return { schema_version: "1.0", query: { ...tree.query, therapy: therapy.trim(), effect },
+  return { schema_version: "1.0", query: { ...tree.query, therapy: therapy.trim(), effect, scope },
     status, counts: totals, excluded: counts, incomplete,
     evidence: selected, limitations: ["This checks a bounded CIViC snapshot, not treatment efficacy, FDA status or patient suitability.",
       "An opposing report is not necessarily a direct contradiction; study contexts may differ.",
diff --git a/web/evidence-tree.js b/web/evidence-tree.js
index ba9d903..d1e8bdd 100644
--- a/web/evidence-tree.js
+++ b/web/evidence-tree.js
@@ -76,6 +76,15 @@ function renderEvidenceTree(entities, relations) {
   therapy.name = "therapy"; therapy.placeholder = "e.g. osimertinib (optional)";
   therapyLabel.append("Therapy ", therapy);
   form.append(therapyLabel);
+  const scopeLabel = document.createElement("label");
+  const scope = document.createElement("select");
+  for (const [value, label] of [["variant", "Single variant (verified membership)"],
+    ["profile", "Exact complete CIViC molecular profile name"]]) {
+    const option = document.createElement("option"); option.value = value; option.textContent = label;
+    scope.append(option);
+  }
+  scopeLabel.append("Interpret input as ", scope);
+  form.append(scopeLabel);
   const effectLabel = document.createElement("label");
   const effect = document.createElement("select");
   for (const value of ["sensitivity", "resistance"]) {
@@ -94,10 +103,11 @@ function renderEvidenceTree(entities, relations) {
       { gene: inputs[0].value, variant: inputs[1].value, disease: inputs[2].value,
         limit: therapy.value.trim() ? 200 : 50 });
     result.replaceChildren();
-    const claim = therapy.value.trim() ? evaluateEvidenceClaim(tree, { therapy: therapy.value, effect: effect.value }) : null;
+    const claim = therapy.value.trim() ? evaluateEvidenceClaim(tree,
+      { therapy: therapy.value, effect: effect.value, scope: scope.value }) : null;
     const summary = document.createElement("p");
     summary.textContent = claim
-      ? `Snapshot check: ${claim.status.replaceAll("_", " ")}. ${claim.counts.supporting} supporting; ${claim.counts.opposing} opposing reports; ${claim.counts.disputed} disputing the stated effect; ${claim.counts.context_only} context only. ${claim.incomplete ? "Results are incomplete. " : ""}This is not a treatment recommendation. Exact therapy name or ID required.`
+      ? `Snapshot check (${claim.query.scope === "profile" ? "whole molecular profile" : "single variant"}): ${claim.status.replaceAll("_", " ")}. ${claim.counts.supporting} supporting; ${claim.counts.opposing} opposing reports; ${claim.counts.disputed} disputing the stated effect; ${claim.counts.context_only} context only. ${claim.incomplete ? "Results are incomplete. " : ""}This is not a treatment recommendation. Exact therapy name or ID required.`
       : `${tree.total_evidence_items} accepted CIViC evidence item(s) in this snapshot. FDA labels and CIViC assertions: not integrated. No match does not mean no evidence exists.`;
     result.append(summary);
     const rows = claim ? Object.entries(claim.evidence).flatMap(([group, items]) => items.map((item) => ({ group, item })))
diff --git a/web/index.html b/web/index.html
index baa4b92..1d5ca18 100644
--- a/web/index.html
+++ b/web/index.html
@@ -94,7 +94,7 @@ <h2 id="molecular-title">Review variant evidence</h2>
           <section id="variant-comparison" class="variant-comparison" aria-label="Compare variant evidence"><p>Loading variant evidence…</p></section>
         </details>
         <details id="evidence-tool" class="tool-panel"><summary>Check a variant claim or inspect an Evidence Tree</summary>
-          <p>Enter an exact CIViC gene, variant and cancer name or DOID. Optionally add a therapy to separate supporting, opposing and inconclusive reports of sensitivity or resistance. This checks recorded evidence, not treatment efficacy. FDA labels and CIViC Assertions are not integrated.</p>
+          <p>Enter an exact CIViC gene, variant or complete molecular profile name, and cancer name or DOID. Optionally add a therapy and choose the input scope to separate reported sensitivity and resistance. A profile-name match does not prove an individual variant's effect. FDA labels and CIViC Assertions are not integrated.</p>
           <div id="evidence-tree">Loading evidence…</div>
         </details>
       </section>

From d5227bc791259785ff9e6a7ead576c246a34b62c Mon Sep 17 00:00:00 2001
From: Yoshitaka Inoue <8393063+inoue0426@users.noreply.github.com>
Date: Mon, 12 Oct 2026 00:40:40 -0400
Subject: [PATCH 7/9] Include source-consistency counts in the Actions report
 artifact name

---
 .github/workflows/claim-abstention-benchmark.yml | 9 ++++++++-
 docs/CLAIM_BENCHMARK.md                          | 7 +++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/claim-abstention-benchmark.yml b/.github/workflows/claim-abstention-benchmark.yml
index e01a58c..ad2f5ca 100644
--- a/.github/workflows/claim-abstention-benchmark.yml
+++ b/.github/workflows/claim-abstention-benchmark.yml
@@ -64,7 +64,14 @@ jobs:
           node -e '
             const fs = require("node:fs");
             let label = "report";
-            if (!fs.existsSync("_claim-benchmark/report.json")) {
+            if (fs.existsSync("_claim-benchmark/report.json")) {
+              const r = JSON.parse(fs.readFileSync("_claim-benchmark/report.json", "utf8"));
+              const m = r.metrics;
+              const u = m.claim_scoped.source_unanswerable_count;
+              const p = m.claim_scoped.source_answerable_count;
+              const falseCount = (method) => Math.round((m[method].false_answer_rate_on_unanswerable || 0) * u);
+              label = `report-${r.scope}-Q${r.question_count}-P${p}-U${u}-B${falseCount("context_blind")}-V${falseCount("variant_only")}-C${falseCount("claim_scoped")}-Y${m.claim_scoped.answered_count}`;
+            } else {
               const log = fs.existsSync("_benchmark-console.txt") ? fs.readFileSync("_benchmark-console.txt", "utf8") : "Missing benchmark output";
               label = "error-" + (log.match(/(?:Error|TypeError): ([^\n]+)/)?.[1] || "unknown").replace(/[^a-zA-Z0-9]+/g, "-").slice(0, 80);
             }
diff --git a/docs/CLAIM_BENCHMARK.md b/docs/CLAIM_BENCHMARK.md
index 0a7fe0f..a316ca0 100644
--- a/docs/CLAIM_BENCHMARK.md
+++ b/docs/CLAIM_BENCHMARK.md
@@ -73,8 +73,11 @@ node scripts/benchmark_claim_abstention.cjs --snapshot web/data --questions 100
 `report.json` includes all 100 query texts, question types, source-derived
 labels, EIDs per method, coverage, abstention counts and SHA-256 hashes of the
 two input JSON files. CI publishes a summary and the report as an Actions
-artifact named `claim-abstention-benchmark-report`; failed runs instead put a
-brief error hint in the artifact name and upload the console log. The workflow
+artifact whose name contains `profile` or `variant`, question count `Q`,
+source-answerable `P`, source-unanswerable `U`, false answers for each strategy
+(`B` context-blind, `V` variant-only, `C` claim-scoped) and claim-scoped
+answers `Y`. The detailed JSON carries exact fractions and EIDs. Failed runs
+instead put a brief error hint in the artifact name and upload the console log. The workflow
 also runs on pull requests, manually, and after successful data refreshes.
 To evaluate scientific novelty, independently review a held-out set of
 question/claim/source tuples, evaluate claim-level citations and free-text

From 2be093311ea546b0ac41355614d5f7cc94f9740e Mon Sep 17 00:00:00 2001
From: Yoshitaka Inoue <8393063+inoue0426@users.noreply.github.com>
Date: Mon, 12 Oct 2026 00:43:40 -0400
Subject: [PATCH 8/9] Require exact profile match before classifying profile
 claims

---
 src/oncograph/claim_evidence.py | 6 ++++--
 tests/test_claim_evidence.cjs   | 4 ++++
 tests/test_claim_evidence.py    | 4 ++++
 web/claim-evidence.js           | 5 +++--
 4 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/oncograph/claim_evidence.py b/src/oncograph/claim_evidence.py
index 05693cb..f8bb45d 100644
--- a/src/oncograph/claim_evidence.py
+++ b/src/oncograph/claim_evidence.py
@@ -46,9 +46,11 @@ def normal(value: Any) -> str:
         profile = item.get("profile") or {}
         exact_profile = (scope == "profile" and profile.get("match_type") == "profile_only"
                          and normal(profile.get("name")) == normal(tree["query"].get("variant")))
-        if not exact_profile and (profile.get("match_type") != "variant_member" or profile.get("complex_profile")):
+        if ((scope == "profile" and not exact_profile)
+                or (scope == "variant" and (profile.get("match_type") != "variant_member"
+                                             or profile.get("complex_profile"))):
             groups["context_only"].append({**item, "reason":
-                "Variant-specific effect unverified for this molecular profile"})
+                "Requested variant or molecular profile is not verified by this record"})
             excluded["incomplete_profile"] += 1
             continue
         direction = normal(item.get("claim_state"))
diff --git a/tests/test_claim_evidence.cjs b/tests/test_claim_evidence.cjs
index 9b8854a..940d319 100644
--- a/tests/test_claim_evidence.cjs
+++ b/tests/test_claim_evidence.cjs
@@ -67,4 +67,8 @@ test("legacy snapshot can answer exact whole-profile question while single varia
   const profile = evaluateEvidenceClaim(tree, { therapy: "Osimertinib", effect: "sensitivity", scope: "profile" });
   assert.equal(profile.status, "supporting_records");
   assert.equal(profile.query.scope, "profile");
+  const other = { ...tree, evidence: [{ ...tree.evidence[0], profile: {
+    ...tree.evidence[0].profile, name: "EGFR C797S" } }] };
+  assert.equal(evaluateEvidenceClaim(other, { therapy: "Osimertinib", effect: "sensitivity", scope: "profile" }).status,
+    "insufficient_variant_context");
 });
diff --git a/tests/test_claim_evidence.py b/tests/test_claim_evidence.py
index 875b50f..068d2e7 100644
--- a/tests/test_claim_evidence.py
+++ b/tests/test_claim_evidence.py
@@ -51,3 +51,7 @@ def test_whole_profile_is_distinct_from_unverified_single_variant():
     assert evaluate_evidence_claim(tree, therapy="Osimertinib", effect="sensitivity")["status"] == "insufficient_variant_context"
     assert evaluate_evidence_claim(tree, therapy="Osimertinib", effect="sensitivity",
                                    scope="profile")["status"] == "supporting_records"
+    other = {**tree, "evidence": [{**tree["evidence"][0], "profile": {
+        **tree["evidence"][0]["profile"], "name": "EGFR C797S"}}]}
+    assert evaluate_evidence_claim(other, therapy="Osimertinib", effect="sensitivity",
+                                   scope="profile")["status"] == "insufficient_variant_context"
diff --git a/web/claim-evidence.js b/web/claim-evidence.js
index c12806a..3900b2d 100644
--- a/web/claim-evidence.js
+++ b/web/claim-evidence.js
@@ -19,8 +19,9 @@ function evaluateEvidenceClaim(tree, { therapy, effect, scope = "variant", limit
     // prove that this single variant has the reported effect.
     const exactProfile = scope === "profile" && item.profile?.match_type === "profile_only" &&
       normal(item.profile.name) === normal(tree.query.variant);
-    if (!exactProfile && (item.profile?.match_type !== "variant_member" || item.profile?.complex_profile)) {
-      groups.context_only.push({ ...item, reason: "Variant-specific effect unverified for this molecular profile" });
+    if ((scope === "profile" && !exactProfile) ||
+        (scope === "variant" && (item.profile?.match_type !== "variant_member" || item.profile?.complex_profile))) {
+      groups.context_only.push({ ...item, reason: "Requested variant or molecular profile is not verified by this record" });
       counts.incomplete_profile++; continue;
     }
     const direction = normal(item.claim_state);

From b621aef3ceead815bef8afc4e5b51a53b423abc1 Mon Sep 17 00:00:00 2001
From: Yoshitaka Inoue <8393063+inoue0426@users.noreply.github.com>
Date: Mon, 12 Oct 2026 00:45:34 -0400
Subject: [PATCH 9/9] Fix Python claim predicate grouping

---
 src/oncograph/claim_evidence.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/oncograph/claim_evidence.py b/src/oncograph/claim_evidence.py
index f8bb45d..2b73e32 100644
--- a/src/oncograph/claim_evidence.py
+++ b/src/oncograph/claim_evidence.py
@@ -48,7 +48,7 @@ def normal(value: Any) -> str:
                          and normal(profile.get("name")) == normal(tree["query"].get("variant")))
         if ((scope == "profile" and not exact_profile)
                 or (scope == "variant" and (profile.get("match_type") != "variant_member"
-                                             or profile.get("complex_profile"))):
+                                             or profile.get("complex_profile")))):
             groups["context_only"].append({**item, "reason":
                 "Requested variant or molecular profile is not verified by this record"})
             excluded["incomplete_profile"] += 1