diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccec423..0af52ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,7 @@ jobs: - run: node tests/test_variant_comparison.cjs - run: node tests/test_evidence_tree.cjs - run: node tests/test_claim_evidence.cjs + - run: node tests/test_claim_benchmark.cjs - run: node tests/test_ask_evidence.cjs - run: node tests/test_cloudflare_ask.cjs - run: node tests/test_variant_question_evaluation.cjs diff --git a/.github/workflows/claim-abstention-benchmark.yml b/.github/workflows/claim-abstention-benchmark.yml new file mode 100644 index 0000000..ad2f5ca --- /dev/null +++ b/.github/workflows/claim-abstention-benchmark.yml @@ -0,0 +1,105 @@ +name: Claim abstention benchmark + +on: + pull_request: + paths: + - 'scripts/benchmark_claim_abstention.cjs' + - 'tests/test_claim_benchmark.cjs' + - 'web/claim-evidence.js' + - '.github/workflows/claim-abstention-benchmark.yml' + push: + branches: [main] + paths: + - 'scripts/benchmark_claim_abstention.cjs' + - 'web/claim-evidence.js' + - '.github/workflows/claim-abstention-benchmark.yml' + workflow_dispatch: + workflow_run: + workflows: ['Refresh data'] + types: [completed] + branches: [main] + +permissions: + contents: read + actions: read + +jobs: + benchmark: + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + - name: Download the published-snapshot refresh artifact + id: refresh + env: + GH_TOKEN: ${{ github.token }} + run: | + run_id=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow refresh-data.yml \ + --status success --limit 1 --json databaseId --jq '.[0].databaseId') + if [ -z "$run_id" ] || [ "$run_id" = "null" ]; then + echo '::error::No successful refresh with a search-index artifact exists.' + exit 1 + fi + echo "run_id=$run_id" >> "$GITHUB_OUTPUT" + - uses: actions/download-artifact@v4 + with: + name: search-index + path: _data-refresh + run-id: ${{ steps.refresh.outputs.run_id }} + github-token: ${{ github.token }} + - name: Run 100 source-consistency questions and three methods + run: | + node tests/test_claim_benchmark.cjs + node scripts/benchmark_claim_abstention.cjs \ + --snapshot _data-refresh --questions 100 --out _claim-benchmark \ + 2>&1 | tee _benchmark-console.txt + - name: Label the result artifact for inspection + id: label + if: always() + run: | + node -e ' + const fs = require("node:fs"); + let label = "report"; + if (fs.existsSync("_claim-benchmark/report.json")) { + const r = JSON.parse(fs.readFileSync("_claim-benchmark/report.json", "utf8")); + const m = r.metrics; + const u = m.claim_scoped.source_unanswerable_count; + const p = m.claim_scoped.source_answerable_count; + const falseCount = (method) => Math.round((m[method].false_answer_rate_on_unanswerable || 0) * u); + label = `report-${r.scope}-Q${r.question_count}-P${p}-U${u}-B${falseCount("context_blind")}-V${falseCount("variant_only")}-C${falseCount("claim_scoped")}-Y${m.claim_scoped.answered_count}`; + } else { + const log = fs.existsSync("_benchmark-console.txt") ? fs.readFileSync("_benchmark-console.txt", "utf8") : "Missing benchmark output"; + label = "error-" + (log.match(/(?:Error|TypeError): ([^\n]+)/)?.[1] || "unknown").replace(/[^a-zA-Z0-9]+/g, "-").slice(0, 80); + } + fs.appendFileSync(process.env.GITHUB_OUTPUT, `label=${label}\n`); + ' + - name: Summarize method and limitations + run: | + node -e ' + const fs = require("node:fs"); + const r = JSON.parse(fs.readFileSync("_claim-benchmark/report.json", "utf8")); + if (r.question_count !== 100) process.exit(1); + const pct = (x) => x == null ? "n/a" : (100 * x).toFixed(1) + "%"; + const lines = ["## 100 source-derived claim questions", "", + "**Not independent biomedical gold or an LLM experiment.** Labels are derived from accepted CIViC fields in this same frozen snapshot; the result tests source consistency only.", "", + "Question scope: **" + r.scope + "**. " + r.scope_note, "", + "Source coverage: `" + JSON.stringify(r.source_coverage) + "`", "", + "Snapshot hashes: `" + JSON.stringify(r.snapshot_sha256) + "`", "", + "Question types: `" + JSON.stringify(r.categories) + "`; truncated: " + r.truncated_count, "", + "| Method | Answered | Answer rate on source-answerable | False answers on source-unanswerable | Error among answered |", + "| --- | ---: | ---: | ---: | ---: |"]; + for (const [key, m] of Object.entries(r.metrics)) lines.push(`| ${key} | ${m.answered_count}/${m.evaluated_count} | ${pct(m.answer_rate_on_answerable)} | ${pct(m.false_answer_rate_on_unanswerable)} | ${pct(m.selective_error_among_answers)} |`); + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join("\n") + "\n"); + ' + - uses: actions/upload-artifact@v4 + if: always() + with: + name: claim-abstention-benchmark-${{ steps.label.outputs.label }} + path: | + _claim-benchmark/report.json + _benchmark-console.txt + retention-days: 30 diff --git a/README.md b/README.md index 796c2cf..ec361f7 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ The [variant-conditioned retrieval experiment](docs/VARIANT_RETRIEVAL_EXPERIMENT The [variant claim check](docs/CLAIM_EVIDENCE.md) additionally filters those records by exact therapy and reported sensitivity or resistance. It keeps supporting, opposing, disputing and insufficient-context records separate in the browser, local API and MCP. This research-only rule set does not validate LLM prose or establish treatment efficacy. +The [100-question CI benchmark](docs/CLAIM_BENCHMARK.md) runs three retrieval strategies on the same frozen Pages artifact and publishes a detailed report. A snapshot lacking variant-member exports is evaluated at **whole-profile scope only**, not as a single-variant study. Its automatically generated labels test source consistency, **not** independently established biomedical accuracy. + **Explore by cancer name** groups disease entities currently in the public index using explicit matches on their names. Open a subtype to inspect its direct relations and, when available, accepted CIViC molecular-profile evidence. The groups are a browsing shortcut, not a complete ontology or a claim that all matching diseases are equivalent. CIViC records retain their molecular profile, named variants, HGVS expression when available, evidence type, level, trust rating, significance, review status, source record, and support/contradiction direction. A profile can contain multiple variants; its clinical evidence does not necessarily apply to each member. Filter to accepted level A/B records rated at least 4/5 by CIViC to focus on stronger study-level evidence; this does not guarantee a positive result or clinical actionability. OncoKB and COSMIC are outbound lookups only; their content is not imported or graded here, and COSMIC may require login. diff --git a/docs/CLAIM_BENCHMARK.md b/docs/CLAIM_BENCHMARK.md new file mode 100644 index 0000000..a316ca0 --- /dev/null +++ b/docs/CLAIM_BENCHMARK.md @@ -0,0 +1,84 @@ +# Controlled 100-question claim and abstention benchmark + +## Purpose and scope + +GitHub Actions runs `scripts/benchmark_claim_abstention.cjs` against the latest +successful `Refresh data` **search-index artifact**, the same source used by +Pages. This is a **source-consistency stress test** for the question "Do +accepted CIViC records report sensitivity or resistance for gene–variant, +disease and therapy?" It is **not** a test of clinical truth, independent +expert adjudication, recommendation quality, or LLM-generated answer quality. + +The script deterministically constructs exactly **100 distinct questions** +from accepted predictive CIViC records. When the snapshot exports variant +membership, questions target **one recorded variant**. When membership is +unavailable, questions target the **exact complete molecular profile name** +instead; the program must not treat that name as proof about a single variant. +The mode and membership count are recorded in the report. It +adds counterfactual conditions (another variant, therapy, disease, opposite +effect) and, where available, complex molecular profiles. It checks every +counterfactual against all eligible source records before assigning its +source-derived label, so a swapped condition with genuine matching evidence +is not mistakenly treated as empty. Target quotas are 25 direct, 20 other +variant, 20 other therapy, 15 other cancer, 10 opposite effect and 10 complex +profile questions **when membership is available**. If a category has insufficient records, other categories +fill the set; the **actual** mixture appears in the report. If fewer than 100 +unique questions exist, CI fails rather than duplicating questions. + +The source reference enumerates raw accepted predictive CIViC records by exact +gene, disease, therapy and either verified single-variant membership or complete +profile name, depending on the available snapshot; a record has a matching +reported effect only when the direction is `SUPPORTS`. A reference question +is marked source-answerable if it has matching support and no supporting +report for the opposite effect or `DOES_NOT_SUPPORT` report of that effect. +This label is derived from the **same CIViC snapshot** as all methods and +is not an independent biomedical gold standard. Conflict and missing evidence +mean abstain **within this snapshot**, not absence of clinical benefit. + +## Strategies + +| Method | Answer condition | Deliberate limitation | +| --- | --- | --- | +| `context_blind` | Any accepted CIViC record for the gene, drug and disease | Ignores variant, direction and significance | +| `variant_only` | Any accepted CIViC record for the gene, variant/profile, drug and disease | Ignores predictive significance, direction and complex membership | +| `claim_scoped` | The structured claim check returns `supporting_records` | Requires exact reported effect and the mode's matching scope; abstains on opposing/disputing reports and incomplete input | + +The baselines are **ablations of OncoGraph's own retrieval**, not standalone +CIViC MCP, an external clinical service or an LLM. The source reference is +field-defined, so the rules of the proposed method align with it by design: +high scores only show consistency with the frozen source fields. In particular, +they cannot demonstrate a novel clinical conclusion or a performance gain on +an independently labeled question set. + +**Profile-mode results cannot establish variant-specific accuracy.** The +profile-name lookup is permitted only when the question explicitly asks about +the whole profile. For a single-variant question without membership, the claim +checker continues to abstain. The workflow cannot silently substitute one +meaning for the other. + +Metrics on the 100 questions: answer rate among source-answerable items; +false-answer rate among source-unanswerable items; answered count/coverage; +and error among answered. A tree with more than 200 matching records is flagged +and excluded from metrics for **all methods**, while remaining in the detailed +question list. There is no claim that a zero error rate generalizes outside +this bounded snapshot. + +## Reproduce and inspect + +```sh +node tests/test_claim_benchmark.cjs +node scripts/benchmark_claim_abstention.cjs --snapshot web/data --questions 100 --out claim-benchmark +``` + +`report.json` includes all 100 query texts, question types, source-derived +labels, EIDs per method, coverage, abstention counts and SHA-256 hashes of the +two input JSON files. CI publishes a summary and the report as an Actions +artifact whose name contains `profile` or `variant`, question count `Q`, +source-answerable `P`, source-unanswerable `U`, false answers for each strategy +(`B` context-blind, `V` variant-only, `C` claim-scoped) and claim-scoped +answers `Y`. The detailed JSON carries exact fractions and EIDs. Failed runs +instead put a brief error hint in the artifact name and upload the console log. The workflow +also runs on pull requests, manually, and after successful data refreshes. +To evaluate scientific novelty, independently review a held-out set of +question/claim/source tuples, evaluate claim-level citations and free-text +answers, and compare with appropriately configured CIViC MCP and LLM baselines. diff --git a/docs/CLAIM_EVIDENCE.md b/docs/CLAIM_EVIDENCE.md index d17700e..b30bd2a 100644 --- a/docs/CLAIM_EVIDENCE.md +++ b/docs/CLAIM_EVIDENCE.md @@ -8,15 +8,18 @@ or patient suitability. It does not yet verify free-text LLM answers. Open **More tools → Review variant evidence → Check a variant claim** and enter an exact gene, variant, disease, therapy, and whether to check a report of -`sensitivity` or `resistance`. Download the structured JSON and follow the +`sensitivity` or `resistance`. If using the whole molecular profile name, +explicitly choose **profile** scope; `variant` remains the conservative default. +An exact whole-profile match does not prove a finding for an individual member +variant. Download the structured JSON and follow the CIViC source URLs to verify the records. Therapy may be an exact name or NCIt ID **as stored in this snapshot**. No cross-source alias mapping is performed. Against a locally imported database, the same algorithm is exposed through: ```text -GET /claim-evidence?gene=EGFR&variant=T790M&disease=&therapy=&effect=resistance -MCP check_variant_claim(gene, variant, disease, therapy, effect, limit=20) +GET /claim-evidence?gene=EGFR&variant=T790M&disease=&therapy=&effect=resistance&scope=variant +MCP check_variant_claim(gene, variant, disease, therapy, effect, limit=20, scope="variant") ``` The browser reads GitHub Pages JSON. The Python API and local MCP read their @@ -28,7 +31,10 @@ First use `buildEvidenceTree` / `get_evidence_tree` with a 200-Evidence-Item cap. This already requires the exact gene and disease, a matching variant member (or flagged legacy profile-name match), accepted CIViC status, and preserves the original evidence ID and claim direction. Then require a therapy -matched by exact stored name or ID and `PREDICTIVE` evidence. Other relation +matched by exact stored name or ID and `PREDICTIVE` evidence. With explicit +`scope=profile`, an exact whole-profile-name match can support a claim about +that **whole profile** even when member variants were not exported; it never +establishes a claim about an individual variant. Other relation types, including drug `targets` gene, trial registrations, pathway membership, and target–disease associations, **cannot** support a variant–therapy effect. @@ -53,6 +59,12 @@ produce `context_only` for profile-name matches. ## Evaluation before publication claims +An automated [100-question benchmark](CLAIM_BENCHMARK.md) runs in GitHub +Actions after a data refresh or on a change to the benchmark. It compares +context-blind, variant-only and claim-scoped retrieval on the **same** frozen +snapshot. Its source-derived labels test internal consistency, not medical +accuracy, model-written answers or independent expert agreement. + This is a **rule-based baseline**, not a demonstrated improvement or a validated algorithm. The current [variant retrieval experiment](VARIANT_RETRIEVAL_EXPERIMENT.md) evaluates Evidence Item retrieval, not this claim classifier or LLM responses. diff --git a/docs/MCP.md b/docs/MCP.md index fc0d460..1c74d28 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -2,7 +2,7 @@ The local stdio MCP server also exposes `get_evidence_tree(gene, variant, disease, jurisdiction="US", limit=50)`. It returns accepted CIViC evidence with source links, preserves contradictory claims, and marks US regulatory labels and CIViC Assertions as not integrated. See [Evidence Tree v1](EVIDENCE_TREE.md) for exact inputs and response fields. GitHub Pages does not serve MCP. -`check_variant_claim(gene, variant, disease, therapy, effect, limit=20)` classifies accepted CIViC predictive items for exact stored therapy and a reported `sensitivity` or `resistance` effect. It returns separate supporting, opposing, disputing and context-only records and abstains on incomplete profile membership or truncated input. See [claim check contract](CLAIM_EVIDENCE.md). It never converts a drug–target–disease path into a therapeutic claim. +`check_variant_claim(gene, variant, disease, therapy, effect, limit=20, scope="variant")` classifies accepted CIViC predictive items for exact stored therapy and a reported `sensitivity` or `resistance` effect. It returns separate supporting, opposing, disputing and context-only records and abstains on incomplete variant membership or truncated input. Explicit `scope="profile"` checks the exact complete molecular profile name, never a single variant. See [claim check contract](CLAIM_EVIDENCE.md). It never converts a drug–target–disease path into a therapeutic claim. **Research use only.** OncoGraph's MCP server exposes structured oncology knowledge-graph data -- entities, relations, and their evidence -- to MCP diff --git a/scripts/benchmark_claim_abstention.cjs b/scripts/benchmark_claim_abstention.cjs new file mode 100644 index 0000000..04317d1 --- /dev/null +++ b/scripts/benchmark_claim_abstention.cjs @@ -0,0 +1,230 @@ +#!/usr/bin/env node +// A 100-question source-consistency stress test, NOT independent clinical gold. +const fs = require("node:fs"); +const path = require("node:path"); +const crypto = require("node:crypto"); +const { buildEvidenceTree } = require("../web/evidence-tree.js"); +const { evaluateEvidenceClaim } = require("../web/claim-evidence.js"); +const { contextBlindIds } = require("./evaluate_variant_questions.cjs"); + +const normal = (value) => String(value || "").trim().toLowerCase(); +function context(row) { + if (row.context && typeof row.context === "object") return row.context; + try { return JSON.parse(row.context || "{}"); } catch { return {}; } +} +function memberName(member, gene) { + const value = String(member).trim(); + return normal(value).startsWith(`${normal(gene)} `) ? value.slice(gene.length + 1) : value; +} +function key(q) { + return [q.gene, q.variant, q.disease, q.therapy, q.effect].map(normal).join("|"); +} + +// Independently enumerate upstream-shaped CIViC records (before calling either +// retrieval method). This reference is still source-derived, not expert gold. +function readSourceRecords(entities, relations) { + const byId = new Map(entities.map((entity) => [String(entity.id), entity])); + const rows = []; + const inspected = { clinical_relations: 0, drug_disease_relations: 0, civic_items: 0, + accepted_items: 0, predictive_items: 0, valid_effect_items: 0, variant_member_items: 0 }; + for (const relation of relations) { + if (relation.predicate !== "clinically_evidenced_for") continue; + inspected.clinical_relations++; + const drug = byId.get(String(relation.subject_id)); + const cancer = byId.get(String(relation.object_id)); + if (normal(drug?.type) !== "drug" || normal(cancer?.type) !== "disease") continue; + inspected.drug_disease_relations++; + for (const item of relation.evidence || []) { + const ctx = context(item); + if (normal(item.source) !== "civic") continue; + inspected.civic_items++; + if (ctx.civic_review_status !== "ACCEPTED") continue; + inspected.accepted_items++; + if (normal(ctx.civic_evidence_type) !== "predictive") continue; + inspected.predictive_items++; + if (!["sensitivity", "resistance"].includes(normal(ctx.significance))) continue; + inspected.valid_effect_items++; + if (!ctx.civic_evidence_id || !ctx.gene_name || !ctx.molecular_profile) continue; + if (Array.isArray(ctx.profile_variants) && ctx.profile_variants.length) inspected.variant_member_items++; + rows.push({ id: String(ctx.civic_evidence_id), gene: String(ctx.gene_name), + profile_name: String(ctx.molecular_profile), + members: Array.isArray(ctx.profile_variants) ? ctx.profile_variants.map((member) => + memberName(member, ctx.gene_name)) : [], + disease: cancer.name, therapy: drug.name, effect: normal(ctx.significance), + direction: normal(item.claim_state) }); + } + } + return { rows, byId, inspected }; +} + +function referenceIndex(records, scope = "variant") { + const index = new Map(); + for (const row of records) { + if (scope === "variant" && row.members.length !== 1) continue; + const name = scope === "profile" ? row.profile_name : row.members[0]; + const base = [row.gene, name, row.disease, row.therapy].map(normal).join("|"); + if (!index.has(base)) index.set(base, []); + index.get(base).push(row); + } + return index; +} +function sourceReference(q, recordsOrIndex) { + const supporting = new Set(), opposing = new Set(), disputed = new Set(); + const index = recordsOrIndex instanceof Map ? recordsOrIndex : referenceIndex(recordsOrIndex, q.scope); + const base = [q.gene, q.variant, q.disease, q.therapy].map(normal).join("|"); + for (const row of index.get(base) || []) { + if (row.direction === "supports" && row.effect === q.effect) supporting.add(row.id); + else if (row.direction === "supports") opposing.add(row.id); + else if (row.direction === "contradicts" && row.effect === q.effect) disputed.add(row.id); + } + return { answerable: supporting.size > 0 && !opposing.size && !disputed.size, + supporting_eids: [...supporting].sort(), opposing_count: opposing.size, disputed_count: disputed.size }; +} + +function makeQuestions(records, count = 100, scope = "variant") { + if (!Number.isInteger(count) || count < 1 || count > 1000) throw Error("Invalid question count"); + if (!["variant", "profile"].includes(scope)) throw Error("Invalid scope"); + const ordered = records.filter((row) => (scope === "profile" || row.members.length === 1) && + row.direction === "supports") + .sort((a, b) => [a.gene, a.disease, a.therapy, a.members[0], a.effect, a.id].join("|") + .localeCompare([b.gene, b.disease, b.therapy, b.members[0], b.effect, b.id].join("|"))); + if (!ordered.length) throw Error("No eligible accepted CIViC predictive records in snapshot"); + const genes = new Map(); + for (const row of ordered) { + if (!genes.has(row.gene)) genes.set(row.gene, []); + genes.get(row.gene).push(row); + } + const seed = []; + while (seed.length < 300 && [...genes.values()].some((list) => list.length)) { + for (const list of genes.values()) if (list.length && seed.length < 300) seed.push(list.shift()); + } + const index = referenceIndex(records, scope); + const variantsByGene = new Map(); + for (const row of records) for (const member of scope === "profile" ? [row.profile_name] : row.members) { + if (!variantsByGene.has(row.gene)) variantsByGene.set(row.gene, new Set()); + variantsByGene.get(row.gene).add(member); + } + const therapies = [...new Set(records.map((row) => row.therapy))].sort(); + const cancers = [...new Set(records.map((row) => row.disease))].sort(); + const pools = Object.fromEntries(["direct", "other_variant", "other_therapy", "other_cancer", + "opposite_effect", "complex_profile"].map((name) => [name, new Map()])); + const add = (category, q, expected) => { + if (pools[category].size >= 250) return; + const reference = sourceReference(q, index); + if (reference.answerable !== expected) return; + const id = key(q); + if (!pools[category].has(id)) pools[category].set(id, { ...q, category, reference }); + }; + // These are perturbations of real source rows. Negatives are checked + // against the full available source record set, never presumed absent. + for (const row of seed) { + const base = { gene: row.gene, variant: scope === "profile" ? row.profile_name : row.members[0], + disease: row.disease, therapy: row.therapy, effect: row.effect, scope }; + add("direct", base, true); + add("opposite_effect", { ...base, effect: row.effect === "sensitivity" ? "resistance" : "sensitivity" }, false); + for (const variant of [...(variantsByGene.get(row.gene) || [])].slice(0, 15)) { + if (normal(variant) !== normal(base.variant)) add("other_variant", { ...base, variant }, false); + } + for (const therapy of therapies.slice(0, 20)) { + if (normal(therapy) !== normal(base.therapy)) add("other_therapy", { ...base, therapy }, false); + } + for (const disease of cancers.slice(0, 20)) { + if (normal(disease) !== normal(base.disease)) add("other_cancer", { ...base, disease }, false); + } + } + for (const row of scope === "variant" ? records.filter((item) => item.members.length > 1) : []) { + for (const variant of row.members) { + add("complex_profile", { gene: row.gene, variant, disease: row.disease, + therapy: row.therapy, effect: row.effect, scope }, false); + } + } + const quotas = { direct: 25, other_variant: 20, other_therapy: 20, + other_cancer: 15, opposite_effect: 10, complex_profile: 10 }; + const selected = new Map(); + for (const [category, size] of Object.entries(quotas)) { + for (const [id, q] of [...pools[category]].slice(0, size)) if (!selected.has(id)) selected.set(id, q); + } + // Rare source distributions can lack a category. Fill the remainder + // deterministically and report actual category counts instead of faking it. + for (const category of Object.keys(pools)) { + for (const [id, q] of pools[category]) { + if (selected.size >= count) break; + if (!selected.has(id)) selected.set(id, q); + } + } + if (selected.size < count) throw Error(`Only ${selected.size} unique eligible questions; need ${count}`); + return [...selected.values()].slice(0, count).map((q, i) => ({ id: `claim-${String(i + 1).padStart(3, "0")}`, + question: `In ${q.disease}, do accepted CIViC records report ${q.effect} for ${scope === "profile" ? "the complete molecular profile" : "the single variant"} ${q.gene} ${q.variant} with ${q.therapy}?`, ...q })); +} + +function metrics(results, strategy) { + const valid = results.filter((row) => !row.tree_truncated); + const positive = valid.filter((row) => row.reference.answerable); + const negative = valid.filter((row) => !row.reference.answerable); + const saidYes = valid.filter((row) => row.predictions[strategy].answers); + const trueYes = positive.filter((row) => row.predictions[strategy].answers); + const falseYes = negative.filter((row) => row.predictions[strategy].answers); + return { evaluated_count: valid.length, source_answerable_count: positive.length, + source_unanswerable_count: negative.length, answered_count: saidYes.length, + answer_coverage: valid.length ? saidYes.length / valid.length : null, + answer_rate_on_answerable: positive.length ? trueYes.length / positive.length : null, + false_answer_rate_on_unanswerable: negative.length ? falseYes.length / negative.length : null, + selective_error_among_answers: saidYes.length ? falseYes.length / saidYes.length : null }; +} + +function evaluate(entities, relations, count = 100) { + const { rows: source, byId, inspected } = readSourceRecords(entities, relations); + if (!source.length) throw Error(`Missing eligible CIViC: c${inspected.civic_items}a${inspected.accepted_items}p${inspected.predictive_items}e${inspected.valid_effect_items}m${inspected.variant_member_items}`); + const scope = inspected.variant_member_items ? "variant" : "profile"; + const questions = makeQuestions(source, count, scope); + const results = questions.map((q) => { + const tree = buildEvidenceTree(entities, relations, { gene: q.gene, + variant: q.variant, disease: q.disease, limit: 200 }); + const claim = evaluateEvidenceClaim(tree, { therapy: q.therapy, effect: q.effect, + scope, limit: 200 }); + const blindIds = contextBlindIds(q, relations, byId); + const scopedIds = tree.evidence.filter((item) => item.therapies.some((drug) => + normal(drug.name) === normal(q.therapy))).map((item) => item.evidence_id); + return { id: q.id, category: q.category, question: q.question, + query: { gene: q.gene, variant: q.variant, disease: q.disease, + therapy: q.therapy, effect: q.effect, scope }, + reference: q.reference, tree_truncated: tree.truncated, + predictions: { context_blind: { answers: blindIds.length > 0, evidence_ids: blindIds }, + variant_only: { answers: scopedIds.length > 0 && !tree.truncated, evidence_ids: scopedIds }, + claim_scoped: { answers: claim.status === "supporting_records", status: claim.status, + evidence_ids: claim.evidence.supporting.map((item) => item.evidence_id) } } }; + }); + return { methodology: "source-derived controlled perturbation; not independent clinical gold or LLM evaluation", + scope, scope_note: scope === "profile" + ? "No exported variant membership: all 100 questions concern exact whole molecular profiles, not single variants." + : "Single-variant membership exported; whole-profile-name-only records are never treated as single-variant proof.", + source_coverage: inspected, + question_count: results.length, + categories: Object.fromEntries(Object.keys({ direct: 1, other_variant: 1, other_therapy: 1, + other_cancer: 1, opposite_effect: 1, complex_profile: 1 }).map((category) => + [category, results.filter((row) => row.category === category).length])), + truncated_count: results.filter((row) => row.tree_truncated).length, + metrics: Object.fromEntries(["context_blind", "variant_only", "claim_scoped"].map((strategy) => + [strategy, metrics(results, strategy)])), questions: results }; +} + +if (require.main === module) { + const args = process.argv.slice(2); + const value = (key, fallback) => args.includes(key) ? args[args.indexOf(key) + 1] : fallback; + const dir = value("--snapshot", "web/data"); + const out = value("--out", "claim-benchmark"); + const count = Number(value("--questions", "100")); + const names = ["search-index.json", "relations.json"]; + const raw = names.map((name) => fs.readFileSync(path.join(dir, name))); + const report = evaluate(...raw.map((item) => JSON.parse(item.toString("utf8"))), count); + report.snapshot_sha256 = Object.fromEntries(names.map((name, i) => + [name, crypto.createHash("sha256").update(raw[i]).digest("hex")])); + fs.mkdirSync(out, { recursive: true }); + fs.writeFileSync(path.join(out, "report.json"), JSON.stringify(report, null, 2) + "\n"); + process.stdout.write(JSON.stringify({ question_count: report.question_count, + scope: report.scope, source_coverage: report.source_coverage, + categories: report.categories, truncated_count: report.truncated_count, + metrics: report.metrics, snapshot_sha256: report.snapshot_sha256 }, null, 2) + "\n"); +} + +module.exports = { evaluate, makeQuestions, readSourceRecords, sourceReference }; diff --git a/src/oncograph/claim_evidence.py b/src/oncograph/claim_evidence.py index 1e72c02..2b73e32 100644 --- a/src/oncograph/claim_evidence.py +++ b/src/oncograph/claim_evidence.py @@ -10,7 +10,7 @@ def evaluate_evidence_claim(tree: dict[str, Any], *, therapy: str, effect: str, - limit: int = 20) -> dict[str, Any]: + scope: str = "variant", limit: int = 20) -> dict[str, Any]: """Classify accepted records without promoting graph paths to clinical claims. Only predictive evidence with an exact named therapy and a single-variant @@ -18,7 +18,7 @@ def evaluate_evidence_claim(tree: dict[str, Any], *, therapy: str, effect: str, An opposing significance is context, not necessarily a logical negation. """ if (not isinstance(therapy, str) or not therapy.strip() - or effect not in {"sensitivity", "resistance"} + or effect not in {"sensitivity", "resistance"} or scope not in {"variant", "profile"} or type(limit) is not int or not 1 <= limit <= 200): raise ValueError("Invalid claim query") if not isinstance(tree, dict) or not isinstance(tree.get("evidence"), list) or "query" not in tree: @@ -44,9 +44,13 @@ def normal(value: Any) -> str: excluded["unrelated_effect"] += 1 continue profile = item.get("profile") or {} - if profile.get("match_type") != "variant_member" or profile.get("complex_profile"): + exact_profile = (scope == "profile" and profile.get("match_type") == "profile_only" + and normal(profile.get("name")) == normal(tree["query"].get("variant"))) + if ((scope == "profile" and not exact_profile) + or (scope == "variant" and (profile.get("match_type") != "variant_member" + or profile.get("complex_profile")))): groups["context_only"].append({**item, "reason": - "Variant-specific effect unverified for this molecular profile"}) + "Requested variant or molecular profile is not verified by this record"}) excluded["incomplete_profile"] += 1 continue direction = normal(item.get("claim_state")) @@ -85,7 +89,7 @@ def normal(value: Any) -> str: else: status = "not_found_in_snapshot" return {"schema_version": "1.0", "query": {**tree["query"], "therapy": therapy.strip(), - "effect": effect}, + "effect": effect, "scope": scope}, "status": status, "counts": totals, "excluded": excluded, "incomplete": incomplete, "evidence": selected, "limitations": ["This checks a bounded CIViC snapshot, not treatment efficacy, FDA status or patient suitability.", @@ -94,8 +98,9 @@ def normal(value: Any) -> str: def get_claim_evidence(session: Session, *, gene: str, variant: str, disease: str, - therapy: str, effect: str, limit: int = 20) -> dict[str, Any]: + therapy: str, effect: str, scope: str = "variant", + limit: int = 20) -> dict[str, Any]: """Use the same capped Evidence Tree as the browser; flag incomplete input.""" return evaluate_evidence_claim(get_evidence_tree(session, gene=gene, variant=variant, disease=disease, limit=200), - therapy=therapy, effect=effect, limit=limit) + therapy=therapy, effect=effect, scope=scope, limit=limit) diff --git a/src/oncograph/main.py b/src/oncograph/main.py index 65669af..120b390 100644 --- a/src/oncograph/main.py +++ b/src/oncograph/main.py @@ -47,11 +47,12 @@ def evidence_tree(session: SessionDep, gene: str, variant: str, disease: str, @app.get("/claim-evidence") def claim_evidence(session: SessionDep, gene: str, variant: str, disease: str, - therapy: str, effect: str, limit: Annotated[int, Query(ge=1, le=200)] = 20): + therapy: str, effect: str, scope: str = "variant", + limit: Annotated[int, Query(ge=1, le=200)] = 20): """Classify scoped CIViC records; never assert patient benefit or FDA approval.""" try: return check_claim_from_db(session, gene=gene, variant=variant, disease=disease, - therapy=therapy, effect=effect, limit=limit) + therapy=therapy, effect=effect, scope=scope, limit=limit) except ValueError as exc: raise HTTPException(422, str(exc)) from exc diff --git a/src/oncograph/mcp/tools.py b/src/oncograph/mcp/tools.py index 5898a09..c9b24bf 100644 --- a/src/oncograph/mcp/tools.py +++ b/src/oncograph/mcp/tools.py @@ -73,12 +73,13 @@ def get_evidence_tree(gene: str, variant: str, disease: str, jurisdiction: str = def check_variant_claim(gene: str, variant: str, disease: str, therapy: str, - effect: str, limit: int = 20) -> dict[str, Any]: + effect: str, limit: int = 20, scope: str = "variant") -> dict[str, Any]: """Separate supporting, opposing and inconclusive CIViC records for one research claim.""" try: with session_scope() as session: return _check_claim_from_db(session, gene=gene, variant=variant, - disease=disease, therapy=therapy, effect=effect, limit=limit) + disease=disease, therapy=therapy, effect=effect, + scope=scope, limit=limit) except ValueError as exc: raise OncoGraphMCPError(str(exc)) from exc diff --git a/tests/test_claim_benchmark.cjs b/tests/test_claim_benchmark.cjs new file mode 100644 index 0000000..44b7e7b --- /dev/null +++ b/tests/test_claim_benchmark.cjs @@ -0,0 +1,75 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const { evaluate, makeQuestions, readSourceRecords } = require("../scripts/benchmark_claim_abstention.cjs"); + +function fixture() { + const entities = []; + const relations = []; + let eid = 1; + for (let gene = 0; gene < 3; gene++) { + for (let variant = 0; variant < 6; variant++) { + for (let drug = 0; drug < 3; drug++) { + const drugId = `drug-${drug}`; + const cancerId = `cancer-${gene}`; + if (!entities.some((item) => item.id === drugId)) entities.push({ id: drugId, type: "drug", name: `Drug ${drug}` }); + if (!entities.some((item) => item.id === cancerId)) entities.push({ id: cancerId, + type: "disease", name: `Cancer ${gene}`, canonical_id: `doid:${gene}` }); + const member = `GENE${gene} V${variant}`; + relations.push({ subject_id: drugId, object_id: cancerId, predicate: "clinically_evidenced_for", + evidence: [{ source: "civic", claim_state: "SUPPORTS", source_id: `EID${eid}`, + context: { civic_review_status: "ACCEPTED", civic_evidence_id: eid, + molecular_profile_id: eid, molecular_profile: member, profile_variants: [member], + gene_name: `GENE${gene}`, civic_evidence_type: "PREDICTIVE", + significance: variant % 2 ? "RESISTANCE" : "SENSITIVITY" } }] }); + eid++; + } + } + } + entities.push({ id: "complex-drug", type: "drug", name: "Combo-only" }); + for (let i = 0; i < 12; i++) { + const gene = i % 3; + relations.push({ subject_id: "complex-drug", object_id: `cancer-${gene}`, + predicate: "clinically_evidenced_for", evidence: [{ source: "civic", claim_state: "SUPPORTS", + context: { civic_review_status: "ACCEPTED", civic_evidence_id: eid, + molecular_profile_id: eid, molecular_profile: `GENE${gene} V${i} + V${i + 1}`, + profile_variants: [`GENE${gene} V${i}`, `GENE${gene} V${i + 1}`], + gene_name: `GENE${gene}`, civic_evidence_type: "PREDICTIVE", significance: "SENSITIVITY" } }] }); + eid++; + } + return { entities, relations }; +} + +test("100 unique snapshot-derived questions compare two ablations and safe abstention", () => { + const { entities, relations } = fixture(); + const source = readSourceRecords(entities, relations).rows; + assert.equal(makeQuestions(source).length, 100); + const report = evaluate(entities, relations); + assert.equal(new Set(report.questions.map((row) => JSON.stringify(row.query))).size, 100); + assert(report.categories.direct > 0); + assert(report.categories.other_variant > 0); + assert(report.categories.complex_profile > 0); + assert(report.metrics.context_blind.false_answer_rate_on_unanswerable > 0); + assert(report.metrics.variant_only.false_answer_rate_on_unanswerable > 0); + assert.equal(report.metrics.claim_scoped.false_answer_rate_on_unanswerable, 0); + assert.equal(report.metrics.claim_scoped.answer_rate_on_answerable, 1); + assert.equal(report.truncated_count, 0); +}); + +test("fails rather than reporting 100 questions if there are too few unique source perturbations", () => { + const { entities, relations } = fixture(); + assert.throws(() => evaluate(entities, relations, 1000), /unique eligible questions/); +}); + +test("legacy snapshot runs 100 profile-level questions without inventing variant membership", () => { + const { entities, relations } = fixture(); + for (const relation of relations) for (const item of relation.evidence) { + delete item.context.profile_variants; + } + const report = evaluate(entities, relations); + assert.equal(report.scope, "profile"); + assert.equal(report.source_coverage.variant_member_items, 0); + assert.equal(report.question_count, 100); + assert(report.questions.every((row) => row.query.scope === "profile")); + assert.equal(report.metrics.claim_scoped.false_answer_rate_on_unanswerable, 0); + assert.equal(report.metrics.claim_scoped.answer_rate_on_answerable, 1); +}); diff --git a/tests/test_claim_evidence.cjs b/tests/test_claim_evidence.cjs index 21b121a..940d319 100644 --- a/tests/test_claim_evidence.cjs +++ b/tests/test_claim_evidence.cjs @@ -58,3 +58,17 @@ test("truncation and display caps force incomplete review; invalid claims are re assert.throws(() => evaluateEvidenceClaim(tree, { therapy: "", effect: "sensitivity" })); assert.throws(() => evaluateEvidenceClaim(tree, { therapy: "Osimertinib", effect: "response" })); }); + +test("legacy snapshot can answer exact whole-profile question while single variant still abstains", () => { + const tree = buildEvidenceTree(entities, [relation("t", 9, "SENSITIVITY", "SUPPORTS", [])], + { gene: "EGFR", variant: "EGFR T790M", disease: "Lung cancer" }); + assert.equal(evaluateEvidenceClaim(tree, { therapy: "Osimertinib", effect: "sensitivity" }).status, + "insufficient_variant_context"); + const profile = evaluateEvidenceClaim(tree, { therapy: "Osimertinib", effect: "sensitivity", scope: "profile" }); + assert.equal(profile.status, "supporting_records"); + assert.equal(profile.query.scope, "profile"); + const other = { ...tree, evidence: [{ ...tree.evidence[0], profile: { + ...tree.evidence[0].profile, name: "EGFR C797S" } }] }; + assert.equal(evaluateEvidenceClaim(other, { therapy: "Osimertinib", effect: "sensitivity", scope: "profile" }).status, + "insufficient_variant_context"); +}); diff --git a/tests/test_claim_evidence.py b/tests/test_claim_evidence.py index 0133baf..068d2e7 100644 --- a/tests/test_claim_evidence.py +++ b/tests/test_claim_evidence.py @@ -38,3 +38,20 @@ def item(eid, significance, members, state="SUPPORTS"): assert evaluate_evidence_claim({**tree, "truncated": True}, therapy="ncit:1", effect="sensitivity")["status"] == "incomplete_review" with pytest.raises(ValueError): evaluate_evidence_claim(tree, therapy="ncit:1", effect="efficacy") + + +def test_whole_profile_is_distinct_from_unverified_single_variant(): + tree = {"query": {"gene": "EGFR", "variant": "EGFR T790M", "disease": "Lung cancer"}, + "truncated": False, "evidence": [ + {"evidence_id": "9", "profile": {"name": "EGFR T790M", "match_type": "profile_only", + "complex_profile": False}, + "evidence_type": "PREDICTIVE", "significance": "SENSITIVITY", + "claim_state": "supports", "therapies": [{"name": "Osimertinib"}]}, + ]} + assert evaluate_evidence_claim(tree, therapy="Osimertinib", effect="sensitivity")["status"] == "insufficient_variant_context" + assert evaluate_evidence_claim(tree, therapy="Osimertinib", effect="sensitivity", + scope="profile")["status"] == "supporting_records" + other = {**tree, "evidence": [{**tree["evidence"][0], "profile": { + **tree["evidence"][0]["profile"], "name": "EGFR C797S"}}]} + assert evaluate_evidence_claim(other, therapy="Osimertinib", effect="sensitivity", + scope="profile")["status"] == "insufficient_variant_context" diff --git a/web/claim-evidence.js b/web/claim-evidence.js index 69b1171..3900b2d 100644 --- a/web/claim-evidence.js +++ b/web/claim-evidence.js @@ -1,7 +1,8 @@ // A conservative, deterministic classifier of *retrieved CIViC records*. // Statuses describe the snapshot only; none are treatment recommendations. -function evaluateEvidenceClaim(tree, { therapy, effect, limit = 20 }) { +function evaluateEvidenceClaim(tree, { therapy, effect, scope = "variant", limit = 20 }) { if (typeof therapy !== "string" || !therapy.trim() || !["sensitivity", "resistance"].includes(effect) + || !["variant", "profile"].includes(scope) || !Number.isInteger(limit) || limit < 1 || limit > 200) throw new Error("Invalid claim query"); if (!tree || !Array.isArray(tree.evidence) || !tree.query) throw new Error("Invalid evidence tree"); const normal = (value) => String(value || "").trim().toLowerCase(); @@ -16,8 +17,11 @@ function evaluateEvidenceClaim(tree, { therapy, effect, limit = 20 }) { if (!["sensitivity", "resistance"].includes(significance)) { counts.unrelated_effect++; continue; } // An item with incomplete membership or a multi-variant profile cannot // prove that this single variant has the reported effect. - if (item.profile?.match_type !== "variant_member" || item.profile?.complex_profile) { - groups.context_only.push({ ...item, reason: "Variant-specific effect unverified for this molecular profile" }); + const exactProfile = scope === "profile" && item.profile?.match_type === "profile_only" && + normal(item.profile.name) === normal(tree.query.variant); + if ((scope === "profile" && !exactProfile) || + (scope === "variant" && (item.profile?.match_type !== "variant_member" || item.profile?.complex_profile))) { + groups.context_only.push({ ...item, reason: "Requested variant or molecular profile is not verified by this record" }); counts.incomplete_profile++; continue; } const direction = normal(item.claim_state); @@ -45,7 +49,7 @@ function evaluateEvidenceClaim(tree, { therapy, effect, limit = 20 }) { else if (totals.supporting) status = "supporting_records"; else if (totals.opposing || totals.disputed) status = "opposing_or_disputed_records"; else if (totals.context_only) status = "insufficient_variant_context"; - return { schema_version: "1.0", query: { ...tree.query, therapy: therapy.trim(), effect }, + return { schema_version: "1.0", query: { ...tree.query, therapy: therapy.trim(), effect, scope }, status, counts: totals, excluded: counts, incomplete, evidence: selected, limitations: ["This checks a bounded CIViC snapshot, not treatment efficacy, FDA status or patient suitability.", "An opposing report is not necessarily a direct contradiction; study contexts may differ.", diff --git a/web/evidence-tree.js b/web/evidence-tree.js index ba9d903..d1e8bdd 100644 --- a/web/evidence-tree.js +++ b/web/evidence-tree.js @@ -76,6 +76,15 @@ function renderEvidenceTree(entities, relations) { therapy.name = "therapy"; therapy.placeholder = "e.g. osimertinib (optional)"; therapyLabel.append("Therapy ", therapy); form.append(therapyLabel); + const scopeLabel = document.createElement("label"); + const scope = document.createElement("select"); + for (const [value, label] of [["variant", "Single variant (verified membership)"], + ["profile", "Exact complete CIViC molecular profile name"]]) { + const option = document.createElement("option"); option.value = value; option.textContent = label; + scope.append(option); + } + scopeLabel.append("Interpret input as ", scope); + form.append(scopeLabel); const effectLabel = document.createElement("label"); const effect = document.createElement("select"); for (const value of ["sensitivity", "resistance"]) { @@ -94,10 +103,11 @@ function renderEvidenceTree(entities, relations) { { gene: inputs[0].value, variant: inputs[1].value, disease: inputs[2].value, limit: therapy.value.trim() ? 200 : 50 }); result.replaceChildren(); - const claim = therapy.value.trim() ? evaluateEvidenceClaim(tree, { therapy: therapy.value, effect: effect.value }) : null; + const claim = therapy.value.trim() ? evaluateEvidenceClaim(tree, + { therapy: therapy.value, effect: effect.value, scope: scope.value }) : null; const summary = document.createElement("p"); summary.textContent = claim - ? `Snapshot check: ${claim.status.replaceAll("_", " ")}. ${claim.counts.supporting} supporting; ${claim.counts.opposing} opposing reports; ${claim.counts.disputed} disputing the stated effect; ${claim.counts.context_only} context only. ${claim.incomplete ? "Results are incomplete. " : ""}This is not a treatment recommendation. Exact therapy name or ID required.` + ? `Snapshot check (${claim.query.scope === "profile" ? "whole molecular profile" : "single variant"}): ${claim.status.replaceAll("_", " ")}. ${claim.counts.supporting} supporting; ${claim.counts.opposing} opposing reports; ${claim.counts.disputed} disputing the stated effect; ${claim.counts.context_only} context only. ${claim.incomplete ? "Results are incomplete. " : ""}This is not a treatment recommendation. Exact therapy name or ID required.` : `${tree.total_evidence_items} accepted CIViC evidence item(s) in this snapshot. FDA labels and CIViC assertions: not integrated. No match does not mean no evidence exists.`; result.append(summary); const rows = claim ? Object.entries(claim.evidence).flatMap(([group, items]) => items.map((item) => ({ group, item }))) diff --git a/web/index.html b/web/index.html index baa4b92..1d5ca18 100644 --- a/web/index.html +++ b/web/index.html @@ -94,7 +94,7 @@

Review variant evidence

Loading variant evidence…

Check a variant claim or inspect an Evidence Tree -

Enter an exact CIViC gene, variant and cancer name or DOID. Optionally add a therapy to separate supporting, opposing and inconclusive reports of sensitivity or resistance. This checks recorded evidence, not treatment efficacy. FDA labels and CIViC Assertions are not integrated.

+

Enter an exact CIViC gene, variant or complete molecular profile name, and cancer name or DOID. Optionally add a therapy and choose the input scope to separate reported sensitivity and resistance. A profile-name match does not prove an individual variant's effect. FDA labels and CIViC Assertions are not integrated.

Loading evidence…