阶段 3.5：外部验证闸门准备 + M 系列审查落地

.gitignore

@@ -9,3 +9,10 @@ __pycache__/
 # 编辑器
 .vscode/
 .idea/
+
+# Python 环境与缓存（M11）
+.venv/
+.pytest_cache/
+
+# 阶段 3.5 闸门的外部数据（scripts/gate/build_gate_data.py 下载，约 110 MB）
+data/gate/

app/core/service.py

@@ -46,7 +46,8 @@ def analyze(
 ) -> AnalysisResult:
     """对一个 CSV 数据集回答一个自然语言问题。
 
-    dataset：CSV 路径（同目录下的同名 .dict.csv 会自动作为数据字典）。
+    dataset：CSV 路径（同目录下的同名 .dict.csv 会自动作为数据字典）；
+             或目录 / CSV 路径列表 = 多表数据集（代码里用 dfs['表名']，见 app/tools/dataset.py）。
     agent："workflow"（固定状态图，带 reviewer 纠错）或 "react"（模型自主调用工具）。
     llm_config：模型、Key、超时、重试配置；None 时从环境变量读取。
     chat_model：直接注入 LangChain ChatModel（测试/离线评测用），此时不需要 API Key。
@@ -62,8 +63,10 @@ def analyze(
         raise ValueError(f"未知 agent: {agent!r}，可选 {AGENTS}")
 
     t0 = time.perf_counter()
-    dataset = os.fspath(dataset)
-    res = AnalysisResult(agent=agent, dataset=dataset, question=question)
+    from app.tools.dataset import dataset_label
+    if not isinstance(dataset, (list, tuple)):
+        dataset = os.fspath(dataset)
+    res = AnalysisResult(agent=agent, dataset=dataset_label(dataset), question=question)
     tracker = UsageTracker()
 
     try:
@@ -78,25 +81,24 @@ def analyze(
             res.usage.retries = None     # ReAct 的重试在 openai SDK 内部完成，无法计数
     if log:
         from app.tools.persist import log_run
-        log_run({**res.to_record(), "df_path": dataset})
+        log_run({**res.to_record(), "df_path": res.dataset})
     return res
 
 
 def _run(res, tracker, dataset, question, *, agent, llm_config, chat_model, max_attempts,
          exec_timeout, recursion_limit, use_dictionary, memory, thread_id, encoding):
-    from app.tools.csv_io import read_csv_with_encoding
+    from app.tools.dataset import describe_data, load_data
     from app.tools.dictionary import format_notes, load_retriever, retrieve_notes
-    from app.tools.schema import describe_df
 
     # 1. 数据：只读一次
     t = time.perf_counter()
     try:
-        df, enc = read_csv_with_encoding(dataset, encoding)
+        df, load_desc = load_data(dataset, encoding)   # 单表 DataFrame 或多表 {表名: DataFrame}
     except Exception as e:  # noqa: BLE001  文件不存在 / 编码无法识别 / CSV 解析失败
         res.error, res.error_kind = f"数据集读取失败：{type(e).__name__}: {e}", "dataset"
         res.steps.append(Step("load", error=res.error))
         return
-    schema = describe_df(df)
+    schema = describe_data(df)
     notes = []
     if use_dictionary:
         try:
@@ -105,8 +107,7 @@ def _run(res, tracker, dataset, question, *, agent, llm_config, chat_model, max_
             res.steps.append(Step("dictionary", error=f"数据字典读取失败，已跳过：{e}"))
     res.field_notes = notes
     res.timings["load"] = round(time.perf_counter() - t, 4)
-    res.steps.append(Step("load", content=f"{len(df)} 行 × {len(df.columns)} 列，编码 {enc}；"
-                                          f"字典命中 {len(notes)} 条",
+    res.steps.append(Step("load", content=f"{load_desc}；字典命中 {len(notes)} 条",
                           duration_s=res.timings["load"]))
 
     # 2. 模型
@@ -129,8 +130,10 @@ def _run_workflow(res, tracker, cfg, chat_model, df, schema, notes_text, questio
                   max_attempts, exec_timeout):
     client = LLMClient(cfg, model=chat_model if chat_model is not None
                        else build_chat_model(cfg, sdk_retries=0), tracker=tracker)
+    from app.tools.dataset import access_rule, dataset_label, review_access_rule
     state = {
-        "question": question, "df_path": dataset, "schema": schema,
+        "question": question, "df_path": dataset_label(dataset), "schema": schema,
+        "data_access": access_rule(df), "data_access_review": review_access_rule(df),
         "field_notes": notes_text, "attempts": 0, "max_attempts": max_attempts,
         "exec_timeout": exec_timeout,
     }

app/eval/comparator.py

@@ -104,6 +104,85 @@ def results_equal(actual, expected, float_tol=1e-4, *, ordered=None, percent_equ
     return False
 
 
+def match_kind(actual, expected, float_tol=1e-4, *, ordered=None, percent_equiv=False):
+    """判对时靠的是哪一层（M02：报告逐题标出，放宽层判对的题供人工抽审）。
+    返回 None（不相等）/ "strict"（形态与数值都直接相等；不要求顺序的题先按标签排序）/
+    "rounded"（同上，但实际值先舍入到期望的小数位）/ "scaled"（靠 ×100 或 ÷100，规则 12）/
+    "extras"（靠规则 20：答案裹在附加信息里）/ "relaxed"（靠规则 1–19 的其他放宽）。"""
+    if ordered is None:
+        ordered = looks_ranked(expected)
+    a, e = _py(actual), _py(expected)
+    for tier, rnd in (("strict", False), ("rounded", True)):
+        try:
+            if _strict_eq(_sorted(a, ordered), _sorted(e, ordered), float_tol, rnd):
+                return tier
+        except Exception:
+            pass
+    variants = [(a, None)]
+    if percent_equiv:
+        variants += [(v, "scaled") for v in (_scale(a, 100), _scale(a, 0.01)) if v is not None]
+    for v, tag in variants:
+        try:
+            if _eq(v, e, float_tol, ordered):
+                return tag or "relaxed"
+        except Exception:
+            pass
+        try:
+            if _extras_eq(v, e, float_tol, ordered):
+                return tag or "extras"
+        except Exception:
+            continue
+    return None
+
+
+def _sorted(x, ordered):
+    if ordered or not isinstance(x, (pd.Series, pd.DataFrame)):
+        return x
+    return x.sort_index(key=lambda idx: idx.map(lambda v: str(v)))
+
+
+def _strict_eq(a, e, tol, rnd=False) -> bool:
+    """最严口径：同一种形态，数值在容差内、标签逐字相同、顺序相同。"""
+    if isinstance(e, pd.DataFrame) or isinstance(e, pd.Series):
+        if type(a) is not type(e) or a.shape != e.shape:
+            return False
+        if [str(x) for x in a.index] != [str(x) for x in e.index]:
+            return False
+        if isinstance(e, pd.DataFrame):
+            if [str(x) for x in a.columns] != [str(x) for x in e.columns]:
+                return False
+            return all(_strict_values(a.iloc[:, i].tolist(), e.iloc[:, i].tolist(), tol, rnd)
+                       for i in range(e.shape[1]))
+        return _strict_values(a.tolist(), e.tolist(), tol, rnd)
+    if isinstance(e, dict):
+        return isinstance(a, dict) and [str(k) for k in a] == [str(k) for k in e] and \
+            _strict_values([_py(v) for v in a.values()], [_py(v) for v in e.values()], tol, rnd)
+    if isinstance(e, (list, tuple)):
+        return isinstance(a, (list, tuple)) and _strict_values([_py(v) for v in a], [_py(v) for v in e], tol, rnd)
+    return _strict_values([a], [e], tol, rnd)
+
+
+def _strict_values(xs, ys, tol, rnd=False) -> bool:
+    if len(xs) != len(ys):
+        return False
+    for x, y in zip(xs, ys):
+        x, y = _py(x), _py(y)
+        if isinstance(x, bool) or isinstance(y, bool):
+            if x != y:
+                return False
+        elif isinstance(x, (int, float)) and isinstance(y, (int, float)):
+            if _is_nan(x) and _is_nan(y):
+                continue
+            if rnd and isinstance(y, float) and not _is_nan(x):
+                d = _decimals(y)
+                x = round(x, d) if d is not None else x
+            if not abs(x - y) <= tol * max(1.0, abs(y)):
+                return False
+        elif str(x).strip() != str(y).strip():
+            return False
+    return True
+
+
 # ------------------------------------------------------------------ 规则 20（B46）
 _NUM_TOKEN_RE = re.compile(r"(?<![A-Za-z0-9_.])[-+]?\d+(?:,\d{3})*(?:\.\d+)?(?![A-Za-z0-9_.])")
 _PCT_TOKEN_RE = re.compile(r"([-+]?\d+(?:\.\d+)?)\s*%")

app/eval/report.py

@@ -13,6 +13,7 @@
 import platform
 import shutil
 import subprocess
+from collections import Counter
 from datetime import datetime
 
 from app.eval.cases import CATEGORY_MAP
@@ -118,6 +119,32 @@ def run_metadata(llm_config=None, data_path: str | None = None) -> dict:
     return meta
 
 
+EVALSET_FILES = ("app/eval/cases.py", "app/eval/cases_extra.py", "app/eval/case_model.py",
+                 "scripts/gen_eval_data.py")
+
+
+def evalset_fingerprint(datasets: dict | None = None) -> dict:
+    """M04：评测集指纹 = 题面代码 + 数据生成脚本 + 数据文件（含字典）。任何一处变了，sha256 就变，
+    两份报告只有 evalset.sha256 相同才可直接比较。CRLF 统一成 LF，Windows / Linux 算出同一个值。"""
+    if datasets is None:
+        from app.eval.cases import DATASETS
+        datasets = DATASETS
+    from app.tools.dictionary import dict_path_for
+    files = [os.path.join(_REPO, f) for f in EVALSET_FILES]
+    for path in datasets.values():
+        files += [path, dict_path_for(path)]
+    h, parts = hashlib.sha256(), {}
+    for f in sorted(set(files)):
+        if not os.path.exists(f):
+            continue
+        data = open(f, "rb").read().replace(b"\r\n", b"\n")
+        rel = os.path.relpath(f, _REPO).replace(os.sep, "/")
+        parts[rel] = hashlib.sha256(data).hexdigest()[:12]
+        h.update(rel.encode() + b"\0" + data + b"\0")
+    from app.eval.cases import ALL_CASES
+    return {"sha256": h.hexdigest()[:12], "n_cases": len(ALL_CASES), "files": parts}
+
+
 def datasets_metadata(datasets: dict) -> dict:
     """阶段 3：每张数据表的指纹，以及旁边有没有数据字典（B38：sample_eval 以前没有字典，"有/无字典"两组其实一样）。"""
     from app.tools.dictionary import dict_path_for
@@ -272,13 +299,16 @@ def block(items):
                 "infra_errors": self.infra_error_count(items),
                 "avg_duration_s": self.avg("duration_s", items),
                 "avg_tokens": self.avg("tokens", items),
+                # M02：判对的题各靠哪一层；非 strict 的都应人工抽审
+                "match_kinds": dict(Counter(r.get("match") for r in items if r.get("correct"))),
             }
         agents = sorted({r.get("agent", "workflow") for r in self.results})
         return {
             "time": datetime.now().isoformat(timespec="seconds"),
             "git_sha": (sha := git_sha()),
             **({"git_sha_note": git_sha_note()} if sha.endswith("?") or sha == "unknown" else {}),
             "agent": agents[0] if len(agents) == 1 else agents,
+            "evalset": evalset_fingerprint(),
             **self.meta,
             "overall": block(self.results),
             "by_category": {k: block(v) for k, v in self.by_category.items()},
@@ -293,7 +323,7 @@ def to_json(self, path: str, extra: dict | None = None) -> None:
             rows.append({
                 "id": c.id, "category": c.category, "dataset": getattr(c, "dataset", "sales"),
                 "question": c.question,
-                "correct": r.get("correct"), "executed": r.get("executed"),
+                "correct": r.get("correct"), "match": r.get("match"), "executed": r.get("executed"),
                 "initial_error": r.get("initial_error"), "infra_error": r.get("infra_error"),
                 "error_kind": r.get("error_kind"), "error": r.get("error"),
                 "attempts": r.get("attempts"), "duration_s": r.get("duration_s"),

app/eval/rescore.py

@@ -6,12 +6,14 @@
   差异会逐题列出，便于人工核对。
 """
 import json
+from collections import Counter
 
 from app.core.llm import api_error_text
 from app.core.result import refusal_from_answer
 from app.eval.cases import ALL_CASES
 from app.eval.report import load_obj
-from app.eval.runner import EvalRunner, score
+from app.eval.report import evalset_fingerprint
+from app.eval.runner import EvalRunner, score_detail
 
 
 def rescore(report_path: str, data_path: str | None = None) -> dict:
@@ -27,11 +29,12 @@ def rescore(report_path: str, data_path: str | None = None) -> dict:
         if case is not None and refusal is not None and not row.get("actual_obj"):
             # B40：ReAct 没调工具、直接在结论里按契约拒答，旧报告里 result 为空
             expected_list, err = runner.expected_all(case)
-            ok = (err is None) and score(refusal, case, expected_list)
+            kind = score_detail(refusal, case, expected_list) if err is None else None
+            ok = kind is not None
             if ok != row.get("correct"):
                 changed.append({"id": row["id"], "before": row.get("correct"), "after": ok})
             rows.append({"id": row["id"], "category": row["category"],
-                         "dataset": row.get("dataset"), "correct": ok})
+                         "dataset": row.get("dataset"), "correct": ok, "match": kind})
             continue
         if case is None or not row.get("executed"):
             rows.append({"id": row["id"], "category": row["category"], "dataset": row.get("dataset"), "correct": False})
@@ -47,10 +50,12 @@ def rescore(report_path: str, data_path: str | None = None) -> dict:
             skipped.append(row["id"])
             rows.append({"id": row["id"], "category": row["category"], "dataset": row.get("dataset"), "correct": row.get("correct")})
             continue
-        ok = (err is None) and score(actual, case, expected_list)
+        kind = score_detail(actual, case, expected_list) if err is None else None
+        ok = kind is not None
         if ok != row.get("correct"):
             changed.append({"id": row["id"], "before": row.get("correct"), "after": ok})
-        rows.append({"id": row["id"], "category": row["category"], "dataset": row.get("dataset"), "correct": ok})
+        rows.append({"id": row["id"], "category": row["category"], "dataset": row.get("dataset"),
+                     "correct": ok, "match": kind})
     by_cat = {}
     for r in rows:
         b = by_cat.setdefault(r["category"], [0, 0])
@@ -67,4 +72,8 @@ def rescore(report_path: str, data_path: str | None = None) -> dict:
             "correctness_rate": correct / n * 100 if n else None,
             "by_category": {k: {"correct": c, "n": t} for k, (c, t) in by_cat.items()},
             "by_dataset": {k: {"correct": c, "n": t} for k, (c, t) in by_ds.items()},
+            "match_kinds": dict(Counter(r.get("match") or "?" for r in rows if r["correct"])),
+            "needs_audit": [r["id"] for r in rows if r["correct"] and r.get("match") not in ("strict", "rounded", "refusal", "alt:strict", "alt:rounded")],
+            "evalset_now": evalset_fingerprint()["sha256"],
+            "evalset_report": (data.get("evalset") or {}).get("sha256"),
             "changed": changed, "skipped_no_obj": skipped, "infra_errors": infra}  # 无结构化结果或还原失败的题，沿用原判定

app/eval/runner.py

@@ -7,7 +7,7 @@
 """
 from app.core import analyze
 from app.eval.cases import DATASETS, EvalCase
-from app.eval.comparator import is_refusal, results_equal
+from app.eval.comparator import is_refusal, match_kind, results_equal
 from app.tools.csv_io import read_csv
 from app.tools.sandbox import run_trusted
 
@@ -102,7 +102,8 @@ def run_single(self, case: EvalCase):
         if res.executed:
             out["executed"] = True
             out["actual"] = res.result
-            out["correct"] = score(res.result, case, expected_list)
+            out["match"] = score_detail(res.result, case, expected_list)
+            out["correct"] = out["match"] is not None
         return out
 
     def run_all(self, cases: list[EvalCase] = None, verbose: bool = False):
@@ -129,9 +130,19 @@ def run_all(self, cases: list[EvalCase] = None, verbose: bool = False):
 
 def score(actual, case: EvalCase, expected_list: list) -> bool:
     """阶段 3 打分：应拒答题看是否拒答且给了原因；其他题拒答即错，否则与任一合理答案相等即对。"""
+    return score_detail(actual, case, expected_list) is not None
+
+
+def score_detail(actual, case: EvalCase, expected_list: list):
+    """判错返回 None；判对返回靠的是哪一层（M02）："refusal"，或 match_kind 的结果，
+    匹配的是歧义题的备选答案时前面加 "alt:"（如 "alt:strict"）。"""
     if case.expect_refusal:
-        return is_refusal(actual)
+        return "refusal" if is_refusal(actual) else None
     if is_refusal(actual):
-        return False
-    return any(results_equal(actual, e, ordered=case.ordered, percent_equiv=case.percent_equiv)
-               for e in expected_list)
+        return None
+    for i, e in enumerate(expected_list):
+        # 对错仍以 results_equal 为准（与阶段 3 判分逐题一致），match_kind 只负责标注层级
+        if results_equal(actual, e, ordered=case.ordered, percent_equiv=case.percent_equiv):
+            kind = match_kind(actual, e, ordered=case.ordered, percent_equiv=case.percent_equiv) or "relaxed"
+            return kind if i == 0 else f"alt:{kind}"
+    return None

app/nodes/codegen.py

@@ -1,4 +1,5 @@
 from app.core.llm import node_llm, strip_code
+from app.tools.dataset import SINGLE_ACCESS
 from app.tools.plotting import CHART_GUIDE
 
 _PROMPT = """根据分析计划写一段 pandas 代码。
@@ -15,7 +16,7 @@
 {plan}
 
 严格遵守以下约定：
-- 数据已加载到变量 df（pandas DataFrame），不要重新读取文件。
+- {data_access}
 - pd（pandas）、np（numpy）、plt（matplotlib.pyplot）已就绪，无需 import；不要导入其他库。
 - 把最终答案赋值给变量 result（可以是 DataFrame / Series / 数值 / 字符串 / dict），result 必须直接回答问题：
   问"哪个/哪一个"：result 只放标签本身（如 '华南'、'A'、'2024-06'），不要拼接说明文字或数值；
@@ -32,6 +33,7 @@ def generate_code(state, config=None):
     llm = node_llm(config)
     msg = _PROMPT.format(
         schema=state["schema"],
+        data_access=state.get("data_access") or SINGLE_ACCESS,
         field_notes=state.get("field_notes") or "（无）",
         question=state["question"],
         plan=state.get("plan", ""),

app/nodes/executor.py

@@ -1,15 +1,15 @@
 """执行节点。B07：result 保留结构化对象，另生成 result_text 预览；
 数据表由 analyze() 读一次后经 RunnableConfig 注入，每次执行用副本（不再每次重试都读 CSV）。"""
 from app.core.result import format_result
-from app.tools.csv_io import read_csv
+from app.tools.dataset import copy_data, load_data
 from app.tools.persist import new_chart_path
 from app.tools.sandbox import run_code
 
 
 def _dataset(state, config):
     conf = (config or {}).get("configurable", {}) if isinstance(config, dict) else {}
     df = conf.get("df")
-    return df.copy() if df is not None else read_csv(state["df_path"])
+    return copy_data(df) if df is not None else load_data(state["df_path"])[0]
 
 
 def execute(state, config=None):