From feaf9d10d4ffc716343ad0b3ef80e6df69f0c10e Mon Sep 17 00:00:00 2001 From: Smilewithoutfalling <91455578+Smilewithoutfalling@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:11 +0800 Subject: [PATCH 01/13] phase 3b: app/core/result.py --- app/core/result.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/app/core/result.py b/app/core/result.py index 82ea587..87321f6 100644 --- a/app/core/result.py +++ b/app/core/result.py @@ -93,3 +93,27 @@ def format_result(result: Any, max_chars: int = 3000, max_rows: int = 60) -> str if len(text) > max_chars: text = text[:max_chars] + f"\n…（文本已截断，原长 {len(text)} 字符）" return text + + +# ---- 拒答契约（阶段 3）：数据回答不了时 result = "无法回答：<原因>" ---- +REFUSAL_PREFIX = "无法回答" + + +def is_refusal(v) -> bool: + """结果契约（阶段 3）：数据回答不了时 result = "无法回答：<原因>"。只有前缀、没有原因的不算。""" + if not isinstance(v, str): + return False + s = v.strip() + if not s.startswith(REFUSAL_PREFIX): + return False + reason = s[len(REFUSAL_PREFIX):].strip(" ：:，,。.；;-—\n\t") + return len(reason) >= 2 + + +def refusal_from_answer(answer): + """规则 19：ReAct 没调用工具、直接在结论里拒答时，取结论首行（须以"无法回答"开头）作为 result。""" + if not isinstance(answer, str): + return None + first = answer.strip().splitlines()[0].strip() if answer.strip() else "" + first = first.strip("*").strip() + return first if is_refusal(first) else None From df5cd08f9691abe1ab1ce665d54b17e0da776ade Mon Sep 17 00:00:00 2001 From: Smilewithoutfalling <91455578+Smilewithoutfalling@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:12 +0800 Subject: [PATCH 02/13] phase 3b: app/core/service.py --- app/core/service.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/core/service.py b/app/core/service.py index 6201d31..2b76aca 100644 --- a/app/core/service.py +++ b/app/core/service.py @@ -17,7 +17,7 @@ from app.core.llm import (LLMClient, LLMConfig, LLMError, UsageCallbackHandler, UsageTracker, build_chat_model, classify_error) -from app.core.result import AnalysisResult, Step, format_result +from app.core.result import AnalysisResult, Step, format_result, refusal_from_answer AGENTS = ("workflow", "react") @@ -37,7 +37,7 @@ def analyze( chat_model=None, max_attempts: int = 3, exec_timeout: int = 20, - recursion_limit: int = 12, + recursion_limit: int = 25, use_dictionary: bool = True, memory=None, thread_id: str | None = None, @@ -236,6 +236,12 @@ def _run_react(res, tracker, cfg, chat_model, df, notes_text, question, dataset, elif isinstance(m, AIMessage): res.steps.append(Step("answer", content=str(m.content))) res.answer = str(m.content) + if res.answer.strip().startswith("Sorry, need more steps"): + # langgraph 剩余步数不足时不抛 GraphRecursionError，而是塞一条英文占位答复（B39） + res.answer = "" + if res.error is None: + res.error = f"ReAct 超过步数上限（recursion_limit={recursion_limit}）仍未给出结论" + res.error_kind = "agent" ok = [e for e in execs if not e["error"]] res.attempts = sum(1 for e in execs if e["error"]) @@ -249,6 +255,10 @@ def _run_react(res, tracker, cfg, chat_model, df, notes_text, question, dataset, res.result = last["result"] res.result_text = format_result(last["result"]) res.chart_path = last["chart"] or (res.charts[-1] if res.charts else None) + elif refusal_from_answer(res.answer) is not None: + # 模型没调用工具、直接在结论里按契约拒答（B40） + res.result = res.result_text = refusal_from_answer(res.answer) + res.executed = True elif execs: res.code = execs[-1]["code"] if res.error is None: From 1da172ea2707e53038b3460a36279a7e4a440712 Mon Sep 17 00:00:00 2001 From: Smilewithoutfalling <91455578+Smilewithoutfalling@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:14 +0800 Subject: [PATCH 03/13] phase 3b: app/eval/cases_extra.py --- app/eval/cases_extra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/eval/cases_extra.py b/app/eval/cases_extra.py index 98d9c25..cd0b8b3 100644 --- a/app/eval/cases_extra.py +++ b/app/eval/cases_extra.py @@ -115,7 +115,7 @@ def _refuse(reason: str) -> str: "d = df[df['status'] == '已完成']; result = round(d['pay_amount'].sum(), 2)", # 未去重 ]), EvalCase(id="ord_028", category="ambiguity", dataset="orders", - question="平均每单买几件？", + question="平均每单买几件？保留两位小数。", ground_truth=_D + "result = round(d['qty'].mean(), 2)", alt_ground_truths=[_DONE + "result = round(d['qty'].mean(), 2)", "result = round(df['qty'].mean(), 2)"]), From cb5ceae4ca7e01d920ad1683f95df6bcac81cbb9 Mon Sep 17 00:00:00 2001 From: Smilewithoutfalling <91455578+Smilewithoutfalling@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:15 +0800 Subject: [PATCH 04/13] phase 3b: app/eval/comparator.py --- app/eval/comparator.py | 81 +++++++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 12 deletions(-) diff --git a/app/eval/comparator.py b/app/eval/comparator.py index eb5827b..cf3e2ea 100644 --- a/app/eval/comparator.py +++ b/app/eval/comparator.py @@ -37,6 +37,9 @@ 期望索引是时间或从 1 起的连续序号时，实际是等长 list/tuple → 按位置比值。 规则之外的形态差异仍判错，需人工抽检（见 DEVLOG）。 +19. 百分数字符串（'7.00%'、Series/dict 里的 '50.58%'）按数值比较；dict 的键里含季度/月份 + （'第一季度销售总额'）且期望是序号索引时按序号对齐，dict 里的非数值项（'哪个季度更高'）忽略； + 问排名时给名次 1..n 的 Series，与期望数值从大到小的次序一致即算对（无并列时）。 阶段 3：应拒答题由 is_refusal() 判定（"无法回答：<原因>"，原因至少 2 个字），不经过上面的规则。 """ import math @@ -53,18 +56,20 @@ _MARGIN_LABELS = {"总计", "合计", "总和", "汇总", "all", "total", "sum"} -REFUSAL_PREFIX = "无法回答" +from app.core.result import REFUSAL_PREFIX, is_refusal, refusal_from_answer # noqa: F401 (re-export) +_PCT_RE = re.compile(r"^\s*([-+]?\d+(?:\.\d+)?)\s*%\s*$") +_EMBED_Q_RE = re.compile(r"(?:第\s*([一二三四1-4])\s*季度|Q([1-4])(?!\d))", re.IGNORECASE) +_EMBED_M_RE = re.compile(r"(? bool: - """结果契约（阶段 3）：数据回答不了时 result = "无法回答：<原因>"。只有前缀、没有原因的不算。""" - if not isinstance(v, str): - return False - s = v.strip() - if not s.startswith(REFUSAL_PREFIX): - return False - reason = s[len(REFUSAL_PREFIX):].strip(" ：:，,。.；;-—\n\t") - return len(reason) >= 2 +def _pct(v): + """规则 19：'7.00%' 这样的百分数字符串 → 7.0（题目已要求用百分数表示，数值按百分数刻度比较）。""" + v = _py(v) + if isinstance(v, str): + m = _PCT_RE.match(v) + if m: + return float(m.group(1)) + return None # ------------------------------------------------------------------ 入口 @@ -206,6 +211,8 @@ def _scalar_eq(a, e, tol) -> bool: return True if _is_nan(a) or _is_nan(e): return False + if _pct(a) is not None and _is_num(e): + return _scalar_eq(_pct(a), e, tol) if _is_num(e) and not _is_num(a) and float(e).is_integer(): p = _period_part(a, int(e)) # 规则 11：月份/季度序号 if p is not None: @@ -347,7 +354,7 @@ def _vs_scalar(a, e, tol) -> bool: if all(isinstance(k, str) and not _month_key(k) for k in keys): vals = list(a.values if isinstance(a, pd.Series) else a.values()) if vals is not None: - return any(_is_num(_py(v)) and _scalar_eq(v, e, tol) for v in vals) + return any((_is_num(_py(v)) or _pct(v) is not None) and _scalar_eq(v, e, tol) for v in vals) return False @@ -400,6 +407,8 @@ def _is_time_index(idx) -> bool: def _series_eq(a: pd.Series, e: pd.Series, tol, ordered) -> bool: if len(a) != len(e): return False + if _rank_eq(a, e): + return True if _mapping_eq(dict(zip(a.index, a.values)), dict(zip(e.index, e.values)), tol, ordered): return True # 规则 6：两边都是时间索引（月末 Timestamp / Period / 'YYYY-MM'），标签写法不同 → 先按月份对齐 @@ -430,6 +439,27 @@ def _series_eq(a: pd.Series, e: pd.Series, tol, ordered) -> bool: return False +def _rank_eq(a: pd.Series, e: pd.Series) -> bool: + """规则 19：问"排名"时实际给名次（1..n），期望给数值：标签相同且名次 = 期望值从大到小的次序即算对。""" + try: + av = [_py(x) for x in a.values] + if sorted(av) != list(range(1, len(e) + 1)) or len(e) < 2: + return False + if not all(_is_num(_py(x)) for x in e.values) or e.index.has_duplicates: + return False + if sorted(_py(x) for x in e.values) == list(range(1, len(e) + 1)): + return False # 期望本身就是 1..n，不能当名次放宽 + if sorted(map(_label, a.index)) != sorted(map(_label, e.index)): + return False + order = [_label(k) for k in e.sort_values(ascending=False, kind="stable").index] + if len(set(e.values)) != len(e): + return False # 有并列时名次口径不唯一，不放宽 + rank = {k: i + 1 for i, k in enumerate(order)} + return all(rank[_label(k)] == v for k, v in zip(a.index, av)) + except Exception: + return False + + def _increasing_ints(idx, lo, hi) -> bool: try: v = [_py(x) for x in idx] @@ -479,7 +509,9 @@ def _vs_series(a, e: pd.Series, tol, ordered) -> bool: if isinstance(a, pd.Series): return _series_eq(a, e, tol, ordered) if isinstance(a, dict): - return _series_eq(pd.Series(a), e, tol, ordered) + if _series_eq(pd.Series(a), e, tol, ordered): + return True + return _dict_embedded_periods(a, e, tol, ordered) if isinstance(a, (list, tuple)): # 只有当期望本身没有有意义的标签（默认整数索引）时才按位置比； # 规则 18：期望索引是时间或从 1 起的连续序号（季度 1–4、月份 1–12）时也按位置比 @@ -516,6 +548,31 @@ def _vs_series(a, e: pd.Series, tol, ordered) -> bool: return False +def _dict_embedded_periods(a: dict, e: pd.Series, tol, ordered) -> bool: + """规则 19：{'第一季度销售总额': 131210.0, '第二季度销售总额': 143463.5, '哪个季度更高': '第二季度'} + 对 Series(index=[1, 2])：只取数值项，键里能读出季度（或月份）序号且一一对应时按序号比较。""" + nums = {k: v for k, v in a.items() if _is_num(_py(v)) or _pct(v) is not None} + if len(nums) != len(e) or len(e) < 2: + return False + ev = [_py(x) for x in e.index] + if not all(_is_num(x) and float(x).is_integer() for x in ev): + return False + for rx, hi in ((_EMBED_Q_RE, 4), (_EMBED_M_RE, 12)): + if not all(1 <= x <= hi for x in ev): + continue + parts = [] + for k in nums: + m = rx.search(str(k)) if isinstance(k, str) else None + if not m: + parts = None + break + g = next(x for x in m.groups() if x) + parts.append(_CN_NUM.get(g) or int(g)) + if parts and len(set(parts)) == len(parts): + return _mapping_eq(dict(zip(parts, nums.values())), dict(zip(e.index, e.values)), tol, ordered) + return False + + def _label_value_series(a: pd.DataFrame, n: int, max_cols: int = 20): """规则 8：把表里每一对（标签列, 数值列）还原成 Series；有意义的行索引也算一列标签。""" if len(a) != n: From 2f6c35d8a12998caed557ec6244b6288fec38a01 Mon Sep 17 00:00:00 2001 From: Smilewithoutfalling <91455578+Smilewithoutfalling@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:17 +0800 Subject: [PATCH 05/13] phase 3b: app/eval/report.py --- app/eval/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/eval/report.py b/app/eval/report.py index f569e16..b6e479e 100644 --- a/app/eval/report.py +++ b/app/eval/report.py @@ -91,7 +91,7 @@ def datasets_metadata(datasets: dict) -> dict: d = dict_path_for(path) out[name] = {"path": os.path.relpath(path, _REPO).replace(os.sep, "/"), "sha256": hashlib.sha256(open(path, "rb").read()).hexdigest()[:12], - "dictionary": os.path.exists(d)} + "dictionary_file": os.path.exists(d)} # 文件是否存在；本次是否使用看顶层 use_dictionary return out From 47933c8315d36b697a54a9ae788176b4a06238bd Mon Sep 17 00:00:00 2001 From: Smilewithoutfalling <91455578+Smilewithoutfalling@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:18 +0800 Subject: [PATCH 06/13] phase 3b: app/eval/rescore.py --- app/eval/rescore.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/app/eval/rescore.py b/app/eval/rescore.py index e35644d..4cad2ec 100644 --- a/app/eval/rescore.py +++ b/app/eval/rescore.py @@ -7,6 +7,7 @@ """ import json +from app.core.result import refusal_from_answer from app.eval.cases import ALL_CASES from app.eval.report import load_obj from app.eval.runner import EvalRunner, score @@ -19,32 +20,48 @@ def rescore(report_path: str, data_path: str | None = None) -> dict: rows, changed, skipped = [], [], [] for row in data["cases"]: case = cases.get(row["id"]) + refusal = refusal_from_answer(row.get("answer")) + if case is not None and refusal is not None and not row.get("actual_obj"): + # B40：ReAct 没调工具、直接在结论里按契约拒答，旧报告里 result 为空 + expected_list, err = runner.expected_all(case) + ok = (err is None) and score(refusal, case, expected_list) + if ok != row.get("correct"): + changed.append({"id": row["id"], "before": row.get("correct"), "after": ok}) + rows.append({"id": row["id"], "category": row["category"], + "dataset": row.get("dataset"), "correct": ok}) + continue if case is None or not row.get("executed"): - rows.append({"id": row["id"], "category": row["category"], "correct": False}) + rows.append({"id": row["id"], "category": row["category"], "dataset": row.get("dataset"), "correct": False}) continue if not row.get("actual_obj"): skipped.append(row["id"]) - rows.append({"id": row["id"], "category": row["category"], "correct": row.get("correct")}) + rows.append({"id": row["id"], "category": row["category"], "dataset": row.get("dataset"), "correct": row.get("correct")}) continue expected_list, err = runner.expected_all(case) try: actual = load_obj(row["actual_obj"]) except Exception: skipped.append(row["id"]) - rows.append({"id": row["id"], "category": row["category"], "correct": row.get("correct")}) + rows.append({"id": row["id"], "category": row["category"], "dataset": row.get("dataset"), "correct": row.get("correct")}) continue ok = (err is None) and score(actual, case, expected_list) if ok != row.get("correct"): changed.append({"id": row["id"], "before": row.get("correct"), "after": ok}) - rows.append({"id": row["id"], "category": row["category"], "correct": ok}) + rows.append({"id": row["id"], "category": row["category"], "dataset": row.get("dataset"), "correct": ok}) by_cat = {} for r in rows: b = by_cat.setdefault(r["category"], [0, 0]) b[0] += bool(r["correct"]) b[1] += 1 + by_ds = {} + for r in rows: + b = by_ds.setdefault(r.get("dataset") or "sales", [0, 0]) + b[0] += bool(r["correct"]) + b[1] += 1 n = len(rows) correct = sum(bool(r["correct"]) for r in rows) return {"report": report_path, "n": n, "correct": correct, "correctness_rate": correct / n * 100 if n else None, "by_category": {k: {"correct": c, "n": t} for k, (c, t) in by_cat.items()}, + "by_dataset": {k: {"correct": c, "n": t} for k, (c, t) in by_ds.items()}, "changed": changed, "skipped_no_obj": skipped} # 无结构化结果或还原失败的题，沿用原判定 From 83d2c1826a2ca5ca1e6dbd31084dd8446cb830be Mon Sep 17 00:00:00 2001 From: Smilewithoutfalling <91455578+Smilewithoutfalling@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:19 +0800 Subject: [PATCH 07/13] phase 3b: app/react_agent.py --- app/react_agent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/react_agent.py b/app/react_agent.py index ced09ce..9e0376d 100644 --- a/app/react_agent.py +++ b/app/react_agent.py @@ -80,7 +80,8 @@ def run_python_on_data(code: str) -> str: "不要用中间表或整张排序表收尾。" "如果数据回答不了（表里没有所需的字段或信息），不要编造或用别的字段凑：" "仍然调用一次 run_python_on_data，代码为 result = '无法回答：<一句话原因>'（原因说明缺什么，" - "如 '无法回答：数据中没有成本字段'），结论里同样说明原因。" + "如 '无法回答：数据中没有成本字段'），最终结论的第一行也必须原样以 '无法回答：' 开头" + "（不要写成'无法直接回答''无法计算'等其他说法）。" "如果问题需要图表，请务必画图并保存。\n\n" f"{CHART_GUIDE}\n\n" "最终用中文给出简洁、直接的结论；若已生成图表，说明图表展示了什么。" From 1277623a001b621d2db281c2e03c44aae1a2dbce Mon Sep 17 00:00:00 2001 From: Smilewithoutfalling <91455578+Smilewithoutfalling@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:20 +0800 Subject: [PATCH 08/13] phase 3b: app/tools/plotting.py --- app/tools/plotting.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/tools/plotting.py b/app/tools/plotting.py index c4dc6fa..f87b23e 100644 --- a/app/tools/plotting.py +++ b/app/tools/plotting.py @@ -32,6 +32,8 @@ "- 必须有中文标题（ax.set_title）和中文坐标轴标签（set_xlabel/set_ylabel）。\n" "- 选择合适的图型：对比用柱状图、趋势用折线图、占比用饼图/堆叠图。\n" "- 关键数据点加数值标注（ax.text / bar_label），让人不看坐标也能读数。\n" + "- 横轴是字符串标签（如 '2024-01'）时，pandas 折线/柱状图的横坐标实际是位置 0..n-1：" + "用 for i, (lab, y) in enumerate(s.items()): ax.text(i, y, ...) 标注，不要把字符串当 x 传给 ax.text/annotate。\n" "- 不要堆砌过多元素，保持简洁；如有多类别再加图例。\n" "- 用 plt.savefig(chart_path) 保存（清晰度与白边已全局处理，无需传 dpi）。" ) From f94dcea6124e4d388e37a07cf58907008e50ace5 Mon Sep 17 00:00:00 2001 From: Smilewithoutfalling <91455578+Smilewithoutfalling@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:22 +0800 Subject: [PATCH 09/13] phase 3b: docs/DEVLOG.md --- docs/DEVLOG.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/DEVLOG.md b/docs/DEVLOG.md index b1cdb1e..4de8a42 100644 --- a/docs/DEVLOG.md +++ b/docs/DEVLOG.md @@ -390,3 +390,40 @@ xfail 用 strict 模式：问题一旦修好，对应测试会"意外通过"并 - 基线命令（在 30df587 之后的 master 上跑，报告自带 commit SHA 与各表 SHA）： `python run_eval.py --agent both` 与 `python run_eval.py --agent both --no-dict` - 结果回填到此处，作为 130 题的第一条基线。 + +--- + +## [7] 2026-10-10 · 阶段 3b 基线复核与修复 · commit <待提交> + +### 基线（S 运行，分支 eval-3 b87aef3，docs/eval/phase3/） +- 模型 Qwen3.5-397B-A17B（网关 grg-aikits-test.grgbanking.com），temperature 0，Python 3.11.17，pandas 3.0.6，130 题，n=1。 +- 代码 1b27da1。报告里 git_sha 为 "1b27da1?"："?" 表示 S 的环境里调用 git 失败，退回读 .git 文件，**无法判断工作区是否有改动**（不是 -dirty）。 +- 表指纹：sales abe40a78ff53、orders b21bad987c65、employees 3fd0f03966b8、inventory 0aac24693528。 + +| 配置 | 原判 | 重评（本阶段比较器） | 应拒答 | 剩余错题（重评后） | +|---|---|---|---|---| +| workflow 有字典 | 93.8% | **98.5%**（128/130） | 9/9 | ts_001 画图报错，ord_028 题目没写精度（B43） | +| workflow 无字典 | 93.8% | **97.7%**（127/130） | 9/9 | corr_001 只给地区合计，ts_011、ord_019 画图报错 | +| react 有字典 | 86.9% | **90.8%**（118/130） | 6/9 | 拒答不守契约 2、步数上限 1，其余见报告 | +| react 无字典 | 83.8% | **87.7%**（114/130） | 6/9 | 同上，另 4 题撞步数上限 | + +- 有/无字典差异在 1 题以内，属单次运行波动范围；在当前 4 张表上，字典没有带来可测的增益（列名已足够自解释）。要测 RAG 增益需要列名含糊的表。 +- react 比 workflow 低约 8 个百分点，主要来自拒答（6/9 vs 9/9）和步数上限，不是计算能力。 + +### 本阶段修复 +| ID | 级别 | 位置 | 问题与修复 | +|---|---|---|---| +| B39 | P1 | app/core/service.py | langgraph 剩余步数不足时不抛异常，而是返回英文占位 "Sorry, need more steps…"，被当成结论。改为记 error_kind=agent；ReAct 默认 recursion_limit 12→25 | +| B40 | P1 | app/core/service.py、rescore.py | ReAct 不调工具、直接在结论里写"无法回答：…"时 result 为空，拒答题判错。现在结论首行符合契约时取作 result（`refusal_from_answer`，移到 app/core/result.py）；--rescore 对旧报告同样处理。不守契约的说法（"无法直接回答""无法计算"）仍判错；ReAct 提示词要求结论首行原样以"无法回答："开头 | +| B41 | P1 | app/eval/comparator.py | 比较器规则 19：百分数字符串 '7.00%' 按数值比；dict 键里含季度/月份（'第一季度销售总额'、'3月'）对序号索引 Series 按序号对齐，忽略 dict 中的非数值项；问排名时给名次 1..n 且与期望值从大到小一致即对（并列时不放宽，期望本身是 1..n 时不放宽） | +| B42 | P2 | run_eval.py | `--agent both` 的 compare 文件固定写进 outputs/，单测因此往真实 outputs/ 落文件（S 的 eval-3 里那份 eval_compare_oracle 就来自 pytest 临时目录，应删除）。现在给了 --out 时写在同目录 | +| B43 | P2 | cases_extra.py ord_028 | "平均每单买几件？"没写精度，标准答案却 round 到 2 位，模型答 2.3 被判错。题目补"保留两位小数"。本次基线中这题人工判对 | +| B44 | P2 | app/tools/plotting.py | 横轴是 '2024-01' 这类字符串时，pandas 画图的横坐标实际是位置 0..n-1，模型用 ax.text('2024-01', y) 标注触发 matplotlib ConversionError，整段代码失败（结果其实已算对）。CHART_GUIDE 补了用位置标注的说明。是否在图表失败时保留已算出的 result 留到阶段 4 讨论 | +| — | P3 | report.py | datasets.*.dictionary 改名为 dictionary_file（只表示文件存在；本次是否使用看顶层 use_dictionary），避免 --no-dict 报告里显示 true 引起误读 | + +### 指标快照 +- 单测：Linux Python 3.11 **374 passed**，约 3 分钟。 + +### 下一步 +- S 删除 eval-3 上的 eval_compare_oracle_20261010_092954.json 后合并（或由本次 PR 一并处理）。 +- 合并后建议 S 再跑一次 `--agent react`，验证 B39/B40/B44 的实际效果；workflow 不必重跑。 From 2804203929c4f6a6cb9975ac58bf0fd3a624842d Mon Sep 17 00:00:00 2001 From: Smilewithoutfalling <91455578+Smilewithoutfalling@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:23 +0800 Subject: [PATCH 10/13] phase 3b: run_eval.py --- run_eval.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/run_eval.py b/run_eval.py index ea99ac8..f18cb8a 100644 --- a/run_eval.py +++ b/run_eval.py @@ -104,7 +104,9 @@ def main(): all_correct = all_correct and ok compare[agent] = runs if len(agents) > 1: - _print_compare(compare, os.path.join(OUTPUT_DIR, f"eval_compare{'_oracle' if args.oracle else ''}_{stamp}.json")) + # 给了 --out 时 compare 写在同一目录，测试不会往真实 outputs/ 里落文件（B42） + out_dir = os.path.dirname(os.path.abspath(args.out)) if args.out else OUTPUT_DIR + _print_compare(compare, os.path.join(out_dir, f"eval_compare{'_oracle' if args.oracle else ''}_{stamp}.json")) # 返回码：全部正确为0，否则为1 sys.exit(0 if all_correct else 1) From 33ecb0dae9a4d93880fe16638b3c0728f5b6337f Mon Sep 17 00:00:00 2001 From: Smilewithoutfalling <91455578+Smilewithoutfalling@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:24 +0800 Subject: [PATCH 11/13] phase 3b: tests/test_comparator.py --- tests/test_comparator.py | 46 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/test_comparator.py b/tests/test_comparator.py index e01a4b4..d86f951 100644 --- a/tests/test_comparator.py +++ b/tests/test_comparator.py @@ -166,3 +166,49 @@ def test_refusal_detection(): assert not is_refusal("无法回答：") assert not is_refusal("数据中没有成本字段") assert not is_refusal(None) and not is_refusal(0) + + +# ---------------------------------------------------------------- 规则 19（阶段 3b，B41） +def test_rule19_percent_string_scalar(): + assert results_equal("7.00%", 7.0) + assert results_equal("45.81%", 45.81) + assert not results_equal("45.8%", 45.81) + assert not results_equal("7.00%", 0.5) + + +def test_rule19_percent_strings_in_series_and_dict(): + idx = ["APP", "小程序", "网页"] + assert results_equal(pd.Series(["50.58%", "34.58%", "14.83%"], index=idx), + pd.Series([50.58, 34.58, 14.83], index=idx)) + assert results_equal({"总订单数": 1200, "已退款订单数": 84, "退款率": "7.00%"}, 7.0) + + +def test_rule19_dict_with_embedded_quarter_keys(): + e = pd.Series([131210.0, 143463.5], index=pd.Index([1, 2], name="quarter")) + assert results_equal({"第一季度销售总额": 131210.0, "第二季度销售总额": 143463.5, "哪个季度更高": "第二季度"}, e) + assert results_equal({"2024年Q1": 131210.0, "2024年Q2": 143463.5}, e) + assert not results_equal({"第一季度": 143463.5, "第二季度": 131210.0}, e) # 对错季度 + assert not results_equal({"华东": 131210.0, "华南": 143463.5}, e) # 键里没有季度 + + +def test_rule19_dict_with_embedded_month_keys(): + e = pd.Series([34, 21, 5], index=pd.Index([3, 4, 5], name="date")) + assert results_equal({"3月": 34, "4月": 21, "5月": 5}, e) + assert not results_equal({"3月": 21, "4月": 34, "5月": 5}, e) + + +def test_refusal_from_answer(): + from app.core.result import refusal_from_answer + assert refusal_from_answer("无法回答：数据中没有店长字段。\n更多说明") == "无法回答：数据中没有店长字段。" + assert refusal_from_answer("**无法回答：没有供应商字段**") == "无法回答：没有供应商字段" + assert refusal_from_answer("无法直接回答，因为没有成本字段") is None + assert refusal_from_answer("无法回答") is None + assert refusal_from_answer(None) is None + + +def test_rule19_rank_numbers_vs_values(): + e = pd.Series([13757.5, 10721.5, 4760.0, 4550.0, 4270.0], index=pd.Index(list("BEACD"), name="product")) + assert results_equal(pd.Series([1, 2, 3, 4, 5], index=list("BEACD")), e) + assert results_equal(pd.Series([3, 1, 2, 4, 5], index=list("ABECD")), e) + assert not results_equal(pd.Series([1, 2, 3, 4, 5], index=list("EBACD")), e) + assert not results_equal(pd.Series([1, 2, 3], index=list("BEA")), e) From 6dc4c37b051f8b52ef6f507bca73e9394aee94fb Mon Sep 17 00:00:00 2001 From: Smilewithoutfalling <91455578+Smilewithoutfalling@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:55 +0800 Subject: [PATCH 12/13] phase 3b: tests/test_core_analyze.py --- tests/test_core_analyze.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_core_analyze.py b/tests/test_core_analyze.py index 7f60b70..ef56299 100644 --- a/tests/test_core_analyze.py +++ b/tests/test_core_analyze.py @@ -180,3 +180,23 @@ def test_react_multi_turn_memory(sample_path): assert [s.name for s in r2.steps] == ["load", "answer"] # 只含本轮 assert "问一" in prompt_text(m.calls[-1]) and "第一轮" in prompt_text(m.calls[-1]) assert not r2.executed and r2.result is None + + +def test_react_refusal_in_answer_without_tool(sample_path): + """B40：模型不调工具、直接在结论里按契约拒答 → result 取结论首行。""" + m = ScriptedChatModel(replies=["无法回答：数据中没有成本字段。\n所以算不了毛利率。"], calls=[]) + res = analyze(sample_path, "毛利率是多少", agent="react", chat_model=m, llm_config=FAST) + assert res.result == "无法回答：数据中没有成本字段。" and res.executed and res.error is None + + +def test_react_non_contract_refusal_not_rescued(sample_path): + m = ScriptedChatModel(replies=["无法直接回答，因为没有成本字段。"], calls=[]) + res = analyze(sample_path, "毛利率是多少", agent="react", chat_model=m, llm_config=FAST) + assert res.result is None and not res.executed + + +def test_react_sorry_placeholder_is_error(sample_path): + """B39：langgraph 步数不足时塞 'Sorry, need more steps…'，不能当结论。""" + m = ScriptedChatModel(replies=["Sorry, need more steps to process this request."], calls=[]) + res = analyze(sample_path, "q", agent="react", chat_model=m, llm_config=FAST) + assert res.answer == "" and res.error_kind == "agent" and "recursion_limit" in res.error From cd19f7a8bc58a054501d356018c12f9232f4b351 Mon Sep 17 00:00:00 2001 From: Smilewithoutfalling <91455578+Smilewithoutfalling@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:57 +0800 Subject: [PATCH 13/13] phase 3b: tests/test_entrypoints.py --- tests/test_entrypoints.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_entrypoints.py b/tests/test_entrypoints.py index 416f0b9..a3361ec 100644 --- a/tests/test_entrypoints.py +++ b/tests/test_entrypoints.py @@ -55,8 +55,9 @@ def test_run_eval_both_agents_and_rescore_new_tables(tmp_path): assert "应拒答" in p.stdout and "workflow" in p.stdout and "react" in p.stdout reports = sorted(tmp_path.glob("r_*.json")) assert [r.name for r in reports] == ["r_react.json", "r_workflow.json"] + assert len(list(tmp_path.glob("eval_compare_oracle_*.json"))) == 1 # B42：compare 跟 --out 同目录 data = json.loads(reports[1].read_text(encoding="utf-8")) - assert data["datasets"]["orders"]["dictionary"] and {c["dataset"] for c in data["cases"]} == { + assert data["datasets"]["orders"]["dictionary_file"] and {c["dataset"] for c in data["cases"]} == { "orders", "employees", "inventory"} p = subprocess.run([sys.executable, "run_eval.py", "--rescore", str(reports[1])], cwd=ROOT, capture_output=True, text=True, encoding="utf-8", timeout=120)