diff --git a/app/core/llm.py b/app/core/llm.py index 59a85b7..5fc51d1 100644 --- a/app/core/llm.py +++ b/app/core/llm.py @@ -118,6 +118,38 @@ def __init__(self, message: str, *, kind: str, attempts: int, cause: BaseExcepti self.cause = cause +_API_ERROR_KEYS = {"message", "error", "detail", "msg", "code", "status", "type", "param", "request_id"} + + +def api_error_text(text) -> Optional[str]: + """B45:网关有时以 200 返回一段错误 JSON 作为"回复内容",例如 + {"message":"prompt: A user's message must contain at least one image or a PDF or audio."}。 + 整段内容是一个 JSON 对象、键全是错误字段(message/error/detail…)时返回错误说明,否则 None。 + 正常结论(中文文字、代码、含其他键的 JSON)不受影响。""" + import json + + if not isinstance(text, str): + return None + t = text.strip() + if not (t.startswith("{") and t.endswith("}")): + return None + try: + obj = json.loads(t) + except Exception: + return None + if not isinstance(obj, dict) or not obj or not set(obj) <= _API_ERROR_KEYS: + return None + err = obj.get("error") + if isinstance(err, dict): + err = err.get("message") or json.dumps(err, ensure_ascii=False) + msg = obj.get("message") or err or obj.get("detail") or obj.get("msg") + return str(msg) if msg else None + + +class GatewayErrorContent(Exception): + """模型接口返回了错误 JSON 而不是回复(B45)。按短暂故障处理,可重试。""" + + def classify_error(exc: BaseException) -> tuple[str, bool]: """返回 (kind, 是否可重试)。只重试短暂性故障;鉴权/参数错误重试无意义。""" try: @@ -139,6 +171,8 @@ def classify_error(exc: BaseException) -> tuple[str, bool]: if isinstance(exc, openai.APIStatusError): code = getattr(exc, "status_code", 0) or 0 return ("server", True) if code >= 500 else ("http_%d" % code, False) + if isinstance(exc, GatewayErrorContent): + return "gateway_error", True if isinstance(exc, TimeoutError) or "timeout" in type(exc).__name__.lower(): return "timeout", True if isinstance(exc, ConnectionError): @@ -186,6 +220,10 @@ def invoke_message(self, prompt): attempts += 1 try: msg = self.model.invoke(prompt) + bad = api_error_text(getattr(msg, "content", None)) + if bad is not None: + self.tracker.add_message(msg) + raise GatewayErrorContent(f"接口返回错误内容:{bad}") except Exception as e: # noqa: BLE001 kind, retryable = classify_error(e) if not retryable or attempts > self.config.max_retries: diff --git a/app/core/service.py b/app/core/service.py index 2b76aca..d686958 100644 --- a/app/core/service.py +++ b/app/core/service.py @@ -15,7 +15,7 @@ from collections import defaultdict from functools import lru_cache -from app.core.llm import (LLMClient, LLMConfig, LLMError, UsageCallbackHandler, +from app.core.llm import (api_error_text, LLMClient, LLMConfig, LLMError, UsageCallbackHandler, UsageTracker, build_chat_model, classify_error) from app.core.result import AnalysisResult, Step, format_result, refusal_from_answer @@ -236,6 +236,13 @@ def _run_react(res, tracker, cfg, chat_model, df, notes_text, question, dataset, elif isinstance(m, AIMessage): res.steps.append(Step("answer", content=str(m.content))) res.answer = str(m.content) + bad = api_error_text(res.answer) + if bad is not None: + # 网关以错误 JSON 充当回复(B45):不是答案,按 LLM 基础设施失败记 + res.answer = "" + if res.error is None: + res.error = f"LLM 调用失败(gateway_error):接口返回错误内容:{bad}" + res.error_kind = "llm" if res.answer.strip().startswith("Sorry, need more steps"): # langgraph 剩余步数不足时不抛 GraphRecursionError,而是塞一条英文占位答复(B39) res.answer = "" diff --git a/app/eval/cases.py b/app/eval/cases.py index 04bbb43..c37f0c7 100644 --- a/app/eval/cases.py +++ b/app/eval/cases.py @@ -100,7 +100,7 @@ ), EvalCase( id="agg_015", category="aggregation", - question="平均每个地区的销售额是多少?", + question="各地区总销售额的平均值是多少?", # B47:原题"平均每个地区的销售额"可读成各地区的单笔平均 ground_truth="result = (df['units'] * df['price']).groupby(df['region']).sum().mean()", keywords=["groupby", "sum", "mean"], ), @@ -229,7 +229,7 @@ ), EvalCase( id="corr_007", category="correlation", - question="先按产品和地区汇总销量,再求每个产品在各地区之间销量的标准差,判断分布是否均匀。", + question="先按产品和地区汇总销量,再求每个产品在各地区之间销量的样本标准差(保留两位小数),判断分布是否均匀。", # B47:原题未说明样本/总体口径与精度 ground_truth="result = df.groupby(['product', 'region'])['units'].sum().groupby(level=0).std().round(2)", keywords=["groupby", "std"], ), diff --git a/app/eval/cases_extra.py b/app/eval/cases_extra.py index cd0b8b3..4f7063f 100644 --- a/app/eval/cases_extra.py +++ b/app/eval/cases_extra.py @@ -131,7 +131,10 @@ def _refuse(reason: str) -> str: question="复购率是多少?", ground_truth=_D + "n = d.groupby('user_id').size(); result = round((n >= 2).mean() * 100, 2)", alt_ground_truths=[_DONE + "n = d.groupby('user_id').size(); result = round((n >= 2).mean() * 100, 2)", - _D + "n = d.groupby('user_id').size(); result = round(n[n >= 2].sum() / n.sum() * 100, 2)"]), + _D + "n = d.groupby('user_id').size(); result = round(n[n >= 2].sum() / n.sum() * 100, 2)", + # B47:排除已退款的有效订单口径 + _D + "d = d[d['status'] != '已退款']; n = d.groupby('user_id').size(); " + "result = round((n >= 2).mean() * 100, 2)"]), ] diff --git a/app/eval/comparator.py b/app/eval/comparator.py index cf3e2ea..422a570 100644 --- a/app/eval/comparator.py +++ b/app/eval/comparator.py @@ -40,6 +40,14 @@ 19. 百分数字符串('7.00%'、Series/dict 里的 '50.58%')按数值比较;dict 的键里含季度/月份 ('第一季度销售总额')且期望是序号索引时按序号对齐,dict 里的非数值项('哪个季度更高')忽略; 问排名时给名次 1..n 的 Series,与期望数值从大到小的次序一致即算对(无并列时)。 +阶段 3c(B46,来自 eval-3b 的 ReAct 复核;只在上面的规则都不匹配时才用,且要求"唯一"以免奖励罗列多种口径): + 20. a) 期望是标签、实际是按值排好序的数值 Series(榜首不并列)→ 榜首标签即答案; + b) 实际是带附加信息的 dict(≤8 项、键是文字而不是月份): + 期望数值时,第一层数值项里恰好一个等于期望;期望标签时,第一层恰好一个标签值且等于期望; + 期望 Series/表时,dict 里恰好一个子对象(Series/表/dict)等于期望; + 期望 Series 且索引是文字标签时,每个标签恰好对应一个"含该标签的键"且数值相等('工作日平均销量' ↔ '工作日'); + 嵌套 dict 里的标量不参与({'口径A': {'复购率': …}, '口径B': {…}} 这种罗列口径的不算对); + c) 实际是一句话字符串、期望数值:句中恰好一个百分数(没有百分数时恰好一个数字)且等于期望。 阶段 3:应拒答题由 is_refusal() 判定("无法回答:<原因>",原因至少 2 个字),不经过上面的规则。 """ import math @@ -86,11 +94,88 @@ def results_equal(actual, expected, float_tol=1e-4, *, ordered=None, percent_equ try: if _eq(v, e, float_tol, ordered): return True + except Exception: + pass + try: + if _extras_eq(v, e, float_tol, ordered): + return True except Exception: continue return False +# ------------------------------------------------------------------ 规则 20(B46) +_NUM_TOKEN_RE = re.compile(r"(? bool: + return isinstance(v, (pd.Timestamp, pd.Period)) or (isinstance(v, str) and not _is_num(v) and _pct(v) is None) + + +def _extras_eq(a, e, tol, ordered) -> bool: + """规则 20:答案对、但外面裹了附加信息的形态。见模块说明。""" + # a) 期望标签,实际是排好序的数值 Series + if _is_label_value(e) and isinstance(a, pd.Series) and len(a) > 1 \ + and pd.api.types.is_numeric_dtype(a) and not isinstance(a.index, pd.MultiIndex): + if (a.is_monotonic_decreasing or a.is_monotonic_increasing) and a.iloc[0] != a.iloc[1]: + return _scalar_eq(_py(a.index[0]), e, tol) + return False + # c) 一句话字符串 + if isinstance(a, str) and _is_num(e): + pcts = _PCT_TOKEN_RE.findall(a) + if pcts: + return len(pcts) == 1 and _scalar_eq(float(pcts[0]), e, tol) + nums = _NUM_TOKEN_RE.findall(a) + return len(nums) == 1 and _scalar_eq(float(nums[0].replace(",", "")), e, tol) + # b) 带附加信息的 dict + if not isinstance(a, dict) or not (1 < len(a) <= _EXTRAS_MAX): + return False + if not all(isinstance(k, str) and not _DATE_KEY_RE.match(k.strip()) for k in a): + return False # 键本身是日期/月份 → 是时间序列,不是"答案 + 附加信息" + vals = {k: _py(v) for k, v in a.items()} + if _is_num(e): + leaves = [v for v in vals.values() if _is_num(v) or _pct(v) is not None] + if len(leaves) == len(vals) and max(len(k.strip()) for k in vals) <= 4: + return False # {'华东': 1, '华北': 2, …}:是分组汇总表,碰巧含期望值不算(同规则 10) + return sum(1 for v in leaves if _scalar_eq(v, e, tol)) == 1 + if _is_label_value(e): + labels = [v for v in vals.values() if _is_label_value(v)] + return len(labels) == 1 and _scalar_eq(labels[0], e, tol) + if isinstance(e, (pd.Series, pd.DataFrame)): + subs = [v for v in vals.values() if isinstance(v, (pd.Series, pd.DataFrame, dict))] + hits = 0 + for v in subs: + try: + hits += bool(_eq(v, e, tol, ordered)) + except Exception: + pass + if hits == 1: + return True + if isinstance(e, pd.Series) and not ordered: + return _keys_contain_labels(vals, e, tol) + return False + + +def _keys_contain_labels(vals: dict, e: pd.Series, tol) -> bool: + """规则 20 b:{'工作日平均销量': 14.85, '周末平均销量': 19.72, '工作日记录数': 1320} 对 Series(周末, 工作日)。""" + labels = [_py(x) for x in e.index] + if len(labels) < 2 or not all(isinstance(x, str) and x for x in labels): + return False + used = set() + for lab, ev in zip(labels, e.values): + others = [o for o in labels if o != lab and lab in o] # '周末' 也是 '非周末' 的一部分时 + keys = [k for k, v in vals.items() + if lab in k and not any(o in k for o in others) + and (_is_num(v) or _pct(v) is not None) and _scalar_eq(v, _py(ev), tol)] + if len(keys) != 1 or keys[0] in used: + return False + used.add(keys[0]) + return True + + def _scale(v, f): """把结果里的数值乘以 f;没有可缩放的数值时返回 None。""" try: diff --git a/app/eval/report.py b/app/eval/report.py index b6e479e..474d24a 100644 --- a/app/eval/report.py +++ b/app/eval/report.py @@ -11,6 +11,7 @@ import json import os import platform +import shutil import subprocess from datetime import datetime @@ -20,8 +21,39 @@ _REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +_WIN_GIT = (r"C:\Program Files\Git\cmd\git.exe", r"C:\Program Files (x86)\Git\cmd\git.exe") +_git_problem = None # B48:最近一次没能调用 git 的原因,报告里写进 git_sha_note + + +def _git_exe() -> str: + """git 可执行文件:DATACHAT_GIT 环境变量 > PATH > Windows 默认安装位置。""" + exe = os.environ.get("DATACHAT_GIT") or shutil.which("git") + if exe: + return exe + for p in _WIN_GIT: + if os.path.exists(p): + return p + return "git" + + def _git(*args): - return subprocess.run(["git", "-C", _REPO, *args], capture_output=True, text=True, timeout=5) + global _git_problem + try: + r = subprocess.run([_git_exe(), "-C", _REPO, *args], capture_output=True, text=True, timeout=5) + except FileNotFoundError: + _git_problem = "找不到 git(不在 PATH 上;可设环境变量 DATACHAT_GIT 指向 git.exe)" + raise + except Exception as e: + _git_problem = f"调用 git 失败:{type(e).__name__}: {e}" + raise + if r.returncode != 0: + _git_problem = f"git {args[0]} 退出码 {r.returncode}:{(r.stderr or '').strip()[:200]}" + return r + + +def git_sha_note(): + """git_sha 带 "?" 时说明原因(如 safe.directory 拒绝、git 不在 PATH);正常时为 None。""" + return _git_problem def _sha_from_files(short: bool): @@ -49,6 +81,8 @@ def _sha_from_files(short: bool): def git_sha(short: bool = True) -> str: """当前 commit;工作区有改动时加 -dirty。git 不可用时退回读 .git 文件,并在末尾加 "?"。""" + global _git_problem + _git_problem = None try: r = _git("rev-parse", *(["--short"] if short else []), "HEAD") sha = r.stdout.strip() @@ -239,7 +273,8 @@ def block(items): agents = sorted({r.get("agent", "workflow") for r in self.results}) return { "time": datetime.now().isoformat(timespec="seconds"), - "git_sha": git_sha(), + "git_sha": (sha := git_sha()), + **({"git_sha_note": git_sha_note()} if sha.endswith("?") or sha == "unknown" else {}), "agent": agents[0] if len(agents) == 1 else agents, **self.meta, "overall": block(self.results), @@ -280,6 +315,8 @@ def print_summary(self): print("=" * 60) print(" DataChat 评测报告") print(f" 时间: {s['time']} commit: {s['git_sha']} agent: {s['agent']}") + if s.get("git_sha_note"): + print(f" ⚠ commit 后的 ? 表示没能确认工作区是否干净:{s['git_sha_note']}") if s.get("model"): print(f" 模型: {s['model']} temperature: {s.get('temperature')} 数据指纹: {s.get('data_sha256')}") print(f" 总用例: {self.total}") diff --git a/app/eval/rescore.py b/app/eval/rescore.py index 4cad2ec..ef3bad5 100644 --- a/app/eval/rescore.py +++ b/app/eval/rescore.py @@ -7,6 +7,7 @@ """ import json +from app.core.llm import api_error_text from app.core.result import refusal_from_answer from app.eval.cases import ALL_CASES from app.eval.report import load_obj @@ -17,9 +18,11 @@ def rescore(report_path: str, data_path: str | None = None) -> dict: data = json.load(open(report_path, encoding="utf-8")) cases = {c.id: c for c in ALL_CASES} runner = EvalRunner(data_path or data.get("data")) # 阶段 3:按题目的 dataset 取表 - rows, changed, skipped = [], [], [] + rows, changed, skipped, infra = [], [], [], [] for row in data["cases"]: case = cases.get(row["id"]) + if api_error_text(row.get("answer")) is not None: + infra.append(row["id"]) # B45:旧报告里网关错误 JSON 被当成了结论,按基础设施失败记 refusal = refusal_from_answer(row.get("answer")) if case is not None and refusal is not None and not row.get("actual_obj"): # B40:ReAct 没调工具、直接在结论里按契约拒答,旧报告里 result 为空 @@ -64,4 +67,4 @@ def rescore(report_path: str, data_path: str | None = None) -> dict: "correctness_rate": correct / n * 100 if n else None, "by_category": {k: {"correct": c, "n": t} for k, (c, t) in by_cat.items()}, "by_dataset": {k: {"correct": c, "n": t} for k, (c, t) in by_ds.items()}, - "changed": changed, "skipped_no_obj": skipped} # 无结构化结果或还原失败的题,沿用原判定 + "changed": changed, "skipped_no_obj": skipped, "infra_errors": infra} # 无结构化结果或还原失败的题,沿用原判定 diff --git a/app/react_agent.py b/app/react_agent.py index 9e0376d..abcdd2a 100644 --- a/app/react_agent.py +++ b/app/react_agent.py @@ -77,6 +77,8 @@ def run_python_on_data(code: str) -> str: "问\"哪个\"时 result 只放标签本身(如 '华中',不要拼接说明文字或数值),问\"多少\"给数值," "问\"各…/每…\"给以分组键为索引的 Series 或表,问\"哪些/哪条记录\"给筛选出的行(DataFrame)," "问题含多个小问时 result 用 dict 把每个小问都答上;数值、说明放在最终中文结论里。" + "只有一个问题时不要把答案装进 dict,也不要在 result 里附带样本数、计数、明细、中间统计或其他口径的结果" + "(这些写进最终结论);口径有歧义时选一种最合理的作为 result,其他口径在结论里说明。" "不要用中间表或整张排序表收尾。" "如果数据回答不了(表里没有所需的字段或信息),不要编造或用别的字段凑:" "仍然调用一次 run_python_on_data,代码为 result = '无法回答:<一句话原因>'(原因说明缺什么," diff --git a/docs/DEVLOG.md b/docs/DEVLOG.md index 4de8a42..ea7ef3d 100644 --- a/docs/DEVLOG.md +++ b/docs/DEVLOG.md @@ -393,7 +393,7 @@ xfail 用 strict 模式:问题一旦修好,对应测试会"意外通过"并 --- -## [7] 2026-10-10 · 阶段 3b 基线复核与修复 · commit <待提交> +## [7] 2026-10-10 · 阶段 3b 基线复核与修复 · commit 69a63e0(PR #7) ### 基线(S 运行,分支 eval-3 b87aef3,docs/eval/phase3/) - 模型 Qwen3.5-397B-A17B(网关 grg-aikits-test.grgbanking.com),temperature 0,Python 3.11.17,pandas 3.0.6,130 题,n=1。 @@ -427,3 +427,34 @@ xfail 用 strict 模式:问题一旦修好,对应测试会"意外通过"并 ### 下一步 - S 删除 eval-3 上的 eval_compare_oracle_20261010_092954.json 后合并(或由本次 PR 一并处理)。 - 合并后建议 S 再跑一次 `--agent react`,验证 B39/B40/B44 的实际效果;workflow 不必重跑。 + +## [8] 2026-10-10 · 阶段 3c ReAct 复跑复核与修复 · commit <待提交> + +### 复跑(S 运行,分支 eval-3b 4a435e0,docs/eval/phase3b/) +- 代码 69a63e0,模型 Qwen3.5-397B-A17B(同一网关),temperature 0,Python 3.11.17,pandas 3.0.6,130 题,n=1;只跑 react。 +- 报告 git_sha 仍为 "69a63e0?":S 的环境里 Python 调不到 git(B48)。 + +| 配置 | [7] 原判 | [7] 重评 | **3b 实跑** | 本阶段重评 | 应拒答 | +|---|---|---|---|---|---| +| react 有字典 | 86.9% | 90.8% | **92.3%**(120/130) | **96.9%**(126/130) | 8/9(另 1 题网关报错) | +| react 无字典 | 83.8% | 87.7% | **91.5%**(119/130) | **96.9%**(126/130) | 9/9 | + +- B39 生效:两次都没有出现 "Sorry, need more steps";B44 生效:没有出现 ConversionError;B40 生效:拒答 8/9、9/9。 +- 用本阶段比较器重评旧报告:workflow 98.5% / 97.7%(不变,规则 20 没有改动 workflow 的任何判定),react(阶段 3 报告)92.3% / 90.8%。 +- 本阶段重评后剩下的错题:有字典 agg_015(题意歧义,B47 改题)、corr_007(总体标准差且只留 1 位小数,B47 改题)、ord_030(没按 order_id 去重,75.77%)、ord_026(网关报错,B45);无字典 corr_007(只留 1 位小数)、ord_027(没去重)、ord_029(同时给出按销售额、按销量两种答案)、ord_030(两种口径并列)。真正算错的只有“没去重”这类数据陷阱,两次各 1 题。 + +### 本阶段修复 +| ID | 级别 | 位置 | 问题与修复 | +|---|---|---|---| +| B45 | P1 | app/core/llm.py、service.py、rescore.py | 网关以 200 状态码返回 `{"message":"prompt: A user's message must contain at least one image…"}` 作为回复内容,ReAct 把它当成结论(ord_026)。新增 `api_error_text()`:整段内容是只含错误字段(message/error/detail…)的 JSON 时识别为接口错误。ReAct 记 error_kind=llm(计入基础设施失败,不算答错);工作流的 LLMClient 视为可重试的 gateway_error;--rescore 在 infra_errors 里列出旧报告中的这类题 | +| B46 | P1 | app/eval/comparator.py、react_agent.py | ReAct 常把答案装进带附加信息的 dict(样本数、明细、中间统计),或返回排好序的整张 Series 回答"哪个"。比较器规则 20(只在其他规则都不匹配时才用,且要求唯一匹配):a) 期望标签、实际是按值排序的 Series 且榜首不并列 → 比较榜首;b) ≤8 项、键为文字的 dict:第一层恰好一个数值/标签等于期望,或恰好一个子对象等于期望 Series,或每个期望标签恰好对应一个含该标签的键;嵌套 dict 里的标量不参与,以免奖励罗列多种口径;全是数值且键 ≤4 字的视为分组表,不适用;c) 一句话字符串里恰好一个百分数(或恰好一个数字)。同时收紧 ReAct 提示词:单一问题不要把答案装进 dict 或附带样本数、明细、其他口径 | +| B47 | P2 | cases.py、cases_extra.py | agg_015 "平均每个地区的销售额"可读成各地区的单笔平均 → 改为"各地区总销售额的平均值是多少?";corr_007 没写样本/总体口径和精度 → 题目写明"样本标准差(保留两位小数)";ord_030 增加"排除已退款"的有效订单口径(73.15%)作为合理答案。注意:agg_015、corr_007 的题面变了,与之前的基线不完全可比 | +| B48 | P3 | app/eval/report.py | git_sha 带 "?" 的原因没有记录。现在依次找 DATACHAT_GIT 环境变量 → PATH → Windows 默认安装位置的 git;仍失败时报告写 git_sha_note(找不到 git,或 git 的退出码与 stderr,例如 safe.directory 拒绝),控制台也会打印 | + +### 指标快照 +- 单测:Linux Python 3.11 **386 passed**。 + +### 下一步 +- S 再跑一次 `--agent react`(有字典 / 无字典),验证 B46 提示词收紧后 dict 形态是否减少、B47 改题后的效果;顺便看报告里有没有 git_sha_note。 +- 阶段 3 收尾后进入阶段 4(见 roadmap/plan.md)。 + diff --git a/tests/test_comparator.py b/tests/test_comparator.py index d86f951..e6176b3 100644 --- a/tests/test_comparator.py +++ b/tests/test_comparator.py @@ -212,3 +212,56 @@ def test_rule19_rank_numbers_vs_values(): assert results_equal(pd.Series([3, 1, 2, 4, 5], index=list("ABECD")), e) assert not results_equal(pd.Series([1, 2, 3, 4, 5], index=list("EBACD")), e) assert not results_equal(pd.Series([1, 2, 3], index=list("BEA")), e) + + +# ---- 规则 20(B46):答案外面裹了附加信息 +def test_rule20_label_vs_sorted_series(): + s = pd.Series([153906.21, 152786.38, 96935.99], index=pd.Index(["杭州", "上海", "武汉"], name="city")) + assert results_equal(s, "杭州") + assert results_equal(s.sort_values(), "武汉") # 升序时榜首是最小值 + assert not results_equal(s, "上海") + assert not results_equal(s.sort_index(), "杭州") # 没按值排序:看不出答案 + assert not results_equal(pd.Series([5.0, 5.0, 1.0], index=["甲", "乙", "丙"]), "甲") # 榜首并列 + + +def test_rule20_scalar_inside_dict(): + a = {"2023年内离职人数": 14, "2023-01-01在职人数": 183, "流失率(%)": 7.65} + assert results_equal(a, 7.65) + assert not results_equal(a, 7.5) + # 不唯一(≤4 项的 dict 归规则 10 管,这里用 5 项) + assert not results_equal({"甲项指标值": 7.65, "乙项指标值": 7.65, "丙项指标值": 1, "丁项指标值": 2, "戊项指标值": 3}, 7.65) + assert results_equal({"甲项指标值": 7.65, "乙项指标值": 7.7, "丙项指标值": 1, "丁项指标值": 2, "戊项指标值": 3}, 7.65) + assert not results_equal({"2024-01": 1.0, "2024-02": 7.65, "2024-03": 3.0}, 7.65) # 时间序列 + assert not results_equal({f"k{i}": i for i in range(9)} | {"x": 7.65}, 7.65) # 太多项 + # 嵌套的多种口径不算对 + hedged = {"口径A": {"复购率": "73.15%"}, "口径B": {"复购率": "75.46%"}} + assert not results_equal(hedged, 75.46) + + +def test_rule20_label_inside_dict(): + a = {"top_day": "2024-05-04", "top_sold": 579, "top5": {"2024-05-04": 579, "2024-05-05": 560}} + assert results_equal(a, "2024-05-04") + assert results_equal({"总销量最高的SKU": "K100", "总销量": 14769}, "K100") + assert not results_equal({"按销售额最好": "数码", "按销量最好": "服饰", "x": 1}, "数码") # 两个标签 + + +def test_rule20_series_inside_dict(): + e = pd.Series([545, 500, 299], index=pd.Index(["S01", "S02", "S05"], name="store")) + assert results_equal({"各门店期末库存": e.copy(), "合计": 1344}, e) + assert results_equal({"总行数": 1218, "各店": {"S01": 545, "S02": 500, "S05": 299}, + "分布": {"a": 1, "b": 2}}, e) + assert not results_equal({"各店": {"S01": 545, "S02": 501, "S05": 299}, "合计": 1345}, e) + + +def test_rule20_dict_keys_contain_labels(): + e = pd.Series([19.72, 14.85], index=pd.Index(["周末", "工作日"], name="date")) + assert results_equal({"工作日平均销量": 14.85, "周末平均销量": 19.72, "工作日记录数": 1320, "周末记录数": 520}, e) + assert not results_equal({"工作日平均销量": 19.72, "周末平均销量": 14.85}, e) + assert not results_equal({"工作日平均": 14.85, "周末平均": 19.72, "周末均值": 19.72}, e) # 一个标签对应两个键 + + +def test_rule20_sentence_string(): + assert results_equal("退款率 7.00%(已退款 84 / 去重订单 1200)", 7.0) + assert results_equal("共 1,200 单", 1200) + assert not results_equal("7.00% 或 7.5%", 7.0) + assert not results_equal("已退款 84 / 去重订单 1200", 84) diff --git a/tests/test_core_analyze.py b/tests/test_core_analyze.py index ef56299..695c9dd 100644 --- a/tests/test_core_analyze.py +++ b/tests/test_core_analyze.py @@ -200,3 +200,12 @@ def test_react_sorry_placeholder_is_error(sample_path): m = ScriptedChatModel(replies=["Sorry, need more steps to process this request."], calls=[]) res = analyze(sample_path, "q", agent="react", chat_model=m, llm_config=FAST) assert res.answer == "" and res.error_kind == "agent" and "recursion_limit" in res.error + + +def test_react_gateway_error_json_is_llm_error(sample_path): + """B45:网关返回 {"message": ...} 作为回复内容时,不是结论,按 LLM 基础设施失败记。""" + m = ScriptedChatModel(replies=['{"message":"prompt: A user\'s message must contain at least one image."}'], + calls=[]) + res = analyze(sample_path, "q", agent="react", chat_model=m, llm_config=FAST) + assert res.answer == "" and res.error_kind == "llm" and "gateway_error" in res.error + assert not res.executed diff --git a/tests/test_eval_cases.py b/tests/test_eval_cases.py index 0078b9e..1a0fa61 100644 --- a/tests/test_eval_cases.py +++ b/tests/test_eval_cases.py @@ -186,3 +186,22 @@ def boom(*a, **k): monkeypatch.setattr(rep, "_git", boom) sha = rep.git_sha() assert sha != "unknown" and sha.endswith("?") and len(sha) == 8 + + +def test_ord_030_refund_excluded_alt(): + """B47:排除已退款的有效订单口径(73.15%)也算对。""" + case = next(c for c in ALL_CASES if c.id == "ord_030") + outs, _ = _RUNNER.expected_all(case) + assert score(73.15, case, outs) and score(75.46, case, outs) + assert not score(75.77, case, outs) # 没按 order_id 去重 + + +def test_git_sha_note_when_git_missing(monkeypatch): + """B48:调不到 git 时 sha 带 "?",并在报告里写明原因。""" + from app.eval import report + monkeypatch.setenv("DATACHAT_GIT", os.path.join(ROOT, "no-such-git.exe")) + sha = report.git_sha() + assert sha.endswith("?") or sha == "unknown" + assert "DATACHAT_GIT" in (report.git_sha_note() or "") + s = EvalReport([], meta={}).summary() + assert "git_sha_note" in s diff --git a/tests/test_llm.py b/tests/test_llm.py index 82808af..05eb2eb 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -92,3 +92,27 @@ class M: t.add_message(type("M2", (), {"usage_metadata": {"input_tokens": 3, "output_tokens": 4}})()) u = t.snapshot() assert u.calls == 2 and u.total_tokens == 7 and u.reported is False + + +# ---- B45:网关以错误 JSON 充当回复 +def test_api_error_text(): + from app.core.llm import api_error_text + msg = '{"message":"prompt: A user\'s message must contain at least one image or a PDF or audio."}' + assert api_error_text(msg).startswith("prompt: A user's message") + assert api_error_text('{"error": {"message": "rate limited", "code": 429}}') == "rate limited" + assert api_error_text('{"result": 1, "message": "x"}') is None # 有其他键:不是错误包 + assert api_error_text("各地区销售额如下……") is None + assert api_error_text("{不是 JSON}") is None and api_error_text(None) is None + + +def test_gateway_error_content_is_retried(): + client, sleeps = _client(['{"message": "upstream glitch"}', "好"], backoff_base=1.0) + assert client.invoke("hi") == "好" + assert sleeps == [1.0] + + +def test_gateway_error_content_gives_up(): + client, _ = _client(['{"message": "bad"}'] * 3, max_retries=1, backoff_base=0.0) + with pytest.raises(LLMError) as ei: + client.invoke("hi") + assert ei.value.kind == "gateway_error"