From d4836b29135871fe22bb3aeacc134be34e6c159a Mon Sep 17 00:00:00 2001 From: Lgs Date: Tue, 13 Oct 2026 00:11:47 +0800 Subject: [PATCH] fix(ask): preserve multi-module findings when agents fail --- docs/en/SWARM_PROTOCOL.md | 12 ++ docs/es/SWARM_PROTOCOL.md | 12 ++ docs/zh/SWARM_PROTOCOL.md | 9 + engine/repobrain_engine/hub/ask_pipeline.py | 45 ++++- engine/tests/test_ask_partial_results.py | 197 ++++++++++++++++++++ 5 files changed, 271 insertions(+), 4 deletions(-) create mode 100644 engine/tests/test_ask_partial_results.py diff --git a/docs/en/SWARM_PROTOCOL.md b/docs/en/SWARM_PROTOCOL.md index fbdf84aea..5e6548a40 100644 --- a/docs/en/SWARM_PROTOCOL.md +++ b/docs/en/SWARM_PROTOCOL.md @@ -164,6 +164,18 @@ The ask pipeline implements a context-sensitive fallback mechanism: This ensures ask functionality remains available even if knowledge base is partially generated or uses older formats. +For cross-module questions in the `map.md` / `agents/*.md` path, Reader Agents +run in parallel and a Synthesizer combines their findings. If a reader fails +after retries or returns empty output, usable findings from other readers are +retained and the answer names the missing modules in a **Partial answer** notice. +A reader that explicitly answers `(not relevant)` is not counted as a failure. +If synthesis fails or returns empty output, all usable reader answers are +returned in routing/document order with their module labels and original code +references, plus a notice that the findings have not been reconciled. This +fallback makes no additional model call. If no reader produces usable findings, +the existing legacy fallback chain still applies. Retry the question to recover +missing answers; this is separate from incomplete knowledge that needs refresh. + ## 🔧 Configuration & Extension ### Using Different LLM Backends diff --git a/docs/es/SWARM_PROTOCOL.md b/docs/es/SWARM_PROTOCOL.md index f67a106cb..a5ba9639b 100644 --- a/docs/es/SWARM_PROTOCOL.md +++ b/docs/es/SWARM_PROTOCOL.md @@ -164,6 +164,18 @@ El pipeline de ask implementa un mecanismo de fallback sensible al contexto: Esto asegura que la funcionalidad ask permanezca disponible incluso si la base de conocimiento está parcialmente generada o usa formatos antiguos. +Para preguntas entre módulos en la ruta `map.md` / `agents/*.md`, los Reader +Agents trabajan en paralelo y un Synthesizer combina sus hallazgos. Si un lector +falla tras los reintentos o devuelve una salida vacía, se conservan los resultados +útiles y un aviso **Partial answer** identifica los módulos sin respuesta. +La respuesta explícita `(not relevant)` no cuenta como un fallo. Si la síntesis +falla o devuelve una salida vacía, se devuelven todas las respuestas útiles en +orden de enrutamiento y documento, con sus módulos y referencias originales, +indicando que los hallazgos no se han conciliado. Este fallback no realiza otra +llamada al modelo. Si ningún lector aporta resultados útiles, se mantiene la +cadena de fallback existente. Repite la pregunta para recuperar las respuestas +faltantes; esto es distinto de una base de conocimiento que necesita refrescarse. + ## 🔧 Configuración y Extensión ### Usando Diferentes Backends LLM diff --git a/docs/zh/SWARM_PROTOCOL.md b/docs/zh/SWARM_PROTOCOL.md index 04dd8920c..660dea370 100644 --- a/docs/zh/SWARM_PROTOCOL.md +++ b/docs/zh/SWARM_PROTOCOL.md @@ -164,6 +164,15 @@ Ask pipeline 实现了上下文感知的回退机制： 这确保了即使知识库部分生成或使用旧格式，ask 功能仍然可用。 +在 `map.md` / `agents/*.md` 路径中，跨模块问题由多个 Reader Agent 并行回答， +再由 Synthesizer 汇总。如果某个 Reader 重试后仍失败或返回空白，系统会保留 +其他 Reader 的有效结果，并在答案开头通过 **Partial answer** 提示列出缺失模块。 +明确回答 `(not relevant)` 的 Reader 不计为失败。 +如果汇总失败或返回空白，系统会按路由和文档顺序返回全部有效回答，保留模块标识 +和原始代码引用，并提示这些结果尚未经统一汇总，可能存在未解决的矛盾。 +该回退不增加模型调用；如果所有 Reader 都没有有效结果，仍沿用原有的旧版回退链。 +重新提问可重试获取缺失的回答；这与需要刷新知识库的知识不完整提示是两种情况。 + ## 🔧 配置与扩展 ### 使用不同的 LLM 后端 diff --git a/engine/repobrain_engine/hub/ask_pipeline.py b/engine/repobrain_engine/hub/ask_pipeline.py index 0d4fdcd06..63bf8bdad 100644 --- a/engine/repobrain_engine/hub/ask_pipeline.py +++ b/engine/repobrain_engine/hub/ask_pipeline.py @@ -839,6 +839,18 @@ def _prepend_degradation_banner( return banner + answer +def _prepend_reader_failure_banner(answer: str, failed_modules: list[str]) -> str: + """Keep ask-time coverage gaps visible even if synthesis omits them.""" + if not failed_modules: + return answer + banner = ( + "> ⚠ Partial answer: could not obtain findings for module(s): " + f"{', '.join(failed_modules)}. The answer may not cover these modules; " + "retry the question.\n\n" + ) + return banner + answer + + async def _ask_with_agent_md(workspace: Path, question: str) -> str | None: """Answer a question by routing through map.md → agent.md files. @@ -1115,6 +1127,9 @@ async def _answer_from_doc( stream_enabled=False, progress_label=None, ) + if not answer.strip(): + logger.warning("ask: reader for module %s returned empty output", mod_name) + return mod_name, None return mod_name, answer except Exception as exc: logger.warning("ask: reader for module %s failed: %s", mod_name, exc) @@ -1124,6 +1139,9 @@ async def _answer_from_doc( *[_answer_from_doc(name, knowledge) for name, knowledge in module_knowledge] ) + # A failed reader is missing evidence, not a finding of irrelevance. + failed_modules = [name for name, ans in partial_answers if ans is None] + # Filter out failures and irrelevant answers (exact-match only — answers # that mention "(not relevant)" inside a longer reply should not be # dropped). @@ -1138,12 +1156,21 @@ async def _answer_from_doc( if len(valid_answers) == 1: print("[4/4] Answer synthesized.", file=sys.stderr) - return _prepend_degradation_banner(valid_answers[0][1], degraded_modules) + answer = _prepend_reader_failure_banner(valid_answers[0][1], failed_modules) + return _prepend_degradation_banner(answer, degraded_modules) # Synthesize multiple answers synth_parts = "\n\n---\n\n".join( f"**From {name}:**\n{ans}" for name, ans in valid_answers ) + coverage_section = "" + if failed_modules: + coverage_section = ( + "\n\nNo usable reader response was obtained for these modules: " + f"{', '.join(failed_modules)}. Their evidence is missing, not " + "irrelevant. Do not infer their behavior or claim complete " + "coverage of the question." + ) synth_prompt = f"""\ Synthesize these partial answers into one coherent response. Keep all specific references (file paths, function names, line numbers). @@ -1152,6 +1179,7 @@ async def _answer_from_doc( Question: {question} {project_section} +{coverage_section} Partial answers: {synth_parts} @@ -1170,12 +1198,21 @@ async def _answer_from_doc( stream_enabled=settings.STREAM_ENABLED, progress_label="Synthesizing answers from multiple modules...", ) + if not answer.strip(): + raise ValueError("empty synthesis output") print("[4/4] Answer synthesized from multiple modules.", file=sys.stderr) - return _prepend_degradation_banner(answer, degraded_modules) except Exception as exc: logger.warning("ask: multi-module synthesis failed: %s", exc) - # Return the first valid answer as fallback - return _prepend_degradation_banner(valid_answers[0][1], degraded_modules) + # Preserve every successful reader's findings and citations without + # another model call. Keep attribution; the findings may conflict. + answer = ( + "> ⚠ Answer synthesis failed; showing individual module findings " + "below. These findings have not been reconciled.\n\n" + + synth_parts + ) + print("[4/4] Returning individual module findings.", file=sys.stderr) + answer = _prepend_reader_failure_banner(answer, failed_modules) + return _prepend_degradation_banner(answer, degraded_modules) async def _ask_with_legacy_facts(workspace: Path, question: str) -> str | None: diff --git a/engine/tests/test_ask_partial_results.py b/engine/tests/test_ask_partial_results.py new file mode 100644 index 000000000..2cd0a31ae --- /dev/null +++ b/engine/tests/test_ask_partial_results.py @@ -0,0 +1,197 @@ +"""Fault-injection tests for the map -> readers -> synthesizer ask path.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from repobrain_engine.hub import ask_pipeline as ask_mod + + +AUTH_ANSWER = "Authentication checks the token in `auth.py:1-4`." +API_ANSWER = "The API calls authenticate() in `api.py:8-12`." +DB_ANSWER = "Sessions are stored in `db.py:20-24`." + + +@pytest.fixture +def run_ask(tmp_path, monkeypatch): + """Use real Agent/tool construction, replacing only the model runner.""" + import agents + from repobrain_engine import config + from repobrain_engine.hub import agents as hub_agents + + monkeypatch.setattr(config, "get_settings", lambda: SimpleNamespace(STREAM_ENABLED=False)) + monkeypatch.setattr(hub_agents, "create_model", lambda settings: "test-model") + monkeypatch.setenv("RB_ASK_RETRY_COUNT", "0") + monkeypatch.setenv("RB_API_CONCURRENCY", "2") + prompts = {} + calls = [] + + async def run(outcomes, *, docs=None, router="MODULES: auth, api, db"): + if docs is None: + docs = {"auth": "Auth knowledge", "api": "API knowledge", "db": "DB knowledge"} + rb_dir = tmp_path / ".repobrain" + rb_dir.mkdir(exist_ok=True) + (rb_dir / "map.md").write_text("\n".join(docs), encoding="utf-8") + for name, content in docs.items(): + path = rb_dir / "agents" / f"{name}.md" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + async def fake_run(agent, prompt, **kwargs): + calls.append(agent.name) + prompts[agent.name] = prompt + outcome = router if agent.name == "QuickRouter" else outcomes[agent.name] + if isinstance(outcome, BaseException): + raise outcome + return SimpleNamespace(final_output=outcome) + + monkeypatch.setattr(agents.Runner, "run", fake_run) + return await ask_mod._ask_with_agent_md(tmp_path, "How do auth, API and sessions interact?") + + return SimpleNamespace(run=run, prompts=prompts, calls=calls) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("synthesis", [RuntimeError("model unavailable"), "", " \n "]) +async def test_synthesis_failure_preserves_all_module_findings(run_ask, synthesis): + answer = await run_ask.run({ + "Reader_auth": AUTH_ANSWER, + "Reader_api": API_ANSWER, + "Reader_db": DB_ANSWER, + "Synthesizer": synthesis, + }) + + assert answer is not None + for finding in (AUTH_ANSWER, API_ANSWER, DB_ANSWER): + assert finding in answer + assert answer.index(AUTH_ANSWER) < answer.index(API_ANSWER) < answer.index(DB_ANSWER) + assert "synthesis" in answer.lower() + assert "**From auth:**" in answer + assert "**From api:**" in answer + assert "**From db:**" in answer + assert len(run_ask.calls) == 5 # No extra model call for the fallback. + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reader", [RuntimeError("private provider diagnostics"), "", " \n "]) +async def test_one_surviving_reader_reports_missing_coverage(run_ask, reader): + answer = await run_ask.run({ + "Reader_auth": AUTH_ANSWER, + "Reader_api": reader, + "Reader_db": "(not relevant)", + }) + + assert answer is not None + assert AUTH_ANSWER in answer + assert "Partial answer" in answer + assert "api" in answer.split("\n\n")[0] + assert "db" not in answer.split("\n\n")[0] + assert "private provider diagnostics" not in answer + assert "Synthesizer" not in run_ask.calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("synthesis", ["Combined findings.", RuntimeError("synthesis unavailable")]) +async def test_failed_reader_is_visible_after_synthesis(run_ask, synthesis): + answer = await run_ask.run({ + "Reader_auth": AUTH_ANSWER, + "Reader_api": API_ANSWER, + "Reader_db": RuntimeError("private provider diagnostics"), + "Synthesizer": synthesis, + }) + + assert answer is not None + assert answer.count("Partial answer") == 1 + assert "db" in answer.split("\n\n")[0] + assert "private provider diagnostics" not in answer + prompt = run_ask.prompts["Synthesizer"] + assert AUTH_ANSWER in prompt and API_ANSWER in prompt + assert "No usable reader response" in prompt + assert "db" in prompt + assert "Do not infer their behavior" in prompt + if isinstance(synthesis, Exception): + assert AUTH_ANSWER in answer and API_ANSWER in answer + else: + assert answer.endswith(synthesis) + + +@pytest.mark.asyncio +async def test_healthy_synthesis_is_unchanged(run_ask): + answer = await run_ask.run({ + "Reader_auth": AUTH_ANSWER, + "Reader_api": API_ANSWER, + "Reader_db": DB_ANSWER, + "Synthesizer": "The complete answer.", + }) + + assert answer == "The complete answer." + prompt = run_ask.prompts["Synthesizer"] + assert all(finding in prompt for finding in (AUTH_ANSWER, API_ANSWER, DB_ANSWER)) + assert "No usable reader response" not in prompt + assert len(run_ask.calls) == 5 # Router, three readers, synthesizer. + + +@pytest.mark.asyncio +async def test_irrelevance_does_not_report_failure(run_ask): + answer = await run_ask.run({ + "Reader_auth": AUTH_ANSWER, + "Reader_api": "(not relevant)", + "Reader_db": " \n (NOT RELEVANT) \n ", + }) + + assert answer == AUTH_ANSWER + assert "Synthesizer" not in run_ask.calls + + +@pytest.mark.asyncio +async def test_irrelevance_phrase_inside_findings_is_preserved(run_ask): + finding = "The UI is (not relevant), but sessions are stored in `db.py:20-24`." + answer = await run_ask.run({ + "Reader_auth": AUTH_ANSWER, + "Reader_api": "(not relevant)", + "Reader_db": finding, + "Synthesizer": RuntimeError("unavailable"), + }) + + assert AUTH_ANSWER in answer and finding in answer + assert "Partial answer" not in answer + assert "**From api:**" not in answer + + +@pytest.mark.asyncio +async def test_no_usable_findings_preserves_legacy_fallback(run_ask): + answer = await run_ask.run({ + "Reader_auth": RuntimeError("unavailable"), + "Reader_api": " \n ", + "Reader_db": "(not relevant)", + }) + + assert answer is None + assert "Synthesizer" not in run_ask.calls + + +@pytest.mark.asyncio +async def test_group_attribution_and_refresh_warning_survive_fallback(run_ask): + from repobrain_engine.hub._constants import AGENT_MD_FALLBACK_MARKER + + answer = await run_ask.run( + { + "Reader_auth/token": AUTH_ANSWER, + "Reader_auth/session": RuntimeError("unavailable"), + "Reader_api": API_ANSWER, + "Synthesizer": RuntimeError("unavailable"), + }, + docs={ + "auth/token": AGENT_MD_FALLBACK_MARKER, + "auth/session": "Session knowledge", + "api": "API knowledge", + }, + router="MODULES: auth, api", + ) + + assert "Incomplete knowledge for module(s): auth/token" in answer + assert "Partial answer" in answer and "auth/session" in answer + assert "**From auth/token:**" in answer + assert AUTH_ANSWER in answer and API_ANSWER in answer