Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

## [Unreleased]

### Fixed

- Stop treating a bare `review` keyword as automatic research routing. Software/PR/security-review collocates now score as `code`; literature collocates stay on `research`; an ambiguous lone `review` no longer steers the model switcher.

## [3.10.3] - 2026-08-10

### Fixed
Expand Down
110 changes: 100 additions & 10 deletions integrations/hermes/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,59 @@ def model_ref(value: str) -> str:
"default": "low",
}

# Bare "review" is overloaded: code/PR/security review vs literature review.
# Count it only when the surrounding prompt disambiguates; never let a lone
# "review" steal a turn into the research lane.
CODE_REVIEW_CONTEXT = [
"pr",
"pull request",
"pull-request",
"code review",
"codex",
"github",
"gitlab",
"merge",
"commit",
"branch",
"diff",
"patch",
"ci",
"issue",
"thread",
"inline",
"unresolved",
"blocker",
"head",
"lint",
"regression",
"test",
"security review",
"release review",
"exact-head",
"exact head",
]

RESEARCH_REVIEW_CONTEXT = [
"literature",
"paper",
"papers",
"study",
"studies",
"evidence",
"survey",
"journal",
"academic",
"peer-reviewed",
"peer reviewed",
"meta-analysis",
"meta analysis",
"research",
"analyze",
"investigate",
"compare",
"explain",
]


def load_config(path: str | None = None) -> Config | None:
config_path = path or os.getenv("ZEROAPI_CONFIG_PATH")
Expand Down Expand Up @@ -590,6 +643,36 @@ def _resolve_continuation_category(
return _history_continuation_category(config, conversation_history, allowed)


def _has_any_whole_keyword(lower: str, keywords: list[str]) -> bool:
return any(_keyword_regex(keyword).search(lower) for keyword in keywords)


def resolve_review_keyword_category(lower: str) -> str | None:
"""Attribute a matched 'review' token to code, research, or neither."""
code_context = _has_any_whole_keyword(lower, CODE_REVIEW_CONTEXT)
research_context = _has_any_whole_keyword(lower, RESEARCH_REVIEW_CONTEXT)
if code_context and not research_context:
return "code"
if research_context and not code_context:
return "research"
if code_context and research_context:
# Mixed prompts lean code: the durable action is software work.
return "code"
return None


def _score_keyword_match(category: str, keyword: str, lower: str) -> tuple[int, str] | None:
matches = _keyword_regex(keyword).findall(lower)
if not matches:
return None
if keyword.lower() == "review":
resolved = resolve_review_keyword_category(lower)
if resolved is None:
return None
return len(matches), resolved
return len(matches), category


def _classify(config: Config, prompt: str, workspace_hints: list[Any] | None = None) -> tuple[TaskCategory, str, str]:
lower = prompt.lower().strip()
if not lower:
Expand All @@ -600,24 +683,31 @@ def _classify(config: Config, prompt: str, workspace_hints: list[Any] | None = N
best_category = "default"
best_reason = "no_match"
best_score = 0
scores: dict[str, dict[str, Any]] = {}
keywords = config.get("keywords", {})
if isinstance(keywords, dict):
for category, values in keywords.items():
if not isinstance(category, str) or not isinstance(values, list):
continue
score = 0
first = ""
for keyword in values:
if not isinstance(keyword, str):
continue
matches = _keyword_regex(keyword).findall(lower)
if matches:
score += len(matches)
first = first or keyword
if score > best_score:
best_category = category
best_reason = f"keyword:{first}" if first else "no_match"
best_score = score
scored = _score_keyword_match(category, keyword, lower)
if scored is None:
continue
match_score, attributed = scored
bucket = scores.setdefault(attributed, {"score": 0, "first": ""})
bucket["score"] += match_score
if not bucket["first"]:
bucket["first"] = (
f"review→{attributed}" if keyword.lower() == "review" else keyword
)

for category, bucket in scores.items():
if bucket["score"] > best_score:
best_category = category
best_reason = f"keyword:{bucket['first']}" if bucket["first"] else "no_match"
best_score = bucket["score"]

if best_score == 0 and workspace_hints and len(workspace_hints) == 1:
hint = workspace_hints[0]
Expand Down
41 changes: 40 additions & 1 deletion integrations/hermes/test_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,12 @@
HERMES_PROVIDER_MAP,
ZeroAPIRouter,
_allowed_by_subscriptions,
_classify,
_hermes_provider,
_resolve_capacity,
_valid_config,
load_config,
resolve_review_keyword_category,
)


Expand Down Expand Up @@ -54,7 +56,8 @@
},
"workspace_hints": {},
"keywords": {
"code": ["implement", "refactor", "fix", "debug"],
"code": ["implement", "refactor", "fix", "debug", "pr", "diff", "test"],
"research": ["research", "analyze", "investigate", "review", "paper", "evidence"],
"orchestration": ["coordinate", "workflow"],
"fast": ["quick", "format"],
},
Expand All @@ -81,6 +84,42 @@ def test_routes_orchestration_to_hermes_provider(self):
self.assertEqual(route["model"], "glm-5.1")
self.assertIn("zeroapi:orchestration", route["reason"])

def test_bare_review_does_not_route_to_research(self):
category, reason, _risk = _classify(CONFIG, "please review this carefully")
self.assertEqual(category, "default")
self.assertEqual(reason, "no_match")
route = ZeroAPIRouter(CONFIG).resolve(
"please review this carefully",
current_model="openai-codex/gpt-5.4",
)
self.assertIsNone(route)

def test_software_review_language_routes_to_code_not_research(self):
prompt = "Codex review: address the unresolved P2 on this PR head before merge"
category, reason, _risk = _classify(CONFIG, prompt)
self.assertEqual(category, "code")
self.assertIn("keyword:", reason)
self.assertNotEqual(category, "research")
# Already on the code primary => no switch needed.
route = ZeroAPIRouter(CONFIG).resolve(prompt, current_model="openai-codex/gpt-5.4")
self.assertIsNone(route)

def test_literature_review_stays_research(self):
category, reason, _risk = _classify(
CONFIG,
"review the literature and compare evidence across peer-reviewed papers",
)
self.assertEqual(category, "research")
self.assertTrue(reason.startswith("keyword:"))

def test_resolve_review_keyword_category_helpers(self):
self.assertIsNone(resolve_review_keyword_category("please review this carefully"))
self.assertEqual(resolve_review_keyword_category("review this pr head"), "code")
self.assertEqual(
resolve_review_keyword_category("review the literature and papers"),
"research",
)

def test_keeps_current_model_for_default_messages(self):
route = ZeroAPIRouter(CONFIG).resolve("buna bir bak", current_model="openai-codex/gpt-5.4")
self.assertIsNone(route)
Expand Down
40 changes: 39 additions & 1 deletion plugin/__tests__/classifier.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { classifyTask } from "../classifier.js";

const defaultKeywords = {
code: ["implement", "function", "class", "refactor", "fix", "test", "debug", "PR", "diff", "migration", "component", "endpoint"],
research: ["research", "analyze", "explain", "compare", "paper", "evidence", "investigate", "study"],
research: ["research", "analyze", "explain", "compare", "paper", "evidence", "investigate", "study", "review"],
orchestration: ["orchestrate", "coordinate", "pipeline", "workflow", "sequence", "parallel", "fan-out"],
math: ["calculate", "solve", "equation", "proof", "integral", "probability", "optimize", "formula"],
fast: ["quick", "simple", "format", "convert", "translate", "rename", "one-liner", "list"],
Expand Down Expand Up @@ -131,4 +131,42 @@ describe("classifyTask", () => {
// "token" is NOT in our default highRisk list (only deploy, delete, drop, rm, production, credentials, secret, password)
expect(result.risk).not.toBe("high");
});

it("ignores bare 'review' without disambiguating context", () => {
const result = classifyTask("please review this carefully", defaultKeywords, highRisk);
expect(result.category).toBe("default");
expect(result.reason).toBe("no_match");
});

it("routes software/PR review language to code, not research", () => {
const cases = [
"Codex review: address the unresolved P2 on this PR head",
"review the pull request and merge if CI is green",
"exact-head code review before merge",
"security review of this diff and CI blocker",
];
for (const prompt of cases) {
const result = classifyTask(prompt, defaultKeywords, highRisk);
expect(result.category).toBe("code");
expect(result.reason).toMatch(/keyword:review→code|keyword:(pr|diff|test|code review)/i);
}
});

it("keeps literature review language on research", () => {
const result = classifyTask(
"review the literature and compare evidence across peer-reviewed papers",
defaultKeywords,
highRisk,
);
expect(result.category).toBe("research");
});

it("does not let a lone 'review' override an explicit code task", () => {
const result = classifyTask(
"implement the fix after the code review comments",
defaultKeywords,
highRisk,
);
expect(result.category).toBe("code");
});
});
Loading