diff --git a/application/tests/cheatsheet_rerank_test.py b/application/tests/cheatsheet_rerank_test.py new file mode 100644 index 000000000..ba2aa345a --- /dev/null +++ b/application/tests/cheatsheet_rerank_test.py @@ -0,0 +1,220 @@ +import time +import unittest + +from application.utils.external_project_parsers.parsers.cheatsheet_rerank import ( + RerankError, + build_rationale_graph, + classify_confidence, + generate_link_rationale, +) + +# LangGraph's first StateGraph().compile() in a process pays a one-time +# lazy-import/compile cost (observed ~0.5s), unrelated to anything under +# test. Pay it here, at module load, so timing-sensitive assertions (e.g. +# test_llm_timeout_falls_back) measure only our own timeout mechanism, both +# in isolation and as part of the full suite. +build_rationale_graph() + + +SECTION_TEXT = ( + "Secrets Management Cheat Sheet: guidance on secure storage, rotation, " + "and operational handling of secrets." +) +CRE_ID = "623-550" +CRE_TEXT = "Operational secret rotation controls." + + +class ClassifyConfidenceTest(unittest.TestCase): + def test_high(self): + self.assertEqual(classify_confidence(0.9), "high") + self.assertEqual(classify_confidence(0.85), "high") + + def test_medium(self): + self.assertEqual(classify_confidence(0.7), "medium") + self.assertEqual(classify_confidence(0.84), "medium") + + def test_low(self): + self.assertEqual(classify_confidence(0.0), "low") + self.assertEqual(classify_confidence(0.69), "low") + + def test_out_of_range_raises(self): + with self.assertRaises(RerankError): + classify_confidence(1.5) + with self.assertRaises(RerankError): + classify_confidence(-0.1) + + def test_non_numeric_raises(self): + with self.assertRaises(RerankError): + classify_confidence("high") # type: ignore[arg-type] + + +class GenerateLinkRationaleTest(unittest.TestCase): + def test_empty_cre_id_raises(self): + with self.assertRaises(RerankError): + generate_link_rationale(SECTION_TEXT, "", CRE_TEXT, 0.9) + + def test_invalid_score_raises_before_llm_call(self): + called = {"n": 0} + + def stub(system, user, *, model): + called["n"] += 1 + return {"rationale": "x"} + + with self.assertRaises(RerankError): + generate_link_rationale( + SECTION_TEXT, CRE_ID, CRE_TEXT, 1.5, llm_rationale_fn=stub + ) + self.assertEqual(called["n"], 0) + + def test_zero_timeout_seconds_raises(self): + with self.assertRaises(RerankError): + generate_link_rationale( + SECTION_TEXT, CRE_ID, CRE_TEXT, 0.9, timeout_seconds=0 + ) + + def test_infinite_timeout_seconds_raises(self): + with self.assertRaises(RerankError): + generate_link_rationale( + SECTION_TEXT, + CRE_ID, + CRE_TEXT, + 0.9, + timeout_seconds=float("inf"), + ) + + def test_boolean_timeout_seconds_raises(self): + with self.assertRaises(RerankError): + generate_link_rationale( + SECTION_TEXT, CRE_ID, CRE_TEXT, 0.9, timeout_seconds=True + ) + + def test_successful_generation_produces_rationale_and_trace(self): + def stub(system, user, *, model): + self.assertIn("CHEATSHEET_TEXT", user) + self.assertIn(CRE_ID, user) + return {"rationale": "Both cover secret rotation controls directly."} + + result = generate_link_rationale( + SECTION_TEXT, CRE_ID, CRE_TEXT, 0.91, llm_rationale_fn=stub + ) + self.assertEqual(result.cre_id, CRE_ID) + self.assertIn("rotation", result.rationale.lower()) + self.assertEqual(result.confidence, "high") + self.assertFalse(result.fallback_used) + self.assertFalse(result.trace.fallback_used) + self.assertEqual(result.trace.prompt_version, "v2") + self.assertIsNone(result.trace.fallback_reason) + + def test_llm_exception_falls_back_to_score_only_rationale(self): + def stub(system, user, *, model): + raise RuntimeError("provider unavailable") + + result = generate_link_rationale( + SECTION_TEXT, CRE_ID, CRE_TEXT, 0.42, llm_rationale_fn=stub + ) + self.assertTrue(result.fallback_used) + self.assertTrue(result.trace.fallback_used) + self.assertIsNotNone(result.trace.fallback_reason) + self.assertIn("0.42", result.rationale) + + def test_llm_timeout_falls_back(self): + def slow_stub(system, user, *, model): + time.sleep(0.2) + return {"rationale": "too slow"} + + started = time.monotonic() + result = generate_link_rationale( + SECTION_TEXT, + CRE_ID, + CRE_TEXT, + 0.6, + llm_rationale_fn=slow_stub, + timeout_seconds=0.01, + ) + elapsed = time.monotonic() - started + self.assertLess(elapsed, 0.15) # well under the 0.2s stub delay + self.assertTrue(result.fallback_used) + + def test_malformed_json_falls_back(self): + def bad_stub(system, user, *, model): + return {"not_rationale_key": "x"} + + result = generate_link_rationale( + SECTION_TEXT, CRE_ID, CRE_TEXT, 0.6, llm_rationale_fn=bad_stub + ) + self.assertTrue(result.fallback_used) + + def test_empty_rationale_falls_back(self): + def empty_stub(system, user, *, model): + return {"rationale": ""} + + result = generate_link_rationale( + SECTION_TEXT, CRE_ID, CRE_TEXT, 0.6, llm_rationale_fn=empty_stub + ) + self.assertTrue(result.fallback_used) + + def test_missing_cre_text_still_produces_a_rationale(self): + # cre_text may be empty (RFC's original CandidateCRE allowed this); + # the prompt degrades gracefully rather than crashing. + def stub(system, user, *, model): + self.assertIn("", user) + return {"rationale": "Plausible match based on cheat sheet alone."} + + result = generate_link_rationale( + SECTION_TEXT, CRE_ID, "", 0.75, llm_rationale_fn=stub + ) + self.assertFalse(result.fallback_used) + self.assertEqual(result.confidence, "medium") + + +class RationaleGraphIntegrationTest(unittest.TestCase): + """End-to-end execution of the compiled LangGraph flow.""" + + def test_graph_runs_success_path(self): + app = build_rationale_graph() + + def stub(system, user, *, model): + return {"rationale": "match"} + + state = app.invoke( + { + "section_text": SECTION_TEXT, + "cre_id": CRE_ID, + "cre_text": CRE_TEXT, + "score": 0.88, + "llm_rationale_fn": stub, + "model_name": "test-model", + "timeout_seconds": 5.0, + "generated_at": "2026-08-13T00:00:00+00:00", + "fallback_used": False, + "fallback_reason": None, + } + ) + self.assertEqual(state["result"].confidence, "high") + self.assertFalse(state["result"].fallback_used) + + def test_graph_runs_fallback_path(self): + app = build_rationale_graph() + + def failing_stub(system, user, *, model): + raise RuntimeError("boom") + + state = app.invoke( + { + "section_text": SECTION_TEXT, + "cre_id": CRE_ID, + "cre_text": CRE_TEXT, + "score": 0.3, + "llm_rationale_fn": failing_stub, + "model_name": "test-model", + "timeout_seconds": 5.0, + "generated_at": "2026-08-13T00:00:00+00:00", + "fallback_used": False, + "fallback_reason": None, + } + ) + self.assertTrue(state["result"].fallback_used) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/utils/external_project_parsers/parsers/cheatsheet_rerank.py b/application/utils/external_project_parsers/parsers/cheatsheet_rerank.py new file mode 100644 index 000000000..2c70a88c2 --- /dev/null +++ b/application/utils/external_project_parsers/parsers/cheatsheet_rerank.py @@ -0,0 +1,498 @@ +""" +RFC Workstream E — LLM rationale generation for cheat-sheet CRE links. + +See: docs/rfc/cheatsheets-llm-autonomous-mapping-rfc.md, section 5 +("Workstream E: LLM Re-Rank and Decision Graph (LangGraph)"). + +## Why this module looks different from the original Workstream E scope + +The RFC originally scoped Workstream E as "LLM re-rank the top-k candidates +from Workstream D." Since then, retrieval *and* reranking for the mapping +pipeline have consolidated onto Module C ("The Librarian", +``application/utils/librarian/``): C.1 retrieves candidates, C.2 reranks them +with a cross-encoder, C.3 calibrates confidence, and C.4 decides + emits an +RFC ``LinkProposal`` or ``ReviewItem`` (see PR #944's closing comment and +PR #991). Building a second, LLM-based reranker on top of that would compete +with C.2 rather than add anything. + +What Module C's merged code does *not* do, and structurally cannot do with a +cross-encoder, is explain its pick in prose. Concretely: + +* ``schemas.ProposedLink.rationale: Optional[str]`` is a real field in the + RFC wire contract. +* ``emitter.py``'s ``_proposed_links()`` sets ``rationale=None`` on every + single link, unconditionally -- there is no code path anywhere in Module C + that populates it. +* ``decision_engine.decide()`` only ever surfaces a single top-1 + ``cre_id`` per chunk (``candidate_cre_ids[:1]``), so the scope of "explain + the pick" is one candidate, not a shortlist. + +This module fills exactly that gap: given the section text and the one CRE +Module C already chose (id, text, and its calibrated score), it asks an LLM +for a short, grounded, one-sentence rationale -- the same LLM-reasoning +capability Workstream E was always meant to contribute, retargeted at the +one place in the pipeline that's actually missing it, instead of duplicating +C.2's scoring job. + +This is a discussion-first proposal, not a fait accompli: wiring +``generate_link_rationale`` into ``application/utils/librarian/emitter.py`` +is left to the Module C maintainers to decide on, since that module is an +actively developed, separately owned GSoC deliverable. This file stays +self-contained and does not import or modify anything under +``application/utils/librarian/``. + +## Design notes (mostly carried over from the original implementation) + +* The LLM call is dependency-injected as ``llm_rationale_fn`` -- the same + seam pattern used throughout this codebase (``ai_client`` in + ``embed_alignment.py``, ``score_fn`` in ``librarian/cross_encoder.py``). + Production defaults to a lazily-imported LiteLLM call; tests inject a + deterministic stub, keeping the whole flow hermetically testable. +* A small LangGraph flow (generate -> format | generate -> fallback -> + format) still backs the public entrypoint, per the RFC's "Decision Graph + (LangGraph)" framing -- now scoped to one node's worth of real work + (generate a rationale) plus its fallback, rather than a multi-stage + rerank pipeline that would have duplicated C.2/C.3/C.4. +* ``langgraph`` is a **dev/CI-only dependency**: it lives in + ``requirements-dev.txt``, not ``requirements.txt``, and is imported + lazily inside :func:`build_rationale_graph` -- importing this module, or + calling :func:`classify_confidence`, never touches ``langgraph`` at all. + This mirrors the existing ``sentence-transformers`` entry in + ``requirements-dev.txt`` ("never install on Heroku") for exactly the + same reason: this module has no CLI/``cre.py`` wiring yet (grep the repo + for ``cheatsheet_rerank`` -- the only importer is its own test file), so + it must not grow the production web slug. If/when this is wired into a + live import path, that's the point to revisit whether ``langgraph`` + belongs in ``requirements.txt`` proper. +* Never raises on LLM failure -- falls back to a short, honest, templated + rationale ("Retrieval/rerank score S; LLM explanation unavailable.") so a + cheat sheet's link is never blocked or degraded by an LLM hiccup. +* ``classify_confidence`` is kept as a small, independently useful utility + for human-facing review UIs (Module C's own ``decide()`` thresholds + numerically and has no notion of confidence *bands*), but is no longer on + the path that decides whether something links or gets reviewed -- that + call is Module C's alone. +""" + +from __future__ import annotations + +import json +import logging +import math +import os +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError +from dataclasses import dataclass +from functools import partial +from datetime import datetime, timezone +from typing import Any, Callable, Dict, Optional, TypedDict + +from pydantic import BaseModel, Field, ValidationError + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Confidence thresholds (RFC section 11 bootstrap defaults; recalibrate via env) +# --------------------------------------------------------------------------- +_HIGH_THRESHOLD_ENV_VAR = "CRE_CHEATSHEET_RERANK_HIGH_THRESHOLD" +_MEDIUM_THRESHOLD_ENV_VAR = "CRE_CHEATSHEET_RERANK_MEDIUM_THRESHOLD" +_DEFAULT_HIGH_THRESHOLD = "0.85" +_DEFAULT_MEDIUM_THRESHOLD = "0.70" + +# Identifiers persisted into RationaleTrace for the RFC audit trail (mirrors +# RETRIEVER_NAME / RERANKER_NAME conventions in application/utils/librarian/). +RATIONALE_GENERATOR_NAME = "llm-cheatsheet-rationale-generator" +PROMPT_VERSION = "v2" + +DEFAULT_TIMEOUT_SECONDS = 30.0 +DEFAULT_MODEL_ENV_VAR = "CRE_CHEATSHEET_RERANK_MODEL" +DEFAULT_MODEL_FALLBACK = "gemini/gemini-2.5-flash" + +REASON_MAX_LENGTH = 400 + + +class RerankError(ValueError): + """Base class for this module's construction/usage failures.""" + + +def _resolve_threshold(env_var: str, default: str) -> float: + raw = os.environ.get(env_var, default) + try: + value = float(raw) + except (TypeError, ValueError) as exc: + raise RerankError(f"{env_var}={raw!r} is not a valid float") from exc + if not (0.0 <= value <= 1.0): + raise RerankError(f"{env_var}={value!r} must be in [0, 1]") + return value + + +def classify_confidence(score: float) -> str: + """ + Map a 0-1 score to a confidence band ("high" | "medium" | "low"). + + Independent utility for human-facing review UIs -- Module C's own + ``decide()`` thresholds numerically and has no notion of bands; this + does not feed back into that decision. Thresholds are the RFC's + bootstrap defaults (high >= 0.85, medium >= 0.70), recalibratable via + ``CRE_CHEATSHEET_RERANK_HIGH_THRESHOLD`` / + ``CRE_CHEATSHEET_RERANK_MEDIUM_THRESHOLD``, re-read on every call. + """ + if not isinstance(score, (int, float)) or isinstance(score, bool): + raise RerankError(f"score must be a number, got {score!r}") + if not (0.0 <= float(score) <= 1.0): + raise RerankError(f"score must be in [0, 1], got {score!r}") + + high = _resolve_threshold(_HIGH_THRESHOLD_ENV_VAR, _DEFAULT_HIGH_THRESHOLD) + medium = _resolve_threshold(_MEDIUM_THRESHOLD_ENV_VAR, _DEFAULT_MEDIUM_THRESHOLD) + if high < medium: + raise RerankError( + f"{_HIGH_THRESHOLD_ENV_VAR}={high!r} must be >= " + f"{_MEDIUM_THRESHOLD_ENV_VAR}={medium!r}" + ) + + if score >= high: + return "high" + if score >= medium: + return "medium" + return "low" + + +# --------------------------------------------------------------------------- +# Data contracts +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class RationaleTrace: + """Audit metadata captured for every rationale-generation run.""" + + model: str + prompt_version: str + generated_at: str + fallback_used: bool + fallback_reason: Optional[str] = None + + +@dataclass(frozen=True) +class LinkRationale: + """ + One CRE link's generated rationale -- meant to fill + ``librarian.schemas.ProposedLink.rationale``, which every current + Module C code path leaves ``None``. + """ + + cre_id: str + rationale: str + confidence: str + fallback_used: bool + trace: RationaleTrace + + +# --------------------------------------------------------------------------- +# LLM structured-output schema +# --------------------------------------------------------------------------- + + +class _RationalePayload(BaseModel): + rationale: str = Field(min_length=1, max_length=REASON_MAX_LENGTH) + + +def rationale_response_json_schema() -> Dict[str, Any]: + """Provider-friendly JSON schema for strict structured LLM outputs.""" + return _RationalePayload.model_json_schema() + + +# --------------------------------------------------------------------------- +# Prompting +# --------------------------------------------------------------------------- + + +def _system_prompt() -> str: + return ( + "You explain why an OWASP cheat sheet is a good match for a specific " + "Common Requirement (CRE) entry. You will be given the cheat sheet's " + "text, the CRE's id and text, and Module C's calibrated match score. " + "Write ONE short, concrete sentence (max 60 words) grounded in the " + "actual cheat sheet content and CRE text -- do not restate the score, " + "do not invent facts not present in either text. " + 'Return ONLY valid JSON of the form {"rationale": "..."}.' + ) + + +def _user_payload(section_text: str, cre_id: str, cre_text: str, score: float) -> str: + lines = [ + f"CHEATSHEET_TEXT: {section_text[:2000]}", + "", + f"CRE_ID: {cre_id}", + f"CRE_TEXT: {(cre_text or '')[:800]}", + "", + f"CALIBRATED_SCORE: {score:.3f}", + ] + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Default (production) LLM call -- lazy litellm import so this module stays +# import-light and hermetically testable without a real LLM dependency. +# --------------------------------------------------------------------------- + + +def _default_model_name() -> str: + return os.environ.get( + DEFAULT_MODEL_ENV_VAR, + os.environ.get("CRE_LLM_CHAT_MODEL", DEFAULT_MODEL_FALLBACK), + ) + + +def default_llm_rationale_fn( + system: str, user: str, *, model: str, timeout: Optional[float] = None +) -> Dict[str, Any]: + """Production LLM call via LiteLLM. Raises on any failure; callers must + handle fallback (this function intentionally does not swallow errors). + + ``timeout``, when given, is passed straight through to LiteLLM so the + underlying HTTP request itself is bounded, letting the worker thread in + ``_call_with_timeout`` actually terminate rather than just being + abandoned. + """ + try: + import litellm # type: ignore + except ImportError as exc: # pragma: no cover - exercised only without litellm + raise RerankError( + "litellm package is required for LLM rationale calls" + ) from exc + + resp = litellm.completion( + model=model, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + response_format={"type": "json_object"}, + temperature=0.2, + timeout=timeout, + ) + choices = getattr(resp, "choices", None) + if not choices: + raise RerankError("LLM response contained no choices") + content = choices[0].message.content + if isinstance(content, list): # some providers return content blocks + content = "".join( + b.get("text", "") if isinstance(b, dict) else str(b) for b in content + ) + return json.loads(content) + + +def _call_with_timeout( + fn: Callable[[], Dict[str, Any]], timeout_seconds: float +) -> Dict[str, Any]: + """Run ``fn`` with a hard wall-clock timeout so a hung LLM call can never + block the pipeline; raises on timeout or on any exception from ``fn``. + + Uses an explicit (non-context-manager) executor so a timeout returns to + the caller immediately instead of blocking on ``shutdown(wait=True)`` + for a thread that is still running the (now-abandoned) call. + """ + pool = ThreadPoolExecutor(max_workers=1) + future = pool.submit(fn) + try: + return future.result(timeout=timeout_seconds) + except FutureTimeoutError as exc: + pool.shutdown(wait=False) + raise RerankError( + f"LLM rationale call exceeded {timeout_seconds}s timeout" + ) from exc + except Exception: + pool.shutdown(wait=False) + raise + else: + pool.shutdown(wait=False) + + +# --------------------------------------------------------------------------- +# LangGraph flow: generate -> (success: format) | (failure: fallback -> format) +# --------------------------------------------------------------------------- + + +class _RationaleState(TypedDict, total=False): + section_text: str + cre_id: str + cre_text: str + score: float + llm_rationale_fn: Callable[..., Dict[str, Any]] + model_name: str + timeout_seconds: float + generated_at: str + rationale_text: Optional[str] + fallback_used: bool + fallback_reason: Optional[str] + result: LinkRationale + + +def _node_generate(state: _RationaleState) -> _RationaleState: + """Call the LLM and validate its output. On any failure, record the + reason and leave ``rationale_text`` unset; the conditional edge below + routes to the fallback node instead of raising.""" + system = _system_prompt() + user = _user_payload( + state["section_text"], state["cre_id"], state["cre_text"], state["score"] + ) + llm_rationale_fn = state["llm_rationale_fn"] + model_name = state["model_name"] + timeout_seconds = state["timeout_seconds"] + + try: + raw = _call_with_timeout( + lambda: llm_rationale_fn(system, user, model=model_name), timeout_seconds + ) + payload = _RationalePayload.model_validate(raw) + except (RerankError, ValidationError, json.JSONDecodeError, TypeError) as exc: + logger.warning("LLM rationale failed for %s: %s", state["cre_id"], exc) + state["fallback_reason"] = f"{type(exc).__name__}: {exc}"[:REASON_MAX_LENGTH] + state["rationale_text"] = None + return state + except Exception as exc: # defensive: never let an unexpected error crash the run + logger.warning( + "LLM rationale failed unexpectedly for %s: %s", state["cre_id"], exc + ) + state["fallback_reason"] = f"unexpected:{type(exc).__name__}: {exc}"[ + :REASON_MAX_LENGTH + ] + state["rationale_text"] = None + return state + + state["rationale_text"] = payload.rationale[:REASON_MAX_LENGTH] + return state + + +def _route_after_generate(state: _RationaleState) -> str: + return "format" if state.get("rationale_text") else "fallback" + + +def _node_fallback(state: _RationaleState) -> _RationaleState: + """Deterministic, honest fallback: no LLM prose, just the score.""" + state["fallback_used"] = True + state["rationale_text"] = ( + f"Retrieval/rerank score {state['score']:.2f}; LLM explanation unavailable." + ) + return state + + +def _node_format(state: _RationaleState) -> _RationaleState: + rationale_text = state.get("rationale_text") + if not rationale_text: + # Defensive: the router only reaches `format` after `generate` or + # `fallback` has set this, but keep the contract total. + rationale_text = ( + f"Retrieval/rerank score {state['score']:.2f}; " + "LLM explanation unavailable." + ) + state["fallback_used"] = True + trace = RationaleTrace( + model=state["model_name"], + prompt_version=PROMPT_VERSION, + generated_at=state["generated_at"], + fallback_used=state.get("fallback_used", False), + fallback_reason=( + state.get("fallback_reason") if state.get("fallback_used") else None + ), + ) + state["result"] = LinkRationale( + cre_id=state["cre_id"], + rationale=rationale_text, + confidence=classify_confidence(state["score"]), + fallback_used=state.get("fallback_used", False), + trace=trace, + ) + return state + + +def build_rationale_graph(): + """Compile and return the Workstream E LangGraph flow. + + Nodes: ``generate`` -> (``format`` | ``fallback`` -> ``format``) -> END. + """ + from langgraph.graph import StateGraph, START, END + + graph = StateGraph(_RationaleState) + graph.add_node("generate", _node_generate) + graph.add_node("fallback", _node_fallback) + graph.add_node("format", _node_format) + + graph.add_edge(START, "generate") + graph.add_conditional_edges( + "generate", _route_after_generate, {"format": "format", "fallback": "fallback"} + ) + graph.add_edge("fallback", "format") + graph.add_edge("format", END) + + return graph.compile() + + +# --------------------------------------------------------------------------- +# Public entrypoint +# --------------------------------------------------------------------------- + + +def generate_link_rationale( + section_text: str, + cre_id: str, + cre_text: str, + score: float, + *, + llm_rationale_fn: Optional[Callable[..., Dict[str, Any]]] = None, + model_name: Optional[str] = None, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, +) -> LinkRationale: + """ + Generate a natural-language rationale for one CRE link, via the + LangGraph flow above. + + Intended to fill ``librarian.schemas.ProposedLink.rationale`` for the + single top-1 candidate Module C's ``decide()`` already chose -- this + does not rerank or re-decide anything. + + ``llm_rationale_fn`` defaults to a LiteLLM-backed call + (:func:`default_llm_rationale_fn`); tests and harnesses should inject a + deterministic stub instead. Never raises on LLM failure -- falls back to + a short, honest, score-only rationale and marks the trace accordingly. + """ + if not isinstance(cre_id, str) or not cre_id.strip(): + raise RerankError(f"cre_id must be a non-empty string, got {cre_id!r}") + if not isinstance(timeout_seconds, (int, float)) or isinstance( + timeout_seconds, bool + ): + raise RerankError( + f"timeout_seconds must be a non-boolean number, got {timeout_seconds!r}" + ) + if not math.isfinite(timeout_seconds) or timeout_seconds <= 0: + raise RerankError( + f"timeout_seconds must be a finite number > 0, got {timeout_seconds!r}" + ) + # classify_confidence performs the score range/type validation; reuse it + # up front so a bad score fails fast, before any LLM call is attempted. + classify_confidence(score) + + resolved_model = model_name or _default_model_name() + if llm_rationale_fn is not None: + score_fn = llm_rationale_fn + else: + # Bind the request-level timeout only for the built-in LiteLLM path; + # injected stubs are not required to accept a ``timeout`` kwarg. + score_fn = partial(default_llm_rationale_fn, timeout=timeout_seconds) + + app = build_rationale_graph() + result = app.invoke( + { + "section_text": section_text, + "cre_id": cre_id, + "cre_text": cre_text, + "score": float(score), + "llm_rationale_fn": score_fn, + "model_name": resolved_model, + "timeout_seconds": timeout_seconds, + "generated_at": datetime.now(timezone.utc).isoformat(), + "fallback_used": False, + "fallback_reason": None, + } + ) + return result["result"] diff --git a/docs/rfc-llm-rerank.md b/docs/rfc-llm-rerank.md new file mode 100644 index 000000000..d0a8bdcbb --- /dev/null +++ b/docs/rfc-llm-rerank.md @@ -0,0 +1,83 @@ +# RFC Workstream E — LLM rationale generation for cheat-sheet CRE links + +**Status update:** this module was originally built as an LLM re-rank step +(Workstream D candidates in, ranked/explained shortlist out). Since then, +candidate retrieval *and* reranking for the whole mapping pipeline have +consolidated onto Module C ("The Librarian") — see PR #944's closing comment +and the merged PR #991. This doc describes the module as it stands now, +retargeted at a gap in Module C rather than duplicating its C.2 reranker. +Wiring it into `application/utils/librarian/emitter.py` is left to the +Module C maintainers to decide on; see the discussion on PR #1014. + +The implementation is located in: + +* `application/utils/external_project_parsers/parsers/cheatsheet_rerank.py` + +## Dependency footprint + +`langgraph` lives in `requirements-dev.txt`, not `requirements.txt` — it is +a dev/CI-only dependency, lazily imported inside `build_rationale_graph()`. +Importing this module, or calling `classify_confidence()`, never touches +`langgraph`. This matches the existing `sentence-transformers` entry in +`requirements-dev.txt` (Module C's own ML dependency, annotated "never +install on Heroku"), for the same reason: this module has no CLI/`cre.py` +wiring yet, so it must not grow the production web slug. Revisit this once +it's actually wired into a live import path. + +## Sources for more context + +* RFC: `docs/rfc/cheatsheets-llm-autonomous-mapping-rfc.md` +* Module C: `application/utils/librarian/` (`schemas.py`, `decision_engine.py`, + `cross_encoder.py`, `emitter.py`) +* PR #944 (Workstream D, closed in favor of Module C): + https://github.com/OWASP/OpenCRE/pull/944 +* PR #991 (Module C week 6b — emitter + pipeline glue): + https://github.com/OWASP/OpenCRE/pull/991 + +## The gap this fills + +Module C's merged code has no code path that populates +`schemas.ProposedLink.rationale` — `emitter.py`'s `_proposed_links()` sets +`rationale=None` on every link, always. A cross-encoder (C.2) produces a +similarity score, not prose, so nothing downstream of it can fill that field +without an LLM. `decision_engine.decide()` also only ever surfaces a single +top-1 `cre_id` per chunk, so the scope of "explain the pick" is one +candidate, not a shortlist — this module is scoped accordingly. + +## What it implements + +`generate_link_rationale(section_text, cre_id, cre_text, score, ...) -> LinkRationale` +runs a small LangGraph flow: + +```text +START -> generate --(success)--> format -> END + \--(failure)--> fallback -> format -> END +``` + +* **`generate`** — asks an LLM for one short, grounded sentence explaining + why `cre_text` matches `section_text`, given Module C's calibrated + `score`. Validated against a strict Pydantic schema (non-empty, capped + length). +* **`fallback`** — runs on any LLM error, timeout (hard wall-clock cutoff, + default 30s), or malformed/empty output. Produces a short, honest, + score-only rationale ("Retrieval/rerank score 0.42; LLM explanation + unavailable.") so a link is never blocked by an LLM hiccup. +* **`format`** — attaches confidence band (`classify_confidence`, RFC + bootstrap thresholds 0.85/0.70, env-overridable) and an audit + `RationaleTrace` (model, prompt version, UTC timestamp, fallback + flag/reason). + +The LLM call is dependency-injected (`llm_rationale_fn`), same seam pattern +as `embed_alignment.py`'s `ai_client` and `librarian/cross_encoder.py`'s +`score_fn`. Production defaults to a lazily-imported LiteLLM call; the full +test suite runs with a stub, no network/API key required. + +## What this module deliberately does not do + +* It does not retrieve or rerank candidates — Module C's C.1/C.2 own that. +* It does not decide auto-link vs. review — Module C's C.4 `decide()` owns + that; `classify_confidence` here is a separate, human-facing utility only. +* It does not modify `application/utils/librarian/` — that's an actively + developed, separately owned module. Wiring this in as the source of + `ProposedLink.rationale` is proposed, not applied, pending discussion with + the Module C maintainers. diff --git a/requirements-dev.txt b/requirements-dev.txt index 4b8784bdc..992664d08 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,6 +6,13 @@ sentence-transformers>=5.0.0,<6.0.0 setuptools +# Workstream E cheat-sheet LLM rationale generation (application/utils/ +# external_project_parsers/parsers/cheatsheet_rerank.py). Not on the +# production import path (no CLI/cre.py wiring yet) and lazily imported +# inside the function that needs it, so it's dev/CI-only, same as +# sentence-transformers above -- never install on Heroku. +langgraph>=1.0.10,<2 + # importer / embeddings scrape tooling (not needed for prod chat) playwright nltk