Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion backend/app/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ def runs():
improvement
for response in responses
for improvement in response.get("improvements", [])
if isinstance(improvement, dict)
if isinstance(improvement, dict) and not str(improvement.get("known_issue_id") or "").strip()
]
issues = [
issue
Expand Down
24 changes: 19 additions & 5 deletions backend/app/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -2429,7 +2429,7 @@ def _known_unresolved_issues(self, run: Run, *, limit: int = 12) -> list[dict[st
candidates.append(run)
values: list[dict[str, str]] = []
for source_run in candidates:
for result in source_run.supervisor_results:
for record_index, result in enumerate(source_run.supervisor_results):
if (
not isinstance(result, dict)
or result.get("stage", "issue_assessment") != "issue_assessment"
Expand All @@ -2446,9 +2446,11 @@ def _known_unresolved_issues(self, run: Run, *, limit: int = 12) -> list[dict[st
for index, proposal in enumerate(proposals):
if not isinstance(proposal, dict):
continue
proposal_id = f"{source_run.id}:{result.get('iteration', 0)}:{index}"
proposal_id = f"{source_run.id}:{result.get('iteration', 0)}:{record_index}:{index}"
if proposals is reported:
proposal_id = f"{source_run.id}:{result.get('iteration', 0)}:issue:{index}"
proposal_id = (
f"{source_run.id}:{result.get('iteration', 0)}:{record_index}:issue:{index}"
)
management = records.get(proposal_id, {})
if management.get("status") in {"resolved", "deleted"}:
continue
Expand Down Expand Up @@ -3395,6 +3397,11 @@ def parse_timestamp(value: object) -> datetime | None:
improvements = (
response.get("improvements", []) if isinstance(response.get("improvements"), list) else []
)
improvements = [
item
for item in improvements
if isinstance(item, dict) and not str(item.get("known_issue_id") or "").strip()
]
issues = (
response.get("reported_issues", [])
if isinstance(response.get("reported_issues"), list)
Expand Down Expand Up @@ -3575,7 +3582,7 @@ def active_evaluations(self, runs: list[Run] | None = None) -> list[dict[str, An
improvement
for response in responses
for improvement in response.get("improvements", [])
if isinstance(improvement, dict)
if isinstance(improvement, dict) and not str(improvement.get("known_issue_id") or "").strip()
]
issues = [
issue
Expand Down Expand Up @@ -4608,7 +4615,14 @@ def supervisor_result(result: object) -> object:
reported_at = now().isoformat()
for improvement in result["improvements"]:
improvement.setdefault("reported_at", reported_at)
improvement.setdefault("effect_score", (result.get("evaluation") or {}).get("score"))
item_evaluation = improvement.get("evaluation")
item_score = item_evaluation.get("score") if isinstance(item_evaluation, dict) else None
if improvement.get("effect_score") is None:
improvement["effect_score"] = (
item_score
if item_score is not None
else (result.get("evaluation") or {}).get("score")
)
improvement.setdefault(
"attempted", improvement.get("status") in {"adopted", "accepted", "rejected"}
)
Expand Down
20 changes: 17 additions & 3 deletions backend/tests/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -1025,7 +1025,13 @@ def test_active_evaluations_count_feedback_across_all_iterations(monkeypatch):
{
"iteration": 1,
"response": {
"improvements": [{"title": "Add refund intake", "status": "accepted"}],
"improvements": [
{"title": "Add refund intake", "status": "accepted"},
{
"known_issue_id": "feedback-history-run:0:0:0",
"evidence": "The previously reported issue remains visible.",
},
],
"reported_issues": [{"title": "Missing refund details"}],
},
},
Expand Down Expand Up @@ -1738,7 +1744,7 @@ def complete(self, _settings, prompt):
{
"improvements": [
{
"known_issue_id": "known-issue-run:1:0",
"known_issue_id": "known-issue-run:1:0:0",
"evidence": "The basis is still absent in this iteration.",
"evaluation": {"score": 7, "approval": "pending", "summary": "Known issue."},
}
Expand Down Expand Up @@ -1769,10 +1775,12 @@ def complete(self, _settings, prompt):
store._complete_supervision(run.id)

assert "# Known unresolved issues" in captured_prompts[0]
assert "known-issue-run:1:0" in captured_prompts[0]
assert "known-issue-run:1:0:0" in captured_prompts[0]
assert [item["title"] for item in store.proposal_lifecycles()] == [
"Portfolio calculation basis is unclear"
]
retained_evidence = store._load(run.id).supervisor_results[-1]["response"]["improvements"][0]
assert retained_evidence["effect_score"] == 7


def test_runner_context_uses_the_supplied_model_profile_without_exposing_its_secret(tmp_path, monkeypatch):
Expand Down Expand Up @@ -2668,6 +2676,12 @@ def test_issue_management_separates_multiple_supervisor_records_in_one_iteration
by_title = {item["title"]: item for item in store.issue_management_items()}
assert by_title["First issue"]["comments"][0]["body"] == "Only the first issue"
assert by_title["Second issue"]["comments"] == []
assert {issue["id"] for issue in store._known_unresolved_issues(store.runs()[0])} == {
"run-duplicate-assessments:1:0:0",
"run-duplicate-assessments:1:1:0",
"run-duplicate-assessments:2:2:issue:0",
"run-duplicate-assessments:2:3:issue:0",
}


def test_hello_accepts_unsaved_profile_settings():
Expand Down
9 changes: 7 additions & 2 deletions frontend/src/features/evaluations/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -658,6 +658,7 @@
issueStatuses = new Set<string>();
for (const record of selected?.supervisor_results ?? []) {
for (const improvement of record.response?.improvements ?? []) {
if (String(improvement.known_issue_id ?? "").trim()) continue;
if (improvement.status) improvementStatuses.add(String(improvement.status));
}
for (const issue of record.response?.reported_issues ?? []) {
Expand All @@ -681,7 +682,9 @@
recordEvaluation = response?.evaluation,
decision = recordEvaluation?.approval ?? "no_response",
score = recordEvaluation?.score,
improvements = response?.improvements ?? [],
improvements = (response?.improvements ?? []).filter(
(improvement) => !String(improvement.known_issue_id ?? "").trim(),
),
issues = response?.reported_issues ?? [];
const inRange =
(!resultIterationFrom || record.iteration >= Number(resultIterationFrom)) &&
Expand Down Expand Up @@ -729,7 +732,7 @@
(resultIterationFilter !== "latest" || record.iteration === latestIteration)
);
});
}, [

Check warning on line 735 in frontend/src/features/evaluations/page.tsx

View workflow job for this annotation

GitHub Actions / frontend

React Hook useMemo has a missing dependency: 'translateResultResponse'. Either include it or remove the dependency array
selected,
candidateTab,
resultContent,
Expand Down Expand Up @@ -760,7 +763,9 @@
resultTranslationIds.every((templateId) => Boolean(resultTranslations.content(templateId)));
const resultTranslationsLoading = resultTranslations.isLoading(resultTranslationIds);
const resultImprovements = resultRecords.flatMap((record) =>
(translateResultResponse(record)?.improvements ?? []).map((item) => ({
(translateResultResponse(record)?.improvements ?? [])
.filter((item) => !String(item.known_issue_id ?? "").trim())
.map((item) => ({
...item,
__iteration: record.iteration,
})),
Expand Down
7 changes: 5 additions & 2 deletions frontend/src/features/evaluations/run-detail-result-panel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,11 @@ function browserJourney(result?: Record<string, unknown>) {

function ResultList({ items, kind, locale, empty, l }: { items: RecordItem[]; kind: "improvement" | "issue"; locale: Locale; empty: string; l: Messages }) {
return <div className="result-items result-items--scrollable">{items.length ? items.map((item, index) => {
const status = String(item.status ?? "—"), severity = String(item.severity ?? "—"), reportedAt = typeof item.reported_at === "string" ? item.reported_at : undefined;
return <article className="result-row" key={index}><time className="result-row__time">{time(locale, reportedAt)}</time><div className="result-row__body"><strong>{String(item.title ?? "—")}</strong><p>{String(kind === "improvement" ? (item.rationale ?? "—") : (item.evidence ?? "—"))}</p></div><div className="result-row__metrics"><span><small>{l.resultIteration}</small><b>#{String(item.__iteration ?? "—")}</b></span>{kind === "improvement" ? <><span><small>{l.resultStatus}</small><b className={`decision decision--${status}`}>{status}</b></span><span><small>{l.resultScore}</small><b>{String(item.effect_score ?? item.score ?? "—")}</b></span><span><small>{l.resultAttempted}</small><b>{item.attempted === true || ["adopted", "accepted"].includes(status) ? l.resultYes : l.resultNo}</b></span></> : <><span><small>{l.resultSeverity}</small><b className={`decision decision--${severity}`}>{severity}</b></span><span><small>{l.resultStatus}</small><b className={`decision decision--${status}`}>{status}</b></span></>}</div></article>;
const status = String(item.status ?? "—"), severity = String(item.severity ?? "—"), reportedAt = typeof item.reported_at === "string" ? item.reported_at : undefined,
evaluation = item.evaluation,
evaluationScore = evaluation && typeof evaluation === "object" ? (evaluation as RecordItem).score : undefined,
score = item.effect_score ?? item.score ?? evaluationScore ?? "—";
return <article className="result-row" key={index}><time className="result-row__time">{time(locale, reportedAt)}</time><div className="result-row__body"><strong>{String(item.title ?? "—")}</strong><p>{String(kind === "improvement" ? (item.rationale ?? "—") : (item.evidence ?? "—"))}</p></div><div className="result-row__metrics"><span><small>{l.resultIteration}</small><b>#{String(item.__iteration ?? "—")}</b></span>{kind === "improvement" ? <><span><small>{l.resultStatus}</small><b className={`decision decision--${status}`}>{status}</b></span><span><small>{l.resultScore}</small><b>{String(score)}</b></span><span><small>{l.resultAttempted}</small><b>{item.attempted === true || ["adopted", "accepted"].includes(status) ? l.resultYes : l.resultNo}</b></span></> : <><span><small>{l.resultSeverity}</small><b className={`decision decision--${severity}`}>{severity}</b></span><span><small>{l.resultStatus}</small><b className={`decision decision--${status}`}>{status}</b></span></>}</div></article>;
}) : <p className="hint result-empty">{empty}</p>}</div>;
}

Expand Down
Loading