From 5ade9b8d8bc5e8138c90d5b55ce85327f593d379 Mon Sep 17 00:00:00 2001
From: Sonny May <sonnymaywi@gmail.com>
Date: Sat, 7 Nov 2026 15:07:52 +0000
Subject: [PATCH] fix: handle schema-violating model output in AI suggestions

If the model returned valid JSON that wasn't an object with a list under
"matches" (e.g. {"matches": null} or a bare list), suggest_for_ticket
raised and /ai/suggest returned a 500. It now returns an empty match list
with an error, like the existing non-JSON case.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0171zWzjpbovQSMUdE4PmH6R
---
 backend/ai.py               |  9 ++++++--
 backend/test_ai_response.py | 46 +++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 2 deletions(-)
 create mode 100644 backend/test_ai_response.py

diff --git a/backend/ai.py b/backend/ai.py
index d16befa..8046484 100644
--- a/backend/ai.py
+++ b/backend/ai.py
@@ -150,5 +150,10 @@ def suggest_for_ticket(ticket_id: str) -> dict[str, Any]:
     except json.JSONDecodeError:
         return {"matches": [], "raw": raw, "error": "Model returned non-JSON output"}
 
-    matches = parsed.get("matches", [])[:3]
-    return {"matches": matches}
+    # The model may return valid JSON that doesn't follow the schema (a list, or
+    # "matches": null). Treat that like non-JSON output instead of crashing with a 500.
+    matches = parsed.get("matches") if isinstance(parsed, dict) else None
+    if not isinstance(matches, list):
+        return {"matches": [], "raw": raw, "error": "Model returned an unexpected response shape"}
+
+    return {"matches": matches[:3]}
diff --git a/backend/test_ai_response.py b/backend/test_ai_response.py
new file mode 100644
index 0000000..4660566
--- /dev/null
+++ b/backend/test_ai_response.py
@@ -0,0 +1,46 @@
+"""Tests for how ai.suggest_for_ticket handles unexpected model output.
+
+test_main.py replaces ``sys.modules["ai"]`` with a stub, so load the real module
+directly from its file.
+"""
+
+import importlib.util
+import json
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+
+@pytest.fixture
+def real_ai(monkeypatch):
+    monkeypatch.setenv("SUPABASE_URL", "http://fake.supabase.co")
+    monkeypatch.setenv("SUPABASE_KEY", "fake-key")
+    spec = importlib.util.spec_from_file_location("ai_real", Path(__file__).with_name("ai.py"))
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    monkeypatch.setattr(module, "_fetch_ticket", lambda _id: {"id": "t1", "title": "x"})
+    monkeypatch.setattr(module, "_fetch_notes", lambda _id: [])
+    monkeypatch.setattr(module, "_fetch_recent_resolved", lambda limit=30: [{"id": "p1"}])
+    return module
+
+
+def _model_says(module, monkeypatch, text):
+    block = MagicMock(type="text", text=text)
+    client = MagicMock()
+    client.messages.create.return_value = MagicMock(content=[block])
+    monkeypatch.setattr(module, "_get_client", lambda: client)
+
+
+@pytest.mark.parametrize("payload", ['{"matches": null}', '{"matches": "none"}', "[1, 2]", "{}"])
+def test_unexpected_shape_returns_error_not_exception(real_ai, monkeypatch, payload):
+    _model_says(real_ai, monkeypatch, payload)
+    result = real_ai.suggest_for_ticket("t1")
+    assert result["matches"] == []
+    assert result["error"] == "Model returned an unexpected response shape"
+
+
+def test_valid_matches_are_capped_at_three(real_ai, monkeypatch):
+    matches = [{"ticket_id": str(i)} for i in range(5)]
+    _model_says(real_ai, monkeypatch, json.dumps({"matches": matches}))
+    assert real_ai.suggest_for_ticket("t1") == {"matches": matches[:3]}