Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions .github/cursor-approve/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,12 @@ first sentence of `summary`, and `summary` is where a model writes its process
("Checked the only changed file…", "I read root AGENTS.md and CLAUDE.md…") — so
the card said what the axis *did* and never why it ruled as it did, which is
unreadable on a yellow. `headline` must state what decided the verdict, and a
missing or over-long one makes the axis untrusted rather than being truncated:
truncating would reproduce the reason-free row one step later.
missing one makes the axis untrusted. An over-long one is clamped to the cap at
a word boundary with "…", not rejected: the model's length is not
Comment thread
mattmillerai marked this conversation as resolved.
Resolved
deterministic, and rejecting turned a sound verdict's axis job red on length
alone. The opening words carry the reason, so the clamp keeps it — falling
back to a hard cut when the last fitting space is in the first half, and
rejecting one that leaves nothing but punctuation before the "…".

Placeholders — `{{pr_number}}`, `{{repo}}`, `{{head_sha}}`,
`{{merge_base_sha}}`, `{{base_ref}}`, `{{context_file}}` — are filled by
Expand All @@ -66,7 +70,7 @@ aggregate.py decide --outputs-dir out/ [--axes design,correctness] [--max-yellow
`{"event", "verdicts", "axes", "reasons"}`. In order:

1. any expected axis missing, unparsable, with a verdict outside
red/yellow/green/n-a, a confidence outside 0..1, or a missing or over-long
red/yellow/green/n-a, a confidence outside 0..1, or a missing or empty
`headline` → `NONE` — untrusted, so the approval is withheld; it is never a
veto;
2. any `red` → `NONE`, naming the red axes;
Expand Down
38 changes: 30 additions & 8 deletions .github/cursor-approve/aggregate.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@

1. any expected axis missing, unparsable, with a verdict outside
red/yellow/green/n/a, a confidence outside 0..1, a missing or
over-long ``headline``, or (with ``--commit-sha``)
empty ``headline`` (an over-long one is clamped to the cap, not
rejected), or (with ``--commit-sha``)
a stamped commit other than that one → ``NONE`` (the round is
untrusted: withhold the approval, never veto);
2. any ``red`` → ``NONE``, naming the red axes;
Expand Down Expand Up @@ -214,18 +215,37 @@ def validate_output(data):
# The headline is the ONLY part of this that reaches the PR, so it is
# required rather than best-effort: an axis that does not write one leaves
# the card with a verdict and no reason, which is the exact failure this
# field was added to fix. Truncating silently would do the same thing one
# step later, so an over-long one is rejected too.
# field was added to fix. An over-long one is clamped, not rejected: the
# model's word count is not deterministic, and failing a sound verdict
# because its reason ran a few words over turned the axis job red on
# length alone. The clamp keeps the opening words, which carry the reason.
headline = data.get("headline")
if not isinstance(headline, str) or not headline.strip():
raise ValueError("headline is missing or empty")
headline = " ".join(headline.split())
if len(headline) > HEADLINE_MAX:
raise ValueError(
f"headline is {len(headline)} characters, over the {HEADLINE_MAX} limit")
headline = clamp_headline(" ".join(headline.split()))
return verdict, confidence, summary, headline


def clamp_headline(headline: str) -> str:
"""`headline` cut to HEADLINE_MAX characters, at a word boundary, with "…".

Falls back to a hard cut when the last space that fits sits in the first
half (one long token, or "Fix: <long path>"), so the clamp never throws
away most of the reason. Raises ValueError when nothing but separators
survives: a bare "…" is the reason-free row the headline exists to prevent.
"""
if len(headline) <= HEADLINE_MAX:
return headline
hard = headline[:HEADLINE_MAX - 1]
space = headline.rfind(" ", 0, HEADLINE_MAX)
candidates = [headline[:space], hard] if space >= HEADLINE_MAX // 2 else [hard]
for cut in candidates:
cut = cut.rstrip(" ,;:-")
if cut.strip(" ,;:-"):
return cut + "…"
raise ValueError("headline is over the limit and holds no reason to keep")


def load_output(path: str, commit_sha: str = ""):
"""Parse one axis output file; raise ValueError on anything unusable.

Expand Down Expand Up @@ -394,7 +414,9 @@ def cmd_extract(args) -> int:
if len(data) > MAX_RAW_BYTES:
raise ValueError(f"the reply is larger than {MAX_RAW_BYTES} bytes")
obj = extract_object(data.decode("utf-8", errors="replace"))
validate_output(obj)
# The artifact carries the clamped headline, so what is uploaded is
# already within the cap rather than relying on decide to re-clamp it.
obj["headline"] = validate_output(obj)[3]
except (OSError, ValueError) as e:
print(f"::error::{e}", file=sys.stderr)
return 1
Expand Down
66 changes: 61 additions & 5 deletions .github/cursor-approve/tests/test_aggregate.py
Original file line number Diff line number Diff line change
Expand Up @@ -546,6 +546,19 @@ def test_the_headline_reaches_the_artifact(self):
with open(self.out, encoding="utf-8") as f:
self.assertEqual(json.load(f)["headline"], "Both branches handle an empty payload.")

def test_an_overlong_headline_is_clamped_in_the_artifact(self):
"""The axis job's "Extract the verdict" step used to exit 1 here on a
headline a few characters over the cap, failing the whole axis job."""
long = "The handler " + "reads the payload twice " * 5 + "on the retry path."
self.assertGreater(len(long), AG.HEADLINE_MAX)
code, _, err = self.extract(json.dumps(
{"verdict": "green", "confidence": 1, "headline": long, "summary": "ok"}))
self.assertEqual(code, 0, err)
with open(self.out, encoding="utf-8") as f:
headline = json.load(f)["headline"]
self.assertEqual(headline, AG.clamp_headline(long))
self.assertLessEqual(len(headline), AG.HEADLINE_MAX)

def test_rejects_a_bad_commit_sha(self):
code, _, err = self.extract('{"verdict": "green", "confidence": 1, "summary": "ok"}', sha="main")
self.assertEqual(code, 1)
Expand Down Expand Up @@ -604,13 +617,56 @@ def test_a_non_string_headline_is_untrusted(self):
self.write("business", self.raw(headline=["a"]))
self.assertEqual(self.decide()["event"], AG.NONE)

def test_an_overlong_headline_is_rejected_not_truncated(self):
# Truncating would reintroduce the reason-free row one step later.
def test_an_overlong_headline_is_clamped_not_rejected(self):
# A sound verdict must not go untrusted because its reason ran long:
# the model's length is not deterministic, so the cap is enforced here.
self.write_all()
self.write("business", self.raw(headline="x" * (AG.HEADLINE_MAX + 1)))
long = ("The new retry loop drops the context deadline on the second "
"attempt, so a cancelled request keeps retrying past its budget.")
self.assertGreater(len(long), AG.HEADLINE_MAX)
self.write("business", self.raw(headline=long))
result = self.decide()
self.assertEqual(result["event"], AG.NONE)
self.assertIn(f"over the {AG.HEADLINE_MAX} limit", result["reasons"][0])
self.assertEqual(result["event"], AG.APPROVE)
headline = result["axes"]["business"]["headline"]
self.assertLessEqual(len(headline), AG.HEADLINE_MAX)
self.assertTrue(headline.endswith("…"))
self.assertTrue(long.startswith(headline[:-1]))

def test_the_clamp_cuts_at_a_word_boundary(self):
headline = AG.clamp_headline("word " * 30)
self.assertLessEqual(len(headline), AG.HEADLINE_MAX)
self.assertEqual(headline, ("word " * 20).rstrip() + "…")
self.assertEqual(len(headline), AG.HEADLINE_MAX)

def test_the_clamp_hard_cuts_when_the_only_space_is_early(self):
# "Fix…" would keep almost none of the reason.
long = "Fix: " + "x" * 200
headline = AG.clamp_headline(long)
self.assertEqual(headline, long[:AG.HEADLINE_MAX - 1] + "…")

def test_the_clamp_hard_cuts_one_long_token(self):
headline = AG.clamp_headline("x" * (AG.HEADLINE_MAX * 2))
self.assertEqual(headline, "x" * (AG.HEADLINE_MAX - 1) + "…")

def test_the_clamp_drops_trailing_punctuation_before_the_ellipsis(self):
headline = AG.clamp_headline("a" * 90 + ", " + "b" * 20)
self.assertEqual(headline, "a" * 90 + "…")

def test_the_clamp_rejects_a_separator_only_headline(self):
for long in ("-" * 101, ";" * 150, ", " * 80):
with self.subTest(long=long[:10]):
with self.assertRaises(ValueError):
AG.clamp_headline(long)

def test_a_separator_only_overlong_headline_is_untrusted(self):
self.write_all()
self.write("business", self.raw(headline="-" * 101))
self.assertEqual(self.decide()["event"], AG.NONE)

def test_the_clamp_falls_back_when_the_word_cut_is_all_separators(self):
long = "-" * 60 + " " + "abc" * 20
headline = AG.clamp_headline(long)
self.assertEqual(headline, long[:AG.HEADLINE_MAX - 1] + "…")

def test_the_limit_itself_is_accepted(self):
self.write_all()
Expand Down
7 changes: 4 additions & 3 deletions docs/callers/cursor-approve.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,10 @@ The card shows ONE line per axis — the axis's `headline`, capped at 100
characters and required, which has to state **what decided the verdict** rather
than what the axis read. The full `summary` and the confidence are in the job
summary behind the run link; the card is the glance, not the record. A missing
or over-long headline makes that axis untrusted rather than being truncated,
since a verdict with no reason beside it is the exact failure the field was
added to fix.
headline makes that axis untrusted, since a verdict with no reason beside it is
the exact failure the field was added to fix. An over-long one is clamped to the
cap at a word boundary with "…" rather than rejected, because the model's length
is not deterministic; one with nothing but punctuation to keep is untrusted.

## The status card contract

Expand Down
Loading