diff --git a/.github/workflows/linear-ticket.yml b/.github/workflows/linear-ticket.yml index ffd76315..75cb2903 100644 --- a/.github/workflows/linear-ticket.yml +++ b/.github/workflows/linear-ticket.yml @@ -18,6 +18,17 @@ name: Linear Ticket (reusable) # check. Instead validate.sh publishes the `linear-ticket` commit status on the refetched PR # head SHA; branch protection requires THAT context. See docs/callers/linear-ticket.md. # +# REPORTING MODES — how loudly a failure is reported. This workflow NEVER decides whether a PR +# can merge; branch protection does, by requiring the `linear-ticket` context or not: +# enforce: true red status, job exits nonzero. Gating (once required). +# enforce: false red status, job stays green on a failing VERDICT (a +# (soft-fail: true, the default) broken run still exits nonzero). Shows on the PR exactly +# like enforce does but blocks nothing while the context is +# not required — the loud pilot rung, since a green check +# nobody reads teaches the repo nothing during observation. +# enforce: false, soft-fail: false green status; only the job summary and the marker comment +# carry the verdict. The silent variant. +# # PERMISSIONS the caller must grant (the single job's union — GitHub validates the grant at # STARTUP, before any `if:`): # actions: read # read the triggering workflow_run @@ -79,9 +90,29 @@ on: default: true enforce: description: >- - true (default) fails closed on a missing/invalid link or infrastructure error. - false is warn-only: every outcome publishes success but the summary and marker - comment show the verdict enforce mode would have produced. + true (default) fails closed on a missing/invalid link or infrastructure error, and + the job itself exits nonzero. false is warn-only: a failing VERDICT never exits the + job nonzero, and `soft-fail` decides how loudly it is reported. Warn-only is not a + promise the job always exits 0 — a broken run still does (a failed terminal status + write, a missing token/repo, malformed `team-keys`, a non-`pull_request` trigger). + type: boolean + required: false + default: true + soft-fail: + description: >- + Warn-only only (ignored when `enforce: true`). true (default) publishes a red + `failure` commit status so the PR's check list shows the check failing — loud, but + NON-BLOCKING for as long as `linear-ticket` is not a required status in the repo's + branch protection, which is exactly what a warn-only pilot means. The marker comment + says so. false restores the silent variant: warn-only publishes `success` and only + the job summary and comment carry the verdict. + BEHAVIOUR CHANGE for callers already on `enforce: false`: before this input existed + warn-only always published `success`, so taking a SHA bump past this commit flips + those callers to a red (still non-required) status with no caller-side edit. Set + `soft-fail: false` in the caller to keep the old always-green behaviour. + FOOTGUN: `enforce: false` + `soft-fail: true` + `linear-ticket` already required in + branch protection WOULD block merges. Do not require the context before flipping + `enforce: true`. type: boolean required: false default: true @@ -209,5 +240,6 @@ jobs: EXEMPT_ACTORS: ${{ inputs.exempt-actors }} REQUIRE_OPEN_ISSUE: ${{ inputs.require-open-issue }} ENFORCE: ${{ inputs.enforce }} + SOFT_FAIL: ${{ inputs.soft-fail }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: python3 .linear-ticket-tools/scripts/linear-ticket/validate.py diff --git a/README.md b/README.md index 092c23a3..6ec952fb 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ complete, copy-pasteable caller. | Workflow | Purpose | Setup | |---|---|---| | [`detect-unreviewed-merge.yml`](.github/workflows/detect-unreviewed-merge.yml) | SOC 2 compliance — detects PRs merged without prior approval and opens a tracking issue in [`Comfy-Org/unreviewed-merges`](https://github.com/Comfy-Org/unreviewed-merges). | [detect-unreviewed-merge.md](docs/callers/detect-unreviewed-merge.md) | -| [`linear-ticket.yml`](.github/workflows/linear-ticket.yml) | **Requires a Linear issue that Linear has LINKED to the exact PR** before it can merge — not a `TEAM-123`-shaped string in author text. The only thing that passes is an attachment Linear returns for the PR's canonical `html_url` (`attachmentsForURL`, PR URL passed as a GraphQL **variable**) whose issue satisfies the configured team (`team-keys`, matched on the resolved API `team.key`, never a prefix) and state (`require-open-issue` rejects `completed`/`canceled` via `state.type`) policy; a PR linked to several tickets passes when at least one satisfies policy. Identifiers mined from branch/title/body are **diagnostics only** (capped at 20, one batched lookup) — they explain a red check, never turn it green. Fork- and Dependabot-safe by a two-stage split: an unprivileged, checks-out-nothing **signal** workflow on `pull_request`, then a privileged validator on `workflow_run` that GitHub grants the secret + write token. It publishes the required **`linear-ticket`** commit status on the refetched PR head SHA (its own default-branch job is *not* the merge check), refetches the head SHA before every terminal write so a superseded run can't overwrite a newer result, maintains one marker diagnostic comment, and **fails closed** on auth/schema/timeout/rate-limit errors (an infrastructure failure, never a green pass) with bounded retry for Linear's async-link race. `enforce: false` is warn-only (same diagnosis, always-green status); one `exempt-label` (e.g. `linear-exempt`) waives it per-PR, and an opt-in `exempt-actors` list (empty by default) skips bot PRs (e.g. `dependabot[bot]`) with no built-in bypass. Logic + hermetic tests live in [`scripts/linear-ticket/`](scripts/linear-ticket); `workflows_ref` is **required**, shape- and ancestry-checked before checkout. Requires `LINEAR_API_TOKEN` + the two-workflow caller. | [linear-ticket.md](docs/callers/linear-ticket.md) | +| [`linear-ticket.yml`](.github/workflows/linear-ticket.yml) | **Requires a Linear issue that Linear has LINKED to the exact PR** before it can merge — not a `TEAM-123`-shaped string in author text. The only thing that passes is an attachment Linear returns for the PR's canonical `html_url` (`attachmentsForURL`, PR URL passed as a GraphQL **variable**) whose issue satisfies the configured team (`team-keys`, matched on the resolved API `team.key`, never a prefix) and state (`require-open-issue` rejects `completed`/`canceled` via `state.type`) policy; a PR linked to several tickets passes when at least one satisfies policy. Identifiers mined from branch/title/body are **diagnostics only** (capped at 20, one batched lookup) — they explain a red check, never turn it green. Fork- and Dependabot-safe by a two-stage split: an unprivileged, checks-out-nothing **signal** workflow on `pull_request`, then a privileged validator on `workflow_run` that GitHub grants the secret + write token. It publishes the required **`linear-ticket`** commit status on the refetched PR head SHA (its own default-branch job is *not* the merge check), refetches the head SHA before every terminal write so a superseded run can't overwrite a newer result, maintains one marker diagnostic comment, and **fails closed** on auth/schema/timeout/rate-limit errors (an infrastructure failure, never a green pass) with bounded retry for Linear's async-link race. `enforce: false` is warn-only — a failing verdict never exits the job nonzero (a broken run still does), and by default (`soft-fail: true`) it still publishes the **red** status so the pilot is visible on the PR while blocking nothing, since blocking is branch protection requiring the context, not this workflow's call (`soft-fail: false` restores the silent always-green variant); one `exempt-label` (e.g. `linear-exempt`) waives it per-PR, and an opt-in `exempt-actors` list (empty by default) skips bot PRs (e.g. `dependabot[bot]`) with no built-in bypass. Logic + hermetic tests live in [`scripts/linear-ticket/`](scripts/linear-ticket); `workflows_ref` is **required**, shape- and ancestry-checked before checkout. Requires `LINEAR_API_TOKEN` + the two-workflow caller. | [linear-ticket.md](docs/callers/linear-ticket.md) | | [`cursor-review.yml`](.github/workflows/cursor-review.yml) | Label-triggered multi-model code review. A 4-lab × 2-review-type cursor-agent panel runs adversarial + edge-case passes, a judge model consolidates them into one PR review with per-finding severity badges, and the triggerer gets Slack start/complete DMs. Advisory: it posts a review, it does not gate — there is **no `blocking` input** (it shipped in #16 and was deleted by #31; see [the setup guide](docs/callers/cursor-review.md)). Prompts and scripts live in [`.github/cursor-review/`](.github/cursor-review) — the single source of truth, so consumer repos carry only a thin caller. Generated code is excluded from BOTH the diff-size gate and the diff sent to the panel via the shared [`check-pr-size`](scripts/check-pr-size) classifier (base-ref `linguist-generated`, base-blob Go generated markers — a PR cannot self-exempt a file from review, lockfiles, plus `extra_generated_globs` / `extra_lockfiles`) — the same detection as `pr-size.yml`, so codegen never blows the review cap or wastes review signal. Blank and comment-only lines are also discounted from the size count by default (`ignore_comments`) so AI comment volume doesn't trip the cap and skip the review — comments still reach the panel, only the count ignores them. The runner and the panel model list are fixed by the reusable — there are no `runs_on` / `models` inputs (`models` is a job *output*). Requires `CURSOR_API_KEY` (+ optional `SLACK_BOT_TOKEN`). | [cursor-review.md](docs/callers/cursor-review.md) | | [`cursor-review-auto-label.yml`](.github/workflows/cursor-review-auto-label.yml) | Companion to `cursor-review.yml`. On PR assignment, applies the review label for an opted-in reviewer (via the CLOUD_CODE_BOT app token, so the label actually triggers the review). The opt-in roster lives in the caller's `vars.CURSOR_REVIEW_OPTED_IN_LOGINS` — no roster is baked into the workflow. Requires `vars.APP_ID` + `CLOUD_CODE_BOT_PRIVATE_KEY`. | [cursor-review-auto-label.md](docs/callers/cursor-review-auto-label.md) | | [`assign-reviewers.yml`](.github/workflows/assign-reviewers.yml) | Auto-requests expertise-aware, load-balanced PR reviewers with new-folk randomization. Matches changed paths against a caller-repo `.github/reviewers.yml` (path-glob → reviewers, plus a `default_pool`), drops the author + `vars.REVIEWER_EXCLUDE`, ranks candidates by open review load (steering off anyone at/over `vars.REVIEWER_LOAD_CAP`), and may swap a slot for a `vars.REVIEWER_GROWTH_POOL` member. Requests go through the CLOUD_CODE_BOT app token so they work on fork PRs. Requires `vars.APP_ID` + `CLOUD_CODE_BOT_PRIVATE_KEY`. | [assign-reviewers.md](docs/callers/assign-reviewers.md) | diff --git a/docs/callers/linear-ticket.md b/docs/callers/linear-ticket.md index a96b14e4..9c47bd79 100644 --- a/docs/callers/linear-ticket.md +++ b/docs/callers/linear-ticket.md @@ -88,6 +88,8 @@ jobs: # exempt-actors: dependabot[bot],renovate[bot] # optional; bot PRs skip the check # require-open-issue: true # default enforce: false # WARN-ONLY during rollout; flip to true when ready + # soft-fail: true # default; warn-only still shows a RED (non-blocking) + # # check. Set false for a silent always-green pilot. secrets: LINEAR_API_TOKEN: ${{ secrets.LINEAR_API_TOKEN }} ``` @@ -122,13 +124,50 @@ job-level detail. | `exempt-label` | `''` | Single label that waives the requirement (recommended `linear-exempt`). Empty disables exemption. | | `exempt-actors` | `''` | Comma-separated PR-author logins whose PRs skip the check without a label (e.g. `dependabot[bot],renovate[bot]`) — the non-manual hatch for bot PRs that never carry a ticket. Opt-in; empty means no bypass. Listing an actor means **all** of its PRs merge without a Linear ticket, so keep it to trusted automation accounts. | | `require-open-issue` | `true` | Reject a linked issue whose Linear `state.type` is `completed`/`canceled`. `backlog`/`unstarted`/`started`/`triage` pass. | -| `enforce` | `true` | `false` is warn-only: every outcome publishes success, but the summary and marker comment show the verdict enforce mode would have produced. | +| `enforce` | `true` | `false` is warn-only: a failing **verdict** never exits the job nonzero. `soft-fail` decides how loudly it is reported; the diagnosis is identical either way. Warn-only is not a promise the job always exits `0` — a broken *run* still does (a failed terminal status write, a missing token/repo, malformed `team-keys`, a non-`pull_request` trigger). | +| `soft-fail` | `true` | Warn-only only (ignored when `enforce: true`). `true` publishes a **red `failure`** status, so the PR's check list shows the check failing — loud, but non-blocking for as long as `linear-ticket` is not a required status in your ruleset. `false` restores the silent variant (warn-only publishes `success`; only the summary and comment carry the verdict). **Do not** require the `linear-ticket` context while `enforce: false` — required + soft-fail would block merges. | + +## Upgrading an existing `enforce: false` caller + +`soft-fail` is new, and it defaults to `true`. **If your caller already runs `enforce: false`, +taking a SHA bump past the commit that added this input changes what your PRs show:** warn-only +used to publish a green `linear-ticket` status in every outcome, and now publishes a **red** +one. Nothing about *blocking* changes — a status only blocks when your ruleset requires that +context — but the PR check list goes from green to red, and anything you have keyed off the +status (auto-merge bots, dashboards, other automation) sees `failure` where it saw `success`. + +Two things to do before you take the bump: + +1. **Check your ruleset.** If `linear-ticket` is already a required check while you are still on + `enforce: false`, the red status *will* block merges. That combination was always a + misconfiguration (rollout step 6 is the last step for a reason), but soft-fail is what makes + it bite. Remove the required check, or flip `enforce: true` — do not sit in between. +2. **Decide which pilot you want.** Loud (the default) is recommended: authors fix links during + the pilot instead of discovering the requirement on flip day. If you want the old behaviour, + add one line to your caller: + + ```yaml + enforce: false + soft-fail: false # keep the pre-soft-fail behaviour: warn-only stays green + ``` + +An **absent** `SOFT_FAIL` — which only happens if your `uses:` pin and `workflows_ref` have +skewed apart, so an older `linear-ticket.yml` is running newer scripts — is treated as +`false`, so pin skew alone can never flip you to red. Keep both pins on the same SHA anyway. ## Rollout (warn-only first) -1. Land both caller files with `enforce: false`. Observe for ~a week: attachment latency, URL - matching, retry counts, team/state outcomes, fork/Dependabot behaviour, and rate-limit - headroom (the validator logs `X-RateLimit-*` headers). +1. Land both caller files with `enforce: false`, and **leave `linear-ticket` out of your + ruleset's required checks** — that omission, not the input, is what keeps the pilot + non-blocking. Observe for ~a week: attachment latency, URL matching, retry counts, + team/state outcomes, fork/Dependabot behaviour, and rate-limit headroom (the validator logs + `X-RateLimit-*` headers). + + With the default `soft-fail: true`, a warn-only failure shows on the PR as a **red + `linear-ticket` check** plus the marker comment, which is what makes authors notice and fix + links during the pilot instead of discovering the requirement on flip day. Nobody is + blocked: a non-required failing check leaves the merge button live. If you want the quieter + first pass, set `soft-fail: false` and the status stays green. 2. Confirm the `linear-ticket` status publishes on **same-repo, fork, and Dependabot** PR head SHAs, and that automatic (branch/title/body) and manual links both resolve to the same canonical URL. @@ -140,8 +179,11 @@ job-level detail. deliberate v2 follow-up, out of scope here. 4. Decide bot-PR handling: set `exempt-actors` (e.g. `dependabot[bot],renovate[bot]`) so dependency PRs pass without hand-labelling each one. -5. Flip `enforce: true`. Leave the check non-required for a short window. +5. Flip `enforce: true`. Leave the check non-required for a short window — the PR-visible + result is unchanged from the soft-fail pilot; what changes is that the validator's own run + now goes red too, so a systemic breakage is visible in the Actions tab. 6. Add the **`linear-ticket`** status context to your `main` ruleset as a required check. + Only now does anything block. Nothing here changes branch protection automatically — the last step is a manual repo-admin action. @@ -151,6 +193,12 @@ Nothing here changes branch protection automatically — the last step is a manu your ruleset; do not require the `Linear Ticket / validate` job — it runs on the default branch against the workflow_run event and is not tied to the PR's merge check. +**A red check is not the same as a blocked merge.** What blocks is your ruleset requiring the +`linear-ticket` context — nothing this workflow does. That is why warn-only can (and by default +does) publish a red status: it is a real, visible signal that costs nobody a merge. The +corollary is the footgun: do not add the context to your ruleset until you flip `enforce: true`, +or the warn-only pilot starts blocking. + **A manual link created after a red run does not emit a GitHub event.** After linking in Linear, either re-run the failed validate workflow or edit the PR title/body to trigger a fresh `edited` event. The failure comment says exactly this. diff --git a/scripts/linear-ticket/README.md b/scripts/linear-ticket/README.md index 6fb530af..7bd66ef7 100644 --- a/scripts/linear-ticket/README.md +++ b/scripts/linear-ticket/README.md @@ -43,8 +43,24 @@ That is the whole point: a `TEAM-123`-shaped string an author types is not a lin Linear signals rate limiting as HTTP 400 + GraphQL `RATELIMITED`, which is retried within a bounded budget (2s, 4s, 8s, 16s over five attempts). - Comment I/O is **best-effort**: a comment failure is warned, never changes the verdict. -- `enforce: false` is **warn-only** — every outcome publishes success, but the summary and - marker comment show the verdict enforce mode would have produced. +- **Reporting mode is not the verdict.** `failure_outcome()` maps (category, mode) to the + status state, comment headline, and exit code; the diagnosis is identical in every mode. + `enforce: true` → red status + nonzero exit. `enforce: false` (warn-only) never exits + nonzero *on a verdict* and, with `soft-fail: true`, still publishes a **red** status — loud + on the PR but blocking nothing while `linear-ticket` is not a required context, with a + category-aware advisory note on the comment. `soft-fail: false` restores the silent + always-green warn-only behaviour, and so does an **absent** `SOFT_FAIL` env: the workflow + always passes it, so absent means a skewed `workflows_ref`/`uses:` pin, and a caller that + cannot express the input is never silently upgraded to red (`lib.soft_fail_enabled`). +- The advisory note is **hedged on purpose**: the validator never reads the caller's ruleset, + so it cannot claim "this does not block the merge" — in the documented footgun (warn-only + + the context already required) that claim would be false and would misdirect the author. It + names what actually blocks (your ruleset) and the misconfiguration to report. +- Status **descriptions are category-aware**: `infra_error` means Linear could not be queried, + so the status must not read "no linked Linear issue" — a diagnosis the run never made. +- A failed **terminal status write** is exit 1 even in warn-only: soft-fail's whole point is + that the red status lands, and a missing write leaves the previous status standing. This is + one of the run-level exits that warn-only does *not* suppress. ## Running the tests @@ -56,5 +72,5 @@ python3 -m py_compile scripts/linear-ticket/lib.py scripts/linear-ticket/validat The hermetic suite covers the design's acceptance cases that live in pure functions (extraction/dedup/cap, `team-keys` validation, the policy gate incl. multi-link and state/team handling, error classification, category selection, the diagnostic-query -builder). The **real-Linear behaviours** — URL canonicalization and attachment timing — are +builder, and the reporting-mode table). The **real-Linear behaviours** — URL canonicalization and attachment timing — are proven in the pilot, not by mocks (design §11). diff --git a/scripts/linear-ticket/lib.py b/scripts/linear-ticket/lib.py index d582ec92..6abf5510 100644 --- a/scripts/linear-ticket/lib.py +++ b/scripts/linear-ticket/lib.py @@ -21,6 +21,7 @@ from __future__ import annotations import re +from typing import NamedTuple # The hidden marker on the single PR comment this gate maintains. validate.py finds its # existing comment by this exact string, so it must never change casually. @@ -212,6 +213,144 @@ def failure_guidance(category: str) -> str: return _GUIDANCE[category] +# ── how a failure is REPORTED (loudness, not diagnosis) ───────────────────────────────── +# select_failure_category above says WHY a check is red; the mode below says how loudly it is +# reported. Whether a red check BLOCKS a merge is never decided here — that is branch +# protection deciding whether the `linear-ticket` context is required: +# +# enforce -> failure status, exit 1. The gating configuration. +# warn-only + soft-fail -> failure status, exit 0. Shows on the PR exactly like enforce +# does, but blocks nothing while the context is not required. +# The loud pilot rung: a green check nobody looks at teaches a +# repo nothing, and the pilot's whole purpose is observation. +# warn-only, soft-fail off -> success status, exit 0. Silent; only the job summary and the +# marker comment carry the verdict. +# +# The job's OWN exit code is not what a reviewer sees. This validator runs on the default +# branch off workflow_run, so its run is not attached to the PR — the commit status is the +# only PR-visible signal, which is why soft-fail moves the STATUS and leaves the run green. +# A red default-branch run for a check that is deliberately not gating would read as "main is +# broken" in the Actions tab, which is louder in the wrong place. + +# A commit status description is the PR-visible one-liner. It must state the DIAGNOSIS this +# run actually reached: "no linked Linear issue" is wrong for infra_error, where Linear could +# not be queried at all and the run therefore determined nothing about the ticket. +_STATUS_HEADLINE = { + "no_candidate": "No linked Linear issue", + "exists_not_linked": "No linked Linear issue", + "policy_mismatch": "Linked Linear issue fails this repo's policy", + "infra_error": "Could not verify — Linear could not be queried", +} + +# Appended to the marker comment whenever the check is red but warn-only, so nobody reads the +# red X as a merge block (and nobody assumes it will stay non-blocking forever). +# +# WORDING IS DELIBERATELY HEDGED. This validator never reads the caller's ruleset, so it cannot +# know whether `linear-ticket` is a required context — asserting "this does not block" would be +# a flat lie in exactly the configuration docs/callers/linear-ticket.md calls a footgun +# (warn-only + the context already required), and would send that author to debug the wrong +# thing. So the note states what IS knowable (this workflow is not what blocks; a warn-only +# pilot is meant to run before the context is required) and names the misconfiguration to +# report if the merge button is in fact blocked. +_ADVISORY_BASE = ( + "> ⚠️ **Advisory — this check is warn-only.** It is reported as failing so it shows up in " + "the PR's check list, but nothing this workflow does blocks a merge: what blocks is your " + "repository's ruleset requiring the `linear-ticket` status context, and a warn-only pilot " + "is meant to run *before* that context is required. If your merge button is actually " + "blocked on `linear-ticket`, then the context is already required while the pilot is still " + "warn-only — tell the repository owners, because that combination is a misconfiguration." +) +_ADVISORY_TAIL = { + # infra_error is not the author's to fix — telling them to link a ticket would send them + # after a broken credential with the wrong tool. + "infra_error": ( + " This run could not reach Linear, so there is nothing to fix on the PR itself: re-run " + "the check, and if it keeps failing contact the repository owners." + ), +} +_ADVISORY_TAIL_DEFAULT = ( + " Linking the ticket now is what keeps this from blocking you once the pilot starts " + "enforcing." +) + + +def advisory_note(category: str) -> str: + """The non-gating note appended to the marker comment for a warn-only red verdict. + + Category-aware: the shared hedged paragraph plus the one next step that is actually right + for this diagnosis. Raises KeyError on an unknown category, matching failure_guidance. + """ + if category not in _GUIDANCE: + raise KeyError(category) + return _ADVISORY_BASE + _ADVISORY_TAIL.get(category, _ADVISORY_TAIL_DEFAULT) + + +def soft_fail_enabled(raw: str | None) -> bool: + """Parse the ``SOFT_FAIL`` env var. An ABSENT value means the SILENT variant. + + linear-ticket.yml always passes SOFT_FAIL and its own input defaults to true, so a + correctly-pinned caller gets the loud mode regardless of this default. An *absent* value can + only mean the workflow YAML predates the input — `workflows_ref` skewed ahead of the + caller's `uses:` pin, the skew AGENTS.md warns never to create by hand-bumping one of the + two pins. A caller that cannot even express the input must not be silently upgraded from a + green status to a red one, so only an explicit "true" opts in. + """ + return (raw or "").strip().lower() == "true" + + +class FailureOutcome(NamedTuple): + """How one failing verdict is reported. + + verdict headline for the marker comment and job summary + state commit-status state published on the PR head SHA + description commit-status description (validate.py truncates to GitHub's 140 chars) + exit_code the validator process's exit code + advisory whether to append advisory_note(category) to the comment + """ + + verdict: str + state: str + description: str + exit_code: int + advisory: bool + + +def failure_outcome(category: str, enforce: bool, soft_fail: bool) -> FailureOutcome: + """Map (category, mode) to the reported outcome. See the mode table above. + + ``soft_fail`` is read only when ``enforce`` is false — enforcing already publishes the red + status soft-fail exists to produce, and an enforcing run must still exit nonzero. Raises + KeyError on an unknown category, matching failure_guidance, so a typo fails loudly instead + of shipping a status nobody can explain. + """ + if category not in _GUIDANCE: + raise KeyError(category) + headline = _STATUS_HEADLINE[category] + if enforce: + return FailureOutcome( + verdict=f"\u274c fail ({category})", + state="failure", + description=f"{headline} ({category})", + exit_code=1, + advisory=False, + ) + if soft_fail: + return FailureOutcome( + verdict=f"\u274c fail ({category}) \u2014 warn-only", + state="failure", + description=f"Warn-only: {headline[0].lower()}{headline[1:]} ({category})", + exit_code=0, + advisory=True, + ) + return FailureOutcome( + verdict=f"\u26a0\ufe0f warn-only (would fail: {category})", + state="success", + description=f"warn-only: would fail ({category})", + exit_code=0, + advisory=False, + ) + + def build_diagnostic_query(candidates: list[str]) -> str: """ONE aliased GraphQL query (``c0: issueSearch(...) c1: ...``) resolving all candidates. diff --git a/scripts/linear-ticket/tests/test_lib.py b/scripts/linear-ticket/tests/test_lib.py index de777f44..b476766e 100644 --- a/scripts/linear-ticket/tests/test_lib.py +++ b/scripts/linear-ticket/tests/test_lib.py @@ -174,6 +174,150 @@ def test_unknown_category_raises(self): lib.failure_guidance("bogus") +ALL_CATEGORIES = ("no_candidate", "exists_not_linked", "policy_mismatch", "infra_error") + + +class FailureOutcomeModes(unittest.TestCase): + """The reporting-mode table in lib.py. The invariant under all of it: soft-fail changes + only how LOUD a red verdict is, never the diagnosis and never the exit code contract.""" + + def test_every_category_has_copy_for_every_surface(self): + # ALL_CATEGORIES must be exhaustive, or the loops below silently stop covering a + # category; and _STATUS_HEADLINE must cover exactly what _GUIDANCE does, or + # failure_outcome raises KeyError on a category that passed its own guard. + self.assertEqual(set(ALL_CATEGORIES), set(lib._GUIDANCE)) + self.assertEqual(set(lib._STATUS_HEADLINE), set(lib._GUIDANCE)) + + def test_enforce_is_red_and_exits_nonzero(self): + outcome = lib.failure_outcome("no_candidate", enforce=True, soft_fail=False) + self.assertEqual(outcome.state, "failure") + self.assertEqual(outcome.exit_code, 1) + self.assertFalse(outcome.advisory) + + def test_enforce_ignores_soft_fail(self): + # soft-fail is a warn-only knob; enforcing already publishes the red status it exists + # to produce, and must still exit nonzero. + self.assertEqual(lib.failure_outcome("no_candidate", enforce=True, soft_fail=True), + lib.failure_outcome("no_candidate", enforce=True, soft_fail=False)) + + def test_soft_fail_is_red_but_exits_zero(self): + outcome = lib.failure_outcome("exists_not_linked", enforce=False, soft_fail=True) + self.assertEqual(outcome.state, "failure") # loud: the PR check list shows a red X + self.assertEqual(outcome.exit_code, 0) # but the run itself stays green + self.assertTrue(outcome.advisory) # and the comment says it does not block + + def test_silent_warn_only_is_green(self): + outcome = lib.failure_outcome("exists_not_linked", enforce=False, soft_fail=False) + self.assertEqual(outcome.state, "success") + self.assertEqual(outcome.exit_code, 0) + self.assertFalse(outcome.advisory) + + def test_only_enforce_ever_exits_nonzero(self): + # Scope: the VERDICT's exit code. The job can still exit nonzero in warn-only for + # reasons that are not a verdict — a failed terminal status write (finish_fail), or a + # broken run (missing GH_REPO/LINEAR_API_TOKEN, malformed team-keys, a non- + # pull_request event). The docs say "a failing verdict", not "the job", for that + # reason. + for category in ALL_CATEGORIES: + for soft_fail in (True, False): + self.assertEqual( + lib.failure_outcome(category, enforce=False, soft_fail=soft_fail).exit_code, + 0, f"warn-only must not exit nonzero ({category}, soft_fail={soft_fail})") + + def test_advisory_only_when_red_and_non_gating(self): + # The "does not block" note must never ride along with an enforcing (gating) verdict — + # that would tell an author to ignore a check that is actually blocking them. + for category in ALL_CATEGORIES: + for enforce in (True, False): + for soft_fail in (True, False): + outcome = lib.failure_outcome(category, enforce, soft_fail) + self.assertEqual(outcome.advisory, + not enforce and soft_fail and outcome.state == "failure") + + def test_category_is_named_in_every_verdict_and_description(self): + for category in ALL_CATEGORIES: + for enforce in (True, False): + for soft_fail in (True, False): + outcome = lib.failure_outcome(category, enforce, soft_fail) + self.assertIn(category, outcome.verdict) + self.assertIn(category, outcome.description) + + def test_description_fits_the_github_status_limit(self): + # publish_status truncates at 140; a description that only ever survives truncation + # would silently lose the category, so keep every mode's copy inside the limit. + for category in ALL_CATEGORIES: + for enforce in (True, False): + for soft_fail in (True, False): + outcome = lib.failure_outcome(category, enforce, soft_fail) + self.assertLessEqual(len(outcome.description), 140) + + def test_unknown_category_raises(self): + with self.assertRaises(KeyError): + lib.failure_outcome("bogus", enforce=False, soft_fail=True) + + def test_description_names_the_diagnosis_this_run_actually_reached(self): + # infra_error means Linear could not be queried, so the run determined NOTHING about + # the ticket; a status reading "no linked Linear issue" would be a diagnosis it never + # made. Soft-fail promotes this copy to the loudest PR-visible surface. + for enforce in (True, False): + for soft_fail in (True, False): + infra = lib.failure_outcome("infra_error", enforce, soft_fail) + self.assertNotIn("linked Linear issue", infra.description) + self.assertIn("linked Linear issue", + lib.failure_outcome("no_candidate", True, False).description) + + +class AdvisoryNote(unittest.TestCase): + """The note appended to a warn-only red verdict.""" + + def test_never_asserts_flatly_that_nothing_is_blocked(self): + # The validator never reads the caller's ruleset, so it cannot know whether the + # `linear-ticket` context is required. In the documented footgun configuration + # (warn-only + context already required) the red check IS blocking the author, and a + # flat "this does not block the merge" would send them to debug the wrong thing. + for category in ALL_CATEGORIES: + note = lib.advisory_note(category) + self.assertNotIn("This does not block", note) + self.assertIn("warn-only", note) + # It must still name what actually blocks, and the misconfiguration to report. + self.assertIn("ruleset", note) + self.assertIn("misconfiguration", note) + + def test_infra_error_does_not_tell_the_author_to_link_a_ticket(self): + # A missing/invalid LINEAR_API_TOKEN is not the PR author's to fix. + note = lib.advisory_note("infra_error") + self.assertNotIn("Linking the ticket", note) + self.assertIn("could not reach Linear", note) + + def test_ticket_categories_do_tell_the_author_to_link(self): + for category in ("no_candidate", "exists_not_linked", "policy_mismatch"): + self.assertIn("Linking the ticket", lib.advisory_note(category)) + + def test_unknown_category_raises(self): + with self.assertRaises(KeyError): + lib.advisory_note("bogus") + + +class SoftFailEnv(unittest.TestCase): + """Parsing of the SOFT_FAIL env var — the pin-skew guard.""" + + def test_absent_is_the_silent_variant(self): + # linear-ticket.yml ALWAYS passes SOFT_FAIL, so absent means the workflow YAML predates + # the input: `workflows_ref` skewed ahead of the caller's `uses:` pin. A caller that + # cannot express the input must not be silently upgraded from green to red statuses. + self.assertFalse(lib.soft_fail_enabled(None)) + self.assertFalse(lib.soft_fail_enabled("")) + + def test_explicit_true_opts_in(self): + self.assertTrue(lib.soft_fail_enabled("true")) + self.assertTrue(lib.soft_fail_enabled("TRUE")) + self.assertTrue(lib.soft_fail_enabled(" true ")) + + def test_explicit_false_and_anything_unrecognised_stay_silent(self): + for raw in ("false", "False", "0", "yes", "no", "1", "maybe"): + self.assertFalse(lib.soft_fail_enabled(raw), raw) + + class DiagnosticQuery(unittest.TestCase): def test_empty_raises(self): with self.assertRaises(ValueError): diff --git a/scripts/linear-ticket/validate.py b/scripts/linear-ticket/validate.py index 6d990c1e..cfcb7d18 100644 --- a/scripts/linear-ticket/validate.py +++ b/scripts/linear-ticket/validate.py @@ -19,7 +19,12 @@ TEAM_KEYS raw ``team-keys`` input (comma-separated; empty = any team) EXEMPT_LABEL exemption label name; empty disables exemption REQUIRE_OPEN_ISSUE "true"/"false" - ENFORCE "true" (fail closed) / "false" (warn-only: always green, same diagnosis) + ENFORCE "true" (fail closed) / "false" (warn-only: a failing VERDICT never + exits nonzero; a broken run still can — see finish_fail and run()) + SOFT_FAIL warn-only only: "true" publishes a red `failure` status — loud, and + non-blocking for as long as the caller's ruleset does not require the + `linear-ticket` context; "false" (and, deliberately, an ABSENT value) + keeps the silent always-green warn-only behaviour RUN_URL html_url of this workflow run, for the status target and comment LINEAR_API_URL optional override (default https://api.linear.app/graphql) GITHUB_STEP_SUMMARY written with the human-readable outcome @@ -233,12 +238,14 @@ def _log_rate_limit(headers) -> None: # ── orchestration ─────────────────────────────────────────────────────────────────────── class Validator: - def __init__(self, gh: GitHub, token: str, team_keys, require_open, enforce, run_url): + def __init__(self, gh: GitHub, token: str, team_keys, require_open, enforce, soft_fail, + run_url): self.gh = gh self.token = token self.team_keys = team_keys self.require_open = require_open self.enforce = enforce + self.soft_fail = soft_fail self.run_url = run_url self.exempt_label = os.environ.get("EXEMPT_LABEL", "") self.exempt_actors = lib.parse_actor_list(os.environ.get("EXEMPT_ACTORS", "")) @@ -281,24 +288,19 @@ def finish_exempt(self, headline: str, status_desc: str) -> int: def finish_fail(self, category: str, detail: str) -> int: guidance = lib.failure_guidance(category) - if self.enforce: - verdict = f"❌ fail ({category})" - state = "failure" - short = f"No linked Linear issue ({category})" - else: - verdict = f"⚠️ warn-only (would fail: {category})" - state = "success" - short = f"warn-only: would fail ({category})" + outcome = lib.failure_outcome(category, self.enforce, self.soft_fail) body_lines = [ lib.MARKER, "", - f"### Linear ticket check — {verdict}", + f"### Linear ticket check — {outcome.verdict}", "", guidance, ] if detail: body_lines += ["", detail] + if outcome.advisory: + body_lines += ["", lib.advisory_note(category)] body_lines += [ "", "---", @@ -311,20 +313,23 @@ def finish_fail(self, category: str, detail: str) -> int: # superseded run overwrites the newer run's comment (leaving a stale failure beside a # green status) even though its status write is suppressed. if not self._guard_supersession(): - return 0 if not self.enforce else 1 + return outcome.exit_code self.gh.upsert_marker_comment(self.pr_number, "\n".join(body_lines)) - summary(f"## linear-ticket: {verdict}") + summary(f"## linear-ticket: {outcome.verdict}") summary("") summary(guidance) if detail: summary(detail) - if not self.gh.publish_status(self.validated_sha, state, short, self.run_url, - critical=True): + # A failed terminal status write is always exit 1, even in warn-only: the point of + # soft-fail is that the red status LANDS, and a silently-missing one leaves a stale + # prior status standing as this commit's answer. + if not self.gh.publish_status(self.validated_sha, outcome.state, outcome.description, + self.run_url, critical=True): return 1 - return 0 if not self.enforce else 1 + return outcome.exit_code # -- the run --------------------------------------------------------------------------- def run(self, event: dict) -> int: @@ -522,9 +527,14 @@ def main() -> int: require_open = os.environ.get("REQUIRE_OPEN_ISSUE", "true") != "false" enforce = os.environ.get("ENFORCE", "true") != "false" + # NOT defaulted true here even though the workflow input is: linear-ticket.yml always + # passes SOFT_FAIL, so an absent value means a skewed pin (old workflow YAML, new scripts) + # rather than a caller choosing the loud mode. See lib.soft_fail_enabled. + soft_fail = lib.soft_fail_enabled(os.environ.get("SOFT_FAIL")) run_url = os.environ.get("RUN_URL", "") - validator = Validator(GitHub(repo), token, team_keys, require_open, enforce, run_url) + validator = Validator(GitHub(repo), token, team_keys, require_open, enforce, soft_fail, + run_url) return validator.run(event)