diff --git a/.github/workflows/validate-pr.yml b/.github/workflows/validate-pr.yml index ab10305b..17be64be 100644 --- a/.github/workflows/validate-pr.yml +++ b/.github/workflows/validate-pr.yml @@ -57,6 +57,23 @@ jobs: # the tests/ files import yaml. run: python3 -m pip install --user pytest pyyaml + - name: Reject a skill-quality baseline that grew + # tests/skill-quality-baseline.json may only shrink. The pytest gate + # enforces "skill <= its entry"; this step enforces "entry <= what the + # base branch had" so a PR cannot raise or add a ceiling to pass. + # The working tree is the PR's test merge commit, so it already holds any + # slimming that landed on the base branch, and OLD could be base.sha or + # the merge-base. The merge-base is used to match the dist/ and CHANGELOG + # steps. + run: | + set -e + MERGE_BASE="$(git merge-base "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}")" + if git show "${MERGE_BASE}:tests/skill-quality-baseline.json" > /tmp/base-baseline.json 2>/dev/null; then + python3 tests/skill_quality.py --check-ratchet /tmp/base-baseline.json + else + echo "notice: no baseline at the merge-base; skipping ratchet diff" >&2 + fi + - name: Validate the Kiro Power (powers/konductor/) # Users import the Power from the tagged release matching VERSION # (github.com/aws-solutions/konductor/tree/v/powers/konductor), diff --git a/AGENTS.md b/AGENTS.md index 4f90cc02..8826a6f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,6 +65,20 @@ instructions. convert a description from one form to the other for consistency alone. Fix a description that gives no activation condition at all. You may broaden an existing condition (for example, add a disjunct) when the skill's actual scope grew, but do not churn the form otherwise. +- Reference material a skill needs only sometimes (catalogues, long templates, worked examples) + lives in `skills//references/*.md`, one level deep, Markdown only. `SKILL.md` keeps the + instructions and must name each reference file with a cue saying when to read it (for example + "Read `references/aws-best-practices.md` before Stage 3"). `make test-py` runs the checks + listed in the docstring of `tests/skill_quality.py`. Among them, it fails on a frontmatter + `name` that does not match the directory, a description that is too short or gives no + activation condition, a missing link target, an unclosed code fence, an unmentioned reference + file, a reference mention without a read cue, and any size over the thresholds in + `tests/skill-quality-baseline.json` (currently 500 lines, 32 KiB and 512 description + characters) that the baseline does not carry. The baseline may only shrink. + `make test-py` only checks each skill against its entry; CI rejects a raised or added entry, + and so does `python3 tests/skill_quality.py --print-baseline` when you regenerate. Renaming a + skill counts as adding an entry, so bring an oversized skill under every threshold before you + rename it. - Orchestrators are named `konductor`, `konductor-mux-orchestrator`, and `konductor-cmux-orchestrator`; every specialist uses a `k-*` name. Skills are unprefixed unless avoiding a known collision. - **License headers:** All code files must carry an SPDX short-form identifier as the very first line diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f908710..ea7df57f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ not per individual commit. ## [Unreleased] +### Added + +- `make test-py` now checks every skill's frontmatter, description, size, code fences, and + `references/` links against a shrink-only baseline in `tests/skill-quality-baseline.json`. + `python3 tests/skill_quality.py --print-baseline` refuses to write a baseline that grew. + ### Changed - Claude Code subagents now preload 1 to 4 skills each (90 KB in total, down from 1.8 MB); the rest diff --git a/tests/skill-quality-baseline.json b/tests/skill-quality-baseline.json new file mode 100644 index 00000000..566cc761 --- /dev/null +++ b/tests/skill-quality-baseline.json @@ -0,0 +1,69 @@ +{ + "_comment": "Shrink-only ratchet read by tests/test_skill_quality.py. An entry is the exact measured value of a dimension that is over its threshold. Lower or delete an entry when you slim a skill; never raise one; never add one for an existing skill unless the threshold above was lowered in the same change. Renaming a skill adds an entry, so bring a renamed skill under every threshold first. Regenerate with: python3 tests/skill_quality.py --print-baseline", + "skills": { + "cost-estimation": { + "lines": 667 + }, + "cypress-test-implementation": { + "bytes": 53895, + "lines": 1944 + }, + "dynamodb-design": { + "bytes": 75539, + "lines": 2195 + }, + "dynamodb-validation": { + "bytes": 64678, + "lines": 1903 + }, + "e2e-test-strategy": { + "bytes": 46255, + "lines": 1595 + }, + "humanize-writing": { + "bytes": 38522, + "lines": 567 + }, + "iam-policy-design": { + "bytes": 39567, + "lines": 969 + }, + "infra-validation": { + "bytes": 49149, + "lines": 1346 + }, + "policy-validation": { + "bytes": 36839, + "lines": 968 + }, + "security-test-generation": { + "lines": 736 + }, + "smithy-modeling": { + "lines": 603 + }, + "smithy-validation": { + "lines": 574 + }, + "sop-state-management": { + "bytes": 32890 + }, + "test-coverage-analysis": { + "bytes": 69262, + "lines": 2064 + }, + "ui-text-validation": { + "lines": 570 + } + }, + "thresholds": { + "bytes": 32768, + "description_chars": 512, + "lines": 500 + }, + "tolerance": { + "bytes": 1024, + "description_chars": 50, + "lines": 50 + } +} diff --git a/tests/skill_quality.py b/tests/skill_quality.py new file mode 100644 index 00000000..c8ce4a98 --- /dev/null +++ b/tests/skill_quality.py @@ -0,0 +1,625 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Static quality checks for skills/*/SKILL.md, used by tests/test_skill_quality.py. + +Checks each skill for: frontmatter `name` matching its directory; a description +with an activation condition and a length window; line and byte ceilings; no +unclosed code fence; and link, bare-path and `references/` integrity. Sizes over a threshold must be +carried by tests/skill-quality-baseline.json, which may only shrink. + +Needs PyYAML. Without it, run: uvx --with pyyaml pytest -q tests/ + +CLI (run from anywhere): + python3 tests/skill_quality.py --scorecard + python3 tests/skill_quality.py --print-baseline > /tmp/b.json && mv /tmp/b.json tests/skill-quality-baseline.json + python3 tests/skill_quality.py --print-baseline --allow-growth > /tmp/b.json && mv /tmp/b.json tests/skill-quality-baseline.json + (--allow-growth only after lowering a threshold) + python3 tests/skill_quality.py --check-ratchet OLD_BASELINE.json +""" +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +SKILLS_DIR = REPO_ROOT / "skills" +BASELINE_PATH = Path(__file__).resolve().parent / "skill-quality-baseline.json" + +DIMENSIONS = ("bytes", "description_chars", "lines") +DEFAULT_THRESHOLDS = {"bytes": 32768, "description_chars": 512, "lines": 500} +DEFAULT_TOLERANCE = {"bytes": 1024, "description_chars": 50, "lines": 50} +DESCRIPTION_MIN_CHARS = 60 +DESCRIPTION_MIN_WORDS = 8 + +BASELINE_COMMENT = ( + "Shrink-only ratchet read by tests/test_skill_quality.py. An entry is the exact " + "measured value of a dimension that is over its threshold. Lower or delete an entry " + "when you slim a skill; never raise one; never add one for an existing skill unless " + "the threshold above was lowered in the same change. Renaming a skill adds an entry, so " + "bring a renamed skill under every threshold first. Regenerate with: " + "python3 tests/skill_quality.py --print-baseline" +) + +# Activation signals. These are heuristics: any one passing is enough, and the +# verb proxy alone only warns. +# "Use/load/run ... when/for/before ..." where the verb starts a clause. +TRIGGER_CLAUSE = re.compile( + r"(?:^|[.;:!?,]\s+)(use|invoke|run|load|apply)\b.{0,60}?" + r"\b(when|for|after|before|whenever|to|if|during|on)\b", + re.IGNORECASE, +) +# A condition word anywhere in the description. +CONDITION_WORD = re.compile(r"\b(when|whenever|before|after|triggers?)\b", re.IGNORECASE) +# Passive applicability, e.g. "Loaded only in Claude Code sessions". +SCOPE_CLAUSE = re.compile(r"\bonly (in|for|when|on)\b|\bapplies to\b", re.IGNORECASE) +# An explicit mandate. +MUST_USE = re.compile(r"\bMUST USE\b", re.IGNORECASE) +# Behavior-summary form: a capitalised third-person verb first ("Validates ..."). +# A plural noun before a preposition ("Notes on ...") is not a verb. +VERB_PROXY = re.compile(r"^[A-Z][a-z]+s\b(?!\s+(?:on|of|for|about|in|to|from|with|at|by)\b)") +VERB_PROXY_STOPLIST = frozenset({"This", "Has", "Was", "Its", "Is", "Does", "Thus", "Plus"}) + +# A fence opens at up to 3 spaces of indent, or inside a list item at up to 3 spaces +# past the item's content column. Any deeper line is indented code, not a fence. A +# closer may be indented at most 3 spaces more than its opener. +FENCE_OPEN = re.compile(r"^( *)(`{3,}|~{3,})(.*)$") +FENCE_CLOSE = re.compile(r"^( *)(`{3,}|~{3,})[ \t]*$") +LIST_ITEM = re.compile(r"^( *)([-*+]|\d+[.)])(?: ( *)|$)") +# The destination may hold one level of balanced parentheses, e.g. "a(1).md". +INLINE_LINK = re.compile(r"\]\(((?:[^()]|\([^()]*\))*)\)") +REFERENCE_DEFINITION = re.compile(r"^ {0,3}\[(?!\^)[^\]]+\]:\s*(\S+)") +CODE_SPAN = re.compile(r"(`+)(?!`)(.+?)(? None: + self.findings.append(Finding(self.skill, rule, message)) + + def warn(self, rule: str, message: str) -> None: + self.findings.append(Finding(self.skill, rule, message, "warning")) + + +class BaselineError(ValueError): + pass + + +# ── Parsing ────────────────────────────────────────────────────────────────── + + +def count_lines(text: str) -> int: + """Same value as `wc -l`: newline characters, nothing else.""" + return text.count("\n") + + +def split_frontmatter(text: str) -> tuple[str, list[str], int]: + """Return (frontmatter, body lines, 1-based line number of the first body line).""" + if not text.startswith("---\n"): + raise ValueError("file does not start with a '---' frontmatter line") + lines = text.split("\n") + for index in range(1, len(lines)): + if lines[index] == "---": + return "\n".join(lines[1:index]), lines[index + 1 :], index + 2 + raise ValueError("frontmatter has no closing '---' line") + + +def _indent(line: str) -> int: + return len(line) - len(line.lstrip(" ")) + + +def _list_content_column(previous: list[str], indent: int) -> int | None: + """Content column of the list item a line at `indent` belongs to, else None. + + Walks up through ever smaller indents until a list marker (inside) or an + unindented line (outside) settles it. + """ + for line in reversed(previous): + if not line.strip() or _indent(line) >= indent: + continue + match = LIST_ITEM.match(line) + if match: + gap = len(match.group(3) or "") + 1 + return len(match.group(1)) + len(match.group(2)) + (gap if gap <= 4 else 1) + indent = _indent(line) + if indent == 0: + return None + return None + + +def _opens_fence_at(previous: list[str], indent: int) -> bool: + """A marker indented 4 or more past its container's content column is indented code.""" + if indent <= 3: + return True + column = _list_content_column(previous, indent) + return column is not None and indent <= column + 3 + + +def blank_fenced_lines(lines: list[str]) -> tuple[list[str], int | None]: + """Replace every line of a CommonMark fenced block (fences included) with ''. + + Also return the 0-based index of a fence still open at the end, else None. + """ + out = [] + # `out` with each fenced line kept as an indented placeholder, so the list walk + # still sees that a fence at a smaller indent ended the list above it. + shape = [] + fence = None + fence_indent = 0 + opened_at = None + for index, line in enumerate(lines): + if fence is None: + match = FENCE_OPEN.match(line) + if ( + match + and not (match.group(2)[0] == "`" and "`" in match.group(3)) + and _opens_fence_at(shape, len(match.group(1))) + ): + fence = match.group(2) + fence_indent = len(match.group(1)) + opened_at = index + out.append("") + shape.append(" " * fence_indent + "```") + else: + out.append(line) + shape.append(line) + else: + match = FENCE_CLOSE.match(line) + if ( + match + and len(match.group(1)) <= fence_indent + 3 + and match.group(2)[0] == fence[0] + and len(match.group(2)) >= len(fence) + ): + fence = None + out.append("") + shape.append(" " * fence_indent + "```") + return out, (opened_at if fence is not None else None) + + +def link_target(destination: str) -> str: + destination = destination.strip() + if destination.startswith("<"): + end = destination.find(">") + return destination[1:end] if end != -1 else destination[1:] + return destination.split(None, 1)[0] if destination else "" + + +def has_activation_condition(description: str) -> tuple[bool, bool]: + """Return (passes, passes_only_via_verb_proxy).""" + for pattern in (TRIGGER_CLAUSE, CONDITION_WORD, SCOPE_CLAUSE, MUST_USE): + if pattern.search(description): + return True, False + match = VERB_PROXY.match(description) + if match and match.group(0) not in VERB_PROXY_STOPLIST: + return True, True + return False, False + + +# ── Content checks (everything except the baselined sizes) ─────────────────── + + +def check_skill_content(skill_dir: Path) -> SkillReport: + """Measure one skill and run every rule that does not depend on the baseline.""" + report = SkillReport(skill_dir.name) + skill_md = skill_dir / "SKILL.md" + if not skill_md.is_file(): + report.fail("skill-md", f"{skill_md} is missing") + return report + raw = skill_md.read_bytes() + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as error: + report.fail("encoding", f"SKILL.md is not UTF-8 (byte offset {error.start})") + return report + if "\r" in text: + report.fail("encoding", "SKILL.md has CRLF line endings") + return report + try: + frontmatter, body, body_start = split_frontmatter(text) + meta = yaml.safe_load(frontmatter) + except (ValueError, yaml.YAMLError) as error: + report.fail("frontmatter", f"SKILL.md frontmatter does not parse: {error}") + return report + if not isinstance(meta, dict): + report.fail("frontmatter", "SKILL.md frontmatter is not a mapping") + return report + for key in ("name", "description"): + if not isinstance(meta.get(key), str): + report.fail("frontmatter", f"frontmatter '{key}' is missing or not a string") + if report.findings: + return report + + description = " ".join(meta["description"].split()) + report.metrics = { + "bytes": len(raw), + "lines": count_lines(text), + "description_chars": len(description), + } + _check_name(report, meta["name"]) + _check_description(report, description) + unfenced, open_fence = blank_fenced_lines(body) + if open_fence is not None: + # Fail closed: an unclosed fence would hide every later line from the checks below. + report.fail("fence", f"line {body_start + open_fence}: code fence opened here is never closed") + _check_links(report, skill_dir, unfenced, body_start) + _check_bare_paths(report, skill_dir, unfenced, body_start) + _check_references(report, skill_dir, text, unfenced, body_start) + return report + + +def _check_name(report: SkillReport, name: str) -> None: + if name != report.skill: + report.fail("name", f"frontmatter name '{name}' does not match directory '{report.skill}'") + + +def _check_description(report: SkillReport, description: str) -> None: + chars = len(description) + words = len(description.split()) + if chars < DESCRIPTION_MIN_CHARS: + report.fail("description", f"description is {chars} chars, minimum {DESCRIPTION_MIN_CHARS}") + if words < DESCRIPTION_MIN_WORDS: + report.fail("description", f"description is {words} words, minimum {DESCRIPTION_MIN_WORDS}") + passes, proxy_only = has_activation_condition(description) + if not passes: + report.fail( + "activation", + "description gives no activation condition: add a 'Use when ...' clause " + "or open with a behavior summary ('Validates ...')", + ) + elif proxy_only: + report.warn( + "activation", + "description passes only as a behavior summary (first-word verb); " + "check that it says when the skill applies", + ) + + +def _inside(path: Path, root: Path) -> bool: + try: + path.resolve().relative_to(root.resolve()) + return True + except ValueError: + return False + + +def _resolves_to_file(target: Path, root: Path) -> str | None: + """Return None when target is a regular file inside root, else the reason.""" + if not _inside(target, root): + return "escapes the skill directory" + current = target + while current != root and current != current.parent: + if current.is_symlink(): + return "is a symlink" + current = current.parent + if not target.is_file(): + return "is not an existing file" + return None + + +def blank_code_spans(line: str) -> str: + """Replace each inline code span with spaces of the same length.""" + return CODE_SPAN.sub(lambda match: " " * len(match.group(0)), line) + + +def _link_destinations(line: str) -> list[str]: + destinations = [match.group(1) for match in INLINE_LINK.finditer(line)] + definition = REFERENCE_DEFINITION.match(line) + if definition: + destinations.append(definition.group(1)) + return destinations + + +def _check_links(report: SkillReport, skill_dir: Path, unfenced: list[str], start: int) -> None: + for offset, line in enumerate(unfenced): + for destination in _link_destinations(blank_code_spans(line)): + target = link_target(destination) + if URL_SCHEME.match(target) or target.startswith("#"): + continue + path = target.split("#", 1)[0] + reason = _resolves_to_file(skill_dir / path, skill_dir) + if reason: + report.fail("link", f"line {start + offset}: link target '{target}' {reason}") + + +def _check_bare_paths(report: SkillReport, skill_dir: Path, unfenced: list[str], start: int) -> None: + for offset, line in enumerate(unfenced): + for match in BARE_PATH.finditer(line): + path = match.group(1).rstrip(".,;:)") + if not (skill_dir / path.split("/", 1)[0]).is_dir(): + continue + target = skill_dir / path + if not _inside(target, skill_dir) or not target.exists(): + report.fail("path", f"line {start + offset}: '{path}' does not exist in the skill directory") + + +def _check_references( + report: SkillReport, skill_dir: Path, text: str, unfenced: list[str], start: int +) -> None: + references = skill_dir / "references" + if not (references.exists() or references.is_symlink()): + return + if references.is_symlink() or not references.is_dir(): + report.fail("references", "references/ must be a real directory") + return + for entry in sorted(references.iterdir()): + rel = f"references/{entry.name}" + if entry.is_symlink(): + report.fail("references", f"{rel} is a symlink") + elif entry.is_dir(): + report.fail("references", f"{rel} is a subdirectory; references/ is one level deep") + elif not entry.is_file(): + report.fail("references", f"{rel} is not a regular file") + elif entry.name.lower() == "skill.md": + report.fail("references", f"{rel} would look like a second skill; rename it") + elif entry.suffix != ".md": + report.fail("references", f"{rel} is not Markdown (.md)") + else: + _check_reference_cue(report, rel, text, unfenced, start) + + +def _check_reference_cue(report: SkillReport, rel: str, text: str, unfenced: list[str], start: int) -> None: + if rel not in text: + report.fail("references", f"{rel} is never mentioned in SKILL.md (orphan)") + return + mentions = [(start + i, line) for i, line in enumerate(unfenced) if rel in line] + if any(READ_VERB.search(line) and READ_CONDITION.search(line) for _, line in mentions): + return + quoted = "; ".join(f"line {number}: {line.strip()}" for number, line in mentions) or "only inside fences" + report.fail( + "references", + f"{rel} has no read cue: one unfenced line naming it needs a read verb " + f"(read/open/consult/load) and a condition (when/before/after/if/...). Mentions: {quoted}", + ) + + +# ── Baseline ───────────────────────────────────────────────────────────────── + + +def validate_baseline(baseline: object) -> None: + """Raise BaselineError on any schema violation.""" + if not isinstance(baseline, dict): + raise BaselineError("top level is not an object") + expected = {"_comment", "skills", "thresholds", "tolerance"} + if set(baseline) != expected: + raise BaselineError(f"top-level keys must be exactly {sorted(expected)}, got {sorted(baseline)}") + if not isinstance(baseline["_comment"], str): + raise BaselineError("_comment is not a string") + for section in ("thresholds", "tolerance"): + values = baseline[section] + if not isinstance(values, dict) or set(values) != set(DIMENSIONS): + raise BaselineError(f"{section} must have exactly the keys {list(DIMENSIONS)}") + for dim, value in values.items(): + if not _positive_int(value): + raise BaselineError(f"{section}.{dim} is not a positive integer") + if not isinstance(baseline["skills"], dict): + raise BaselineError("skills is not an object") + for skill, entry in baseline["skills"].items(): + if not SKILL_NAME.match(skill): + raise BaselineError(f"skills key '{skill}' is not a skill name") + if not isinstance(entry, dict) or not entry or not set(entry) <= set(DIMENSIONS): + raise BaselineError(f"skills.{skill} must be a non-empty object over {list(DIMENSIONS)}") + for dim, value in entry.items(): + if not _positive_int(value): + raise BaselineError(f"skills.{skill}.{dim} is not a positive integer") + if value <= baseline["thresholds"][dim]: + raise BaselineError( + f"baseline entry is under threshold: skills.{skill}.{dim} = {value} " + f"<= {baseline['thresholds'][dim]}" + ) + + +def _positive_int(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + + +def load_baseline(path: Path | None = None) -> dict: + path = path or BASELINE_PATH + try: + baseline = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise BaselineError(f"cannot read {path}: {error}") from error + validate_baseline(baseline) + return baseline + + +def render_baseline(baseline: dict) -> str: + return json.dumps(baseline, indent=2, sort_keys=True) + "\n" + + +def missing_baseline_skills(baseline: dict, skills_dir: Path) -> list[str]: + return sorted(skill for skill in baseline["skills"] if not (skills_dir / skill).is_dir()) + + +def check_sizes(skill: str, metrics: dict, baseline: dict) -> list[Finding]: + """Apply the shrink-only pass rule to each baselined dimension, first match wins.""" + findings = [] + entry = baseline["skills"].get(skill, {}) + for dim in DIMENSIONS: + measured = metrics[dim] + threshold = baseline["thresholds"][dim] + tolerance = baseline["tolerance"][dim] + ceiling = entry.get(dim) + message = None + if ceiling is None: + if measured > threshold: + message = f"{dim} {measured} is over threshold {threshold} and not in baseline" + elif measured > ceiling: + message = f"{dim} {measured} grew past its baseline ceiling {ceiling}" + elif measured <= threshold: + message = f"{dim} {measured} is at or under threshold {threshold}: stale baseline, delete the entry" + elif measured <= ceiling - tolerance: + message = f"{dim} {measured} is well under ceiling {ceiling}: stale baseline, lower to {measured}" + if message: + findings.append(Finding(skill, "size", message)) + return findings + + +def skill_dirs(skills_dir: Path) -> list[Path]: + return sorted(path for path in skills_dir.iterdir() if path.is_dir()) + + +def build_baseline(skills_dir: Path, thresholds: dict, tolerance: dict) -> dict: + """Return a baseline holding the exact measured value of every over-threshold dimension.""" + skills = {} + for skill_dir in skill_dirs(skills_dir): + report = check_skill_content(skill_dir) + if report.metrics is None: + reasons = "; ".join(f.message for f in report.findings) + raise BaselineError(f"{skill_dir.name}: {reasons}") + entry = {d: report.metrics[d] for d in DIMENSIONS if report.metrics[d] > thresholds[d]} + if entry: + skills[skill_dir.name] = entry + return { + "_comment": BASELINE_COMMENT, + "skills": skills, + "thresholds": dict(thresholds), + "tolerance": dict(tolerance), + } + + +def check_ratchet(old: dict, new: dict) -> list[str]: + """Return one line per way NEW grew relative to OLD; empty when NEW only shrank.""" + violations = [] + for section in ("thresholds", "tolerance"): + for dim in DIMENSIONS: + if new[section][dim] > old[section][dim]: + violations.append(f"{section}.{dim} raised from {old[section][dim]} to {new[section][dim]}") + for skill, entry in sorted(new["skills"].items()): + old_entry = old["skills"].get(skill, {}) + for dim, value in sorted(entry.items()): + if dim not in old_entry: + threshold_cut = new["thresholds"][dim] < old["thresholds"][dim] + if not (threshold_cut and value <= old["thresholds"][dim]): + violations.append(f"skills.{skill}.{dim} = {value} added") + elif value > old_entry[dim]: + violations.append(f"skills.{skill}.{dim} raised from {old_entry[dim]} to {value}") + return violations + + +# ── CLI ────────────────────────────────────────────────────────────────────── + + +def _baseline_limits() -> tuple[dict, dict]: + try: + baseline = load_baseline() + return baseline["thresholds"], baseline["tolerance"] + except BaselineError: + return DEFAULT_THRESHOLDS, DEFAULT_TOLERANCE + + +def _scorecard() -> int: + thresholds, tolerance = _baseline_limits() + baselined = build_baseline(SKILLS_DIR, thresholds, tolerance)["skills"] + rows = [] + for skill_dir in skill_dirs(SKILLS_DIR): + metrics = check_skill_content(skill_dir).metrics or {d: 0 for d in DIMENSIONS} + rows.append((skill_dir.name, metrics, skill_dir.name in baselined)) + rows.sort(key=lambda row: row[1]["bytes"], reverse=True) + print(f"{'name':<40} {'lines':>6} {'bytes':>7} {'desc':>5} baselined") + for name, metrics, flagged in rows: + print( + f"{name:<40} {metrics['lines']:>6} {metrics['bytes']:>7} " + f"{metrics['description_chars']:>5} {'yes' if flagged else ''}" + ) + return 0 + + +def _print_baseline(allow_growth: bool) -> int: + committed = None + if BASELINE_PATH.exists(): + try: + committed = load_baseline() + except BaselineError as error: + # Redirecting straight onto the baseline truncates it before it is read, + # and falling back to the defaults would silently undo a lowered threshold. + print( + f"error: committed baseline is unreadable ({error}); write the output " + "to a temp file, then move it into place", + file=sys.stderr, + ) + return 1 + if committed is None: + thresholds, tolerance = DEFAULT_THRESHOLDS, DEFAULT_TOLERANCE + else: + thresholds, tolerance = committed["thresholds"], committed["tolerance"] + try: + built = build_baseline(SKILLS_DIR, thresholds, tolerance) + except (BaselineError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + violations = [] if allow_growth or committed is None else check_ratchet(committed, built) + if violations: + for line in violations: + print(f"ratchet: {line}", file=sys.stderr) + print( + "error: the new baseline would grow; slim the skill instead, or pass " + "--allow-growth if a threshold was lowered on purpose", + file=sys.stderr, + ) + return 1 + sys.stdout.write(render_baseline(built)) + return 0 + + +def _check_ratchet(old_path: Path) -> int: + try: + old = load_baseline(old_path) + except BaselineError as error: + print(f"error: old baseline {old_path}: {error}", file=sys.stderr) + return 1 + try: + new = load_baseline() + except BaselineError as error: + print(f"error: new baseline {BASELINE_PATH}: {error}", file=sys.stderr) + return 1 + violations = check_ratchet(old, new) + for line in violations: + print(f"ratchet: {line}", file=sys.stderr) + return 1 if violations else 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--scorecard", action="store_true", help="print per-skill sizes") + mode.add_argument("--print-baseline", action="store_true", help="print a baseline built from the tree") + mode.add_argument("--check-ratchet", metavar="OLD", type=Path, help="fail if the baseline grew vs OLD") + parser.add_argument( + "--allow-growth", + action="store_true", + help="with --print-baseline, print even when an entry would be raised or added", + ) + args = parser.parse_args(argv) + if args.allow_growth and not args.print_baseline: + parser.error("--allow-growth only applies to --print-baseline") + if args.scorecard: + return _scorecard() + if args.print_baseline: + return _print_baseline(args.allow_growth) + return _check_ratchet(args.check_ratchet) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_skill_quality.py b/tests/test_skill_quality.py new file mode 100644 index 00000000..bcd17db8 --- /dev/null +++ b/tests/test_skill_quality.py @@ -0,0 +1,569 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Static skill quality gate over skills/*/SKILL.md, plus fixture tests pinning each rule. + +The rules and the shrink-only baseline live in tests/skill_quality.py. +Run: make test-py (or: uvx --with pyyaml pytest -q tests/test_skill_quality.py) +""" +import copy +import importlib.util +import json +import subprocess +import sys +import warnings +from pathlib import Path + +import pytest + +_SPEC = importlib.util.spec_from_file_location( + "skill_quality", Path(__file__).resolve().parent / "skill_quality.py" +) +sq = importlib.util.module_from_spec(_SPEC) +sys.modules["skill_quality"] = sq +_SPEC.loader.exec_module(sq) + +try: + BASELINE = sq.load_baseline() + BASELINE_ERROR = None +except sq.BaselineError as error: + BASELINE = None + BASELINE_ERROR = str(error) + +CORPUS = sq.skill_dirs(sq.SKILLS_DIR) if sq.SKILLS_DIR.is_dir() else [] +GOOD_DESCRIPTION = "Use when designing a widget so the agent follows the widget rules in this repository." + + +def _errors(findings): + return [f for f in findings if f.severity == "error"] + + +def _assert_clean(findings): + errors = _errors(findings) + assert not errors, "\n".join(f"{f.skill} [{f.rule}] {f.message}" for f in errors) + + +# ── Corpus ─────────────────────────────────────────────────────────────────── + + +def test_skills_dir_exists(): + assert sq.SKILLS_DIR.is_dir(), f"skills/ not found at {sq.SKILLS_DIR}" + + +def test_baseline_is_well_formed(): + assert BASELINE_ERROR is None, f"baseline malformed: {BASELINE_ERROR}" + + +def test_baseline_is_canonically_formatted(): + if BASELINE is None: + pytest.skip("baseline malformed") + committed = sq.BASELINE_PATH.read_text(encoding="utf-8") + assert committed == sq.render_baseline(BASELINE), ( + "tests/skill-quality-baseline.json is not in canonical form; " + "regenerate with: python3 tests/skill_quality.py --print-baseline" + ) + + +def test_baseline_names_only_existing_skills(): + if BASELINE is None: + pytest.skip("baseline malformed") + missing = sq.missing_baseline_skills(BASELINE, sq.SKILLS_DIR) + assert not missing, f"baseline names a missing skill: {missing}" + + +@pytest.mark.parametrize("skill_dir", CORPUS, ids=lambda p: p.name) +def test_skill_content(skill_dir): + report = sq.check_skill_content(skill_dir) + for finding in report.findings: + if finding.severity == "warning": + warnings.warn(f"{finding.skill} [{finding.rule}] {finding.message}") + _assert_clean(report.findings) + + +@pytest.mark.parametrize("skill_dir", CORPUS, ids=lambda p: p.name) +def test_skill_size_within_baseline(skill_dir): + if BASELINE is None: + pytest.skip("baseline malformed") + report = sq.check_skill_content(skill_dir) + if report.metrics is None: + pytest.skip("SKILL.md did not parse; see test_skill_content") + _assert_clean(sq.check_sizes(skill_dir.name, report.metrics, BASELINE)) + + +# ── Fixtures ───────────────────────────────────────────────────────────────── + + +def _skill(tmp_path, name="widget", description=GOOD_DESCRIPTION, body="# Widget\n", frontmatter_name=None): + skill_dir = tmp_path / name + skill_dir.mkdir(parents=True, exist_ok=True) + text = f"---\nname: {frontmatter_name or name}\ndescription: {description}\n---\n{body}" + (skill_dir / "SKILL.md").write_text(text, encoding="utf-8") + return skill_dir + + +def _reference(skill_dir, name="guide.md", content="# Guide\n"): + references = skill_dir / "references" + references.mkdir(exist_ok=True) + (references / name).write_text(content, encoding="utf-8") + + +def _rules(skill_dir): + return {f.rule for f in _errors(sq.check_skill_content(skill_dir).findings)} + + +def _messages(skill_dir): + return " | ".join(f.message for f in _errors(sq.check_skill_content(skill_dir).findings)) + + +def _baseline(skills=None, thresholds=None, tolerance=None): + return { + "_comment": "fixture", + "skills": skills or {}, + "thresholds": thresholds or {"bytes": 1000, "description_chars": 512, "lines": 100}, + "tolerance": tolerance or {"bytes": 100, "description_chars": 50, "lines": 50}, + } + + +def _size_messages(metrics, baseline, skill="widget"): + full = {"bytes": 10, "description_chars": 80, "lines": 10} + full.update(metrics) + return " | ".join(f.message for f in sq.check_sizes(skill, full, baseline)) + + +def test_clean_fixture_passes(tmp_path): + assert _rules(_skill(tmp_path)) == set() + + +def test_name_mismatch_fails(tmp_path): + assert "name" in _rules(_skill(tmp_path, frontmatter_name="gadget")) + + +@pytest.mark.parametrize( + "description", + ["Use when short.", "Use when x " + "y" * 60], + ids=["too-short", "too-few-words"], +) +def test_description_minimums_fail(tmp_path, description): + assert "description" in _rules(_skill(tmp_path, description=description)) + + +def test_description_over_max_fails_without_entry(): + assert "not in baseline" in _size_messages({"description_chars": 600}, _baseline()) + + +def test_description_without_activation_condition_fails(tmp_path): + skill = _skill(tmp_path, description="DynamoDB skill covering tables, keys, indexes and capacity planning.") + assert "activation" in _rules(skill) + + +@pytest.mark.parametrize( + "description", + [ + "Reference to use for DynamoDB tables, keys and indexes in general.", + "DynamoDB patterns, and what to do if you are unsure about keys.", + ], + ids=["verb-mid-clause", "bare-if"], +) +def test_loose_activation_wording_fails(tmp_path, description): + assert "activation" in _rules(_skill(tmp_path, description=description)) + + +def test_read_verb_is_not_a_trigger(tmp_path): + skill = _skill(tmp_path, description="Notes on how to read DynamoDB tables to plan capacity units.") + findings = sq.check_skill_content(skill).findings + assert [f.rule for f in _errors(findings)] == ["activation"] + + +def test_trigger_verb_after_a_sentence_passes(tmp_path): + skill = _skill(tmp_path, description="Widget estimates for agent teams. Use to re-baseline a widget plan here.") + assert _rules(skill) == set() + + +def test_stoplist_first_word_is_not_a_behavior_summary(tmp_path): + skill = _skill(tmp_path, description="This skill covers DynamoDB tables, keys, indexes and capacity planning.") + assert "activation" in _rules(skill) + + +def test_verb_proxy_only_warns_and_passes(tmp_path): + skill = _skill(tmp_path, description="Validates DynamoDB tables, keys, indexes and capacity planning choices.") + findings = sq.check_skill_content(skill).findings + assert not _errors(findings) + assert [f.rule for f in findings if f.severity == "warning"] == ["activation"] + + +def test_scope_clause_passes(tmp_path): + skill = _skill(tmp_path, description="Loaded only in Claude Code sessions to set teammate behavior rules for agents.") + assert _rules(skill) == set() + + +def test_folded_description_parses(tmp_path): + skill = _skill(tmp_path, description=">-\n Use when designing a widget so the agent\n follows the widget rules here.") + report = sq.check_skill_content(skill) + assert not _errors(report.findings) + assert report.metrics["description_chars"] == len( + "Use when designing a widget so the agent follows the widget rules here." + ) + + +def test_quoted_description_with_colon_parses(tmp_path): + skill = _skill(tmp_path, description='"Use when: designing a widget so the agent follows the widget rules."') + assert _rules(skill) == set() + + +def test_crlf_fails(tmp_path): + skill = _skill(tmp_path) + path = skill / "SKILL.md" + path.write_bytes(path.read_bytes().replace(b"\n", b"\r\n")) + assert "encoding" in _rules(skill) + + +def test_non_utf8_fails(tmp_path): + skill = _skill(tmp_path) + (skill / "SKILL.md").write_bytes(b"---\nname: widget\n\xff\n---\n") + assert "encoding" in _rules(skill) + + +def test_missing_frontmatter_fails(tmp_path): + skill = tmp_path / "widget" + skill.mkdir() + (skill / "SKILL.md").write_text("# Widget\n", encoding="utf-8") + assert "frontmatter" in _rules(skill) + + +@pytest.mark.parametrize("text,expected", [("a\nb", 1), ("a\x0cb\n", 1), ("a\nb\n", 2)]) +def test_line_count_matches_wc_l(text, expected): + assert sq.count_lines(text) == expected + + +# Size ratchet: threshold 100 lines, tolerance 50. + + +def test_over_threshold_without_entry_fails(): + assert "not in baseline" in _size_messages({"lines": 101}, _baseline()) + + +def test_entry_equal_to_measured_passes(): + assert _size_messages({"lines": 300}, _baseline({"widget": {"lines": 300}})) == "" + + +def test_shrink_within_tolerance_passes(): + assert _size_messages({"lines": 260}, _baseline({"widget": {"lines": 300}})) == "" + + +def test_growth_past_entry_fails(): + assert "grew past" in _size_messages({"lines": 301}, _baseline({"widget": {"lines": 300}})) + + +def test_shrink_past_tolerance_asks_to_lower(): + assert "lower to 250" in _size_messages({"lines": 250}, _baseline({"widget": {"lines": 300}})) + + +def test_under_threshold_with_entry_asks_to_delete(): + assert "delete the entry" in _size_messages({"lines": 90}, _baseline({"widget": {"lines": 300}})) + + +def test_delete_beats_within_tolerance(): + assert "delete the entry" in _size_messages({"lines": 95}, _baseline({"widget": {"lines": 110}})) + + +# Links and bare paths. + + +def test_relative_link_to_missing_file_fails(tmp_path): + assert "link" in _rules(_skill(tmp_path, body="See [notes](notes.md).\n")) + + +def test_relative_link_to_existing_file_passes(tmp_path): + skill = _skill(tmp_path, body="See [notes](notes.md#top).\n") + (skill / "notes.md").write_text("x\n", encoding="utf-8") + assert _rules(skill) == set() + + +@pytest.mark.parametrize("link", ['[n](notes.md "Notes")', "[n]()"], ids=["title", "angle"]) +def test_link_title_and_angle_brackets_resolve_to_path(tmp_path, link): + skill = _skill(tmp_path, body=f"See {link}.\n") + (skill / "notes.md").write_text("x\n", encoding="utf-8") + assert _rules(skill) == set() + + +def test_link_escaping_skill_dir_fails(tmp_path): + (tmp_path / "outside.md").write_text("x\n", encoding="utf-8") + assert "escapes" in _messages(_skill(tmp_path, body="See [o](../outside.md).\n")) + + +def test_symlinked_link_target_fails(tmp_path): + skill = _skill(tmp_path, body="See [n](notes.md).\n") + (tmp_path / "real.md").write_text("x\n", encoding="utf-8") + (skill / "notes.md").symlink_to(tmp_path / "real.md") + assert "link" in _rules(skill) + + +def test_url_and_anchor_links_skipped(tmp_path): + assert _rules(_skill(tmp_path, body="[a](https://example.com/x) [b](#usage)\n")) == set() + + +@pytest.mark.parametrize("body", ["Run `[n](missing.md)` literally.\n", "Run ``a ` [n](missing.md)`` now.\n"]) +def test_link_inside_inline_code_ignored(tmp_path, body): + assert _rules(_skill(tmp_path, body=body)) == set() + + +def test_reference_definition_to_missing_file_fails(tmp_path): + assert "link" in _rules(_skill(tmp_path, body="See [n].\n\n[n]: missing.md\n")) + + +def test_reference_definition_to_existing_file_passes(tmp_path): + skill = _skill(tmp_path, body="See [n].\n\n[n]: \"Notes\"\n") + (skill / "notes.md").write_text("x\n", encoding="utf-8") + assert _rules(skill) == set() + + +def test_link_target_with_parentheses_is_read_whole(tmp_path): + skill = _skill(tmp_path, body="See [n](notes(1).md).\n") + assert "'notes(1).md'" in _messages(skill) + (skill / "notes(1).md").write_text("x\n", encoding="utf-8") + assert _rules(skill) == set() + + +def test_link_inside_fence_ignored(tmp_path): + assert _rules(_skill(tmp_path, body="```markdown\n[n](missing.md)\n```\n")) == set() + + +def test_link_inside_longer_tilde_fence_ignored(tmp_path): + body = "~~~~\n```\n[n](missing.md)\n~~~\n~~~~\n" + assert _rules(_skill(tmp_path, body=body)) == set() + + +def test_link_inside_list_nested_fence_ignored(tmp_path): + body = "1. Step\n\n ```bash\n echo [n](missing.md)\n ```\n\nafter\n" + assert _rules(_skill(tmp_path, body=body)) == set() + + +def test_top_level_indented_fence_marker_is_code_not_a_fence(tmp_path): + body = ( + "Example of a fence, shown as an indented code block:\n\n ```\n\n" + "Read [guide](missing.md) first.\n\n```bash\necho hi\n```\ntail\n" + ) + assert "link" in _rules(_skill(tmp_path, body=body)) + + +def test_fence_under_a_wrapped_list_item_is_a_fence(tmp_path): + body = ( + "- Step one,\n continued\n\n ```bash\n echo [n](missing.md)\n" + " ```\n\nafter\n" + ) + assert _rules(_skill(tmp_path, body=body)) == set() + + +def test_fence_marker_four_past_the_list_content_column_is_code(tmp_path): + body = ( + "- Step\n\n ```\n\n Read [g](missing.md)\n\n" + "```bash\nx\n```\ntail\n" + ) + assert "link" in _rules(_skill(tmp_path, body=body)) + + +def test_list_ended_by_a_top_level_fence_does_not_hold_a_later_marker(tmp_path): + body = ( + "- item\n\n```bash\necho\n```\n\n ```\n\nRead [g](missing.md)\n\n" + "```bash\nx\n```\ntail\n" + ) + assert "link" in _rules(_skill(tmp_path, body=body)) + + +def test_fence_closer_indented_past_the_opener_does_not_close(tmp_path): + body = "```bash\necho hi\n ```\n" + assert "fence" in _rules(_skill(tmp_path, body=body)) + + +def test_unclosed_fence_fails_closed(tmp_path): + body = "Intro.\n\n```bash\necho hi\n\nSee [n](missing.md).\n" + skill = _skill(tmp_path, body=body) + assert "fence" in _rules(skill) + assert "line 7: code fence opened here is never closed" in _messages(skill) + + +def test_bare_reference_path_without_directory_ignored(tmp_path): + body = "Its `references/architecture.md` file explains it.\n" + assert _rules(_skill(tmp_path, body=body)) == set() + + +def test_bare_path_to_missing_file_under_existing_dir_fails(tmp_path): + skill = _skill(tmp_path, body="Run `scripts/missing.sh` first.\n") + (skill / "scripts").mkdir() + assert "path" in _rules(skill) + + +# references/ shape, orphans, and read cues. + + +def test_orphan_reference_fails(tmp_path): + skill = _skill(tmp_path) + _reference(skill) + assert "orphan" in _messages(skill) + + +def test_reference_without_cue_fails(tmp_path): + skill = _skill(tmp_path, body="The file `references/guide.md` exists.\n") + _reference(skill) + assert "no read cue" in _messages(skill) + + +def test_one_cued_line_and_one_uncued_mention_passes(tmp_path): + body = "Read `references/guide.md` when scoring.\n\nAlso see `references/guide.md`.\n" + skill = _skill(tmp_path, body=body) + _reference(skill) + assert _rules(skill) == set() + + +def test_reference_mentioned_only_in_fence_fails_cue_not_orphan(tmp_path): + skill = _skill(tmp_path, body="```\nRead references/guide.md when scoring.\n```\n") + _reference(skill) + messages = _messages(skill) + assert "no read cue" in messages and "orphan" not in messages + + +@pytest.mark.parametrize( + "line", + ["Read `references/guide.md` before Stage 3.", "Open `references/guide.md` and apply it when scoring."], + ids=["read-before", "open-when"], +) +def test_cue_forms_pass(tmp_path, line): + skill = _skill(tmp_path, body=f"{line}\n") + _reference(skill) + assert _rules(skill) == set() + + +def test_see_for_details_is_not_a_cue(tmp_path): + skill = _skill(tmp_path, body="See `references/guide.md` for details.\n") + _reference(skill) + assert "no read cue" in _messages(skill) + + +def test_reference_subdirectory_fails(tmp_path): + skill = _skill(tmp_path) + (skill / "references" / "sub").mkdir(parents=True) + assert "subdirectory" in _messages(skill) + + +@pytest.mark.parametrize("name,expected", [("notes.txt", "not Markdown"), ("SKILL.md", "second skill")]) +def test_reference_bad_file_names_fail(tmp_path, name, expected): + skill = _skill(tmp_path, body=f"Read `references/{name}` when needed.\n") + _reference(skill, name=name) + assert expected in _messages(skill) + + +# Baseline file and ratchet. + + +def test_malformed_baseline_raises_one_error(): + with pytest.raises(sq.BaselineError): + sq.validate_baseline({"skills": {}}) + + +def test_baseline_entry_under_threshold_is_malformed(): + with pytest.raises(sq.BaselineError, match="under threshold"): + sq.validate_baseline(_baseline({"widget": {"lines": 90}})) + + +def test_baseline_comment_content_is_not_checked(): + sq.validate_baseline(_baseline({"widget": {"lines": 300}})) + + +def test_missing_baseline_skill_is_reported(tmp_path): + _skill(tmp_path) + baseline = _baseline({"widget": {"lines": 300}, "gone": {"lines": 300}}) + assert sq.missing_baseline_skills(baseline, tmp_path) == ["gone"] + + +def test_print_baseline_round_trips(): + result = subprocess.run( + [sys.executable, str(Path(sq.__file__)), "--print-baseline"], + capture_output=True, + text=True, + check=True, + ) + parsed = json.loads(result.stdout) + assert sq.render_baseline(parsed) == result.stdout + assert parsed["_comment"] == sq.BASELINE_COMMENT + + +def _grown_tree(tmp_path, monkeypatch): + """A tree whose 'big' skill is over the committed ceiling of 110 lines.""" + skills = tmp_path / "skills" + _skill(skills, name="big", body="x\n" * 120) + baseline_path = tmp_path / "baseline.json" + baseline_path.write_text(sq.render_baseline(_baseline({"big": {"lines": 110}})), encoding="utf-8") + monkeypatch.setattr(sq, "SKILLS_DIR", skills) + monkeypatch.setattr(sq, "BASELINE_PATH", baseline_path) + return baseline_path + + +def test_print_baseline_refuses_growth(tmp_path, monkeypatch, capsys): + _grown_tree(tmp_path, monkeypatch) + assert sq.main(["--print-baseline"]) == 1 + captured = capsys.readouterr() + assert captured.out == "" + assert "skills.big.lines raised from 110 to 124" in captured.err + + +def test_print_baseline_allow_growth_prints(tmp_path, monkeypatch, capsys): + _grown_tree(tmp_path, monkeypatch) + assert sq.main(["--print-baseline", "--allow-growth"]) == 0 + assert json.loads(capsys.readouterr().out)["skills"] == {"big": {"lines": 124}} + + +def test_print_baseline_refuses_a_truncated_baseline(tmp_path, monkeypatch, capsys): + _grown_tree(tmp_path, monkeypatch).write_text("", encoding="utf-8") + assert sq.main(["--print-baseline"]) == 1 + assert "temp file" in capsys.readouterr().err + + +def test_print_baseline_allow_growth_refuses_a_truncated_baseline(tmp_path, monkeypatch, capsys): + _grown_tree(tmp_path, monkeypatch).write_text("", encoding="utf-8") + assert sq.main(["--print-baseline", "--allow-growth"]) == 1 + captured = capsys.readouterr() + assert "temp file" in captured.err + assert captured.out == "" + + +def test_build_baseline_records_exact_over_threshold_values(tmp_path): + _skill(tmp_path, name="big", body="x\n" * 120) + _skill(tmp_path, name="small") + built = sq.build_baseline(tmp_path, _baseline()["thresholds"], _baseline()["tolerance"]) + assert built["skills"] == {"big": {"lines": 124}} + + +def _ratchet(old_skills, new_skills, new_thresholds=None, new_tolerance=None): + old = _baseline(old_skills) + new = _baseline(copy.deepcopy(new_skills), new_thresholds, new_tolerance) + return sq.check_ratchet(old, new) + + +def test_ratchet_rejects_raised_entry(): + assert _ratchet({"widget": {"lines": 300}}, {"widget": {"lines": 301}}) + + +def test_ratchet_rejects_added_entry(): + assert _ratchet({}, {"widget": {"lines": 300}}) + + +def test_ratchet_accepts_lowered_and_deleted_entries(): + old = {"widget": {"lines": 300, "bytes": 2000}, "gadget": {"lines": 200}} + assert _ratchet(old, {"widget": {"lines": 250}}) == [] + + +def test_ratchet_accepts_entry_created_by_lowered_threshold(): + thresholds = {"bytes": 1000, "description_chars": 512, "lines": 80} + assert _ratchet({}, {"widget": {"lines": 90}}, new_thresholds=thresholds) == [] + + +def test_ratchet_rejects_raised_threshold_or_tolerance(): + assert _ratchet({}, {}, new_thresholds={"bytes": 2000, "description_chars": 512, "lines": 100}) + assert _ratchet({}, {}, new_tolerance={"bytes": 100, "description_chars": 50, "lines": 60}) + + +def test_check_ratchet_cli_passes_against_itself(): + result = subprocess.run( + [sys.executable, str(Path(sq.__file__)), "--check-ratchet", str(sq.BASELINE_PATH)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr