diff --git a/.github/workflows/llms_txt_snippet_sync.yml b/.github/workflows/llms_txt_snippet_sync.yml new file mode 100644 index 00000000..d64b54b6 --- /dev/null +++ b/.github/workflows/llms_txt_snippet_sync.yml @@ -0,0 +1,57 @@ +name: llms.txt Snippet Sync + +# llms.txt is hand-maintained in the weaviate-io repo, so a snippet change here can +# strand a block in the published file. Only the weekly llms_txt_tests.yml job notices, +# which means the break surfaces days later. This gives the author the signal at PR time. +# +# It is advisory by design and never fails: when a snippet PR is opened, weaviate-io has +# not merged or deployed yet, so the live llms.txt legitimately cannot match. A blocking +# check would fire on every honest PR and would just be overridden. + +permissions: + contents: read + +on: + pull_request: + paths: + # Kept in step with SNIPPET_GLOBS in tests/test_llms_txt_code.py. Java and C# + # snippets live with their language suites, not under _includes/code/llms-txt/. + - "_includes/code/llms-txt/**" + - "_includes/code/java-v6/src/test/java/LlmsTxtTest.java" + - "_includes/code/csharp/LlmsTxtTest.cs" + - "tests/test_llms_txt_code.py" + - "tests/check_llms_txt_drift.py" + +env: + PYTHON_VERSION: "3.11" + +jobs: + check-snippet-sync: + name: Check llms.txt snippet sync + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Fetch the PR base commit + # Shallow single-commit fetch: the check only needs the base tree to read the + # pre-change snippet files. If it fails the check reports a degraded warning + # rather than blocking. + continue-on-error: true + run: git fetch --no-tags --depth=1 origin ${{ github.event.pull_request.base.sha }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install pytest + # The check imports tests/test_llms_txt_code.py to reuse its matching logic, and + # that module imports pytest. Nothing else from the test suite is needed, so this + # stays far cheaper than the full setup-test-env composite. + run: python -m pip install --quiet "pytest>=8.3.5" + + - name: Check llms.txt snippet sync + run: python tests/check_llms_txt_drift.py --base "${{ github.event.pull_request.base.sha }}" diff --git a/_includes/code/howto/search.filters.py b/_includes/code/howto/search.filters.py index 10e924e2..ec249425 100644 --- a/_includes/code/howto/search.filters.py +++ b/_includes/code/howto/search.filters.py @@ -538,9 +538,8 @@ filters=Filter.by_property("country").is_none(True) # Find objects where the `country` property is null # highlight-end ) -print("despot. othing") for o in response.objects: - print("despot"+o.properties) # Inspect returned objects + print(o.properties) # Inspect returned objects # END FilterByPropertyNullState @@ -573,6 +572,24 @@ }, ) +# Test scaffolding below; not rendered in the docs. +# This block used to run against a local instance, where a write is queryable +# the moment `insert` returns. It now runs against a remote cluster, where it is +# not, so the example could query the collection before the object was visible +# and get back nothing. Wait (bounded) for the write to land, so the assertion +# tests the geo filter and not write visibility. The assertion stays exact. +import time +from weaviate.classes.query import Filter, GeoCoordinate + +readiness_filter = Filter.by_property("headquartersGeoLocation").within_geo_range( + coordinate=GeoCoordinate(latitude=52.39, longitude=4.84), + distance=1000 +) +for _ in range(30): + if len(publications.query.fetch_objects(filters=readiness_filter).objects) >= 1: + break + time.sleep(1) + # START FilterbyGeolocation from weaviate.classes.query import Filter from weaviate.classes.query import GeoCoordinate diff --git a/pyproject.toml b/pyproject.toml index 1f5796bf..657b44ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "playwright>=1.40.0", "pytest>=8.3.5", "python-dotenv>=1.1.1", + "pyyaml>=6.0", "requests>=2.32.3", "tqdm>=4.67.1", "weaviate-agents>=1.7.0", diff --git a/src/scripts/scarf.js b/src/scripts/scarf.js index 6e0d0384..eab4276a 100644 --- a/src/scripts/scarf.js +++ b/src/scripts/scarf.js @@ -5,6 +5,12 @@ const scarfScript = { src: "https://pixel.weaviate.cloud/a.png?x-pxid=a41b0758-a3a9-4874-a880-8b5d5a363d40", referrerPolicy: "no-referrer-when-downgrade", style: "display: none;", + // Decorative, hidden analytics beacon: aria-hidden is what keeps it out of + // the accessibility tree, so assistive technology never reaches it and never + // has to announce it. Do not add `alt: ""` here: Docusaurus validates + // headTags attributes with a Joi `string()` schema that rejects the empty + // string, so an empty alt fails the site build outright. + "aria-hidden": "true", }, }; diff --git a/tests/README-INDEXABILITY.md b/tests/README-INDEXABILITY.md index d0b77b37..ce68a3ce 100644 --- a/tests/README-INDEXABILITY.md +++ b/tests/README-INDEXABILITY.md @@ -21,7 +21,7 @@ Documentation content that is hidden behind JavaScript interactions (tabs, colla ### HTML structure tests (Part 1) -These tests fetch ~11 representative pages from all doc sections and check: +These tests fetch ~13 representative pages from all doc sections and check: | Test | What it checks | |------|---------------| @@ -81,7 +81,7 @@ HTML structure tests always run. Agent tests only run if `ANTHROPIC_API_KEY` and ## Test pages -The suite tests 11 representative URLs covering all doc sections: +The suite tests 13 representative URLs covering all doc sections: | Page | Features tested | |------|----------------| @@ -91,9 +91,11 @@ The suite tests 11 representative URLs covering all doc sections: | `/weaviate/search/hybrid` | tabs, code | | `/weaviate/connections/connect-cloud` | tabs, code | | `/weaviate/config-refs/collections` | details, table | -| `/weaviate/concepts/data-import` | images | -| `/cloud/quickstart` | code, images | -| `/cloud/manage-clusters/create` | images | +| `/weaviate/concepts/data-import` | no structural features (200, meta tags, headings, LLM notice only) | +| `/cloud/quickstart` | code | +| `/cloud/manage-clusters/create` | no structural features (200, meta tags, headings, LLM notice only) | +| `/cloud/tools/query-tool` | images | +| `/weaviate/manage-collections/tenant-states` | images | | `/query-agent/recipes/query-agent-ecommerce-assistant` | code | | `/weaviate/search` | landing page | @@ -129,4 +131,21 @@ TEST_PAGES = [ ] ``` -Available feature tags: `tabs`, `code`, `details`, `images`, `table`. Pages are parametrized — each feature tag enables the corresponding structural test for that page. +Available feature tags: `tabs`, `code`, `details`, `images`, `table`. Pages are parametrized — each feature tag enables the corresponding structural test for that page. Tag only what the page actually has: a page tagged `images` with no content image fails rather than passing quietly, which is what keeps the tags from going stale. + +### Landing pages + +If the page you are adding routes readers onward instead of carrying content of its own — a hub page that is essentially a list of links to its children — also add its path to the `LANDING_PAGES` set in the same file: + +```python +LANDING_PAGES = {"/weaviate/search"} +``` + +`LANDING_PAGES` is the **only** thing that exempts a page from the "content pages have h2 headings" assertion in `test_heading_hierarchy`. An empty feature set does not exempt it. The two mean different things: + +- an empty feature set says the page has none of the structural features listed above (no tabs, no code, no details, no table, no images); +- `LANDING_PAGES` says the page owes the reader no h2 headings at all. + +A page can be plain prose with an empty feature set and still be a content page that must have h2s, which is why the exemption is tracked separately. + +So if you add a landing page with `set()` and leave it out of `LANDING_PAGES`, it fails with `content page has no h2 headings` — a confusing failure, because the feature set already looks like it says "this page has nothing". Add the path to `LANDING_PAGES` instead. diff --git a/tests/README-LLMS-TXT.md b/tests/README-LLMS-TXT.md index b465a0c4..0b22bf4a 100644 --- a/tests/README-LLMS-TXT.md +++ b/tests/README-LLMS-TXT.md @@ -116,6 +116,8 @@ no Weaviate cluster. 2. Run that language's `test_llms_txt` and confirm it passes — **never** hand-write a snippet without running it. 3. Copy the verified marked region **verbatim** into `weaviate-io/static/llms.txt`. + The PR-time sync warning (see *CI* below) flags this for you and prints the exact + block to paste. 4. New file? Add its path to the `test_llms_txt` parametrize list in the matching `tests/test_*.py`. @@ -182,12 +184,13 @@ after the matching `weaviate-io` change is live; otherwise mark with ## CI -Two separate workflows cover this directory: +Three separate workflows cover this directory: | Workflow | What it runs | |---|---| | `.github/workflows/docs_tests.yml` | Per-language **execution** tests — `test_llms_txt*` in `test_python.py`, `test_typescript.py`, `test_java.py`, `test_csharp.py`. Rides the existing `pyv4` / `ts` / `java` / `csharp` / `agents` markers, so no separate job for these. | | `.github/workflows/llms_txt_tests.yml` | The three **guard tests** in `test_llms_txt_code.py` — snippet coverage, version freshness, link validity. Single job `test-llms-txt`, runs `uv run pytest tests/test_llms_txt_code.py -m "llms_txt"`. | +| `.github/workflows/llms_txt_snippet_sync.yml` | The **PR-time warning** (`check_llms_txt_drift.py`). Advisory only, see below. | The guard workflow triggers on: @@ -204,6 +207,41 @@ Results post to Slack via the shared `./.github/actions/handle-test-results` composite under `test-type: 'llms.txt'`, with `continue-on-error: true` on the pytest step so the notification still fires when a guard fails. +### PR-time snippet sync warning + +The guard workflow runs weekly, so a snippet change here can break the published +`llms.txt` days before anyone notices. `tests/check_llms_txt_drift.py` closes that +gap on the docs side. `.github/workflows/llms_txt_snippet_sync.yml` runs it on every +PR touching a file that matches `SNIPPET_GLOBS`, and it answers one question: once +this PR merges, which `llms.txt` code blocks would `test_llms_txt_snippets_are_covered` +no longer find? Those, and only those, are the blocks `weaviate-io/static/llms.txt` +has to update in lockstep. + +It reuses this directory's matching logic (`SNIPPET_GLOBS`, the marker and fence +regexes, `_normalize`, `_load_llms_txt`), so it cannot disagree with the weekly job +about what "matches" means. Findings surface as GitHub warning annotations on the +changed snippet lines, plus a job summary carrying the new block to paste into +`llms.txt`. + +**It is advisory and never fails the job once it runs.** When a snippet PR is opened, weaviate-io has +not merged or deployed yet, so the live `llms.txt` legitimately cannot match yet; a +blocking check would fire on every honest PR and would just get overridden. It also +stays quiet whenever there is nothing to do: weaviate-io shipped first and `llms.txt` +already carries the new block, only scaffolding outside the markers changed, or a +region was moved or renamed without its code changing. + +Run it locally against uncommitted edits: + +```bash +uv run python tests/check_llms_txt_drift.py --base HEAD +``` + +The reverse direction (an `llms.txt` edit in weaviate-io that no longer matches these +snippets) is not automated. There is no check in weaviate-io; this repo's PR-time warning +and the weekly `llms_txt_tests.yml` job are the only automation. An `llms.txt` edit made +directly in weaviate-io that breaks the verbatim match is not caught until the weekly job +runs. + ## Per-language gotchas - **Python** — `text2vec_ollama` / `generative_ollama` take `api_endpoint` and diff --git a/tests/check_llms_txt_drift.py b/tests/check_llms_txt_drift.py new file mode 100644 index 00000000..0f32277d --- /dev/null +++ b/tests/check_llms_txt_drift.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Advisory PR check: warn when a snippet change strands a block in llms.txt. + +`llms.txt` is hand-maintained in the **weaviate-io** repo (`static/llms.txt`), but its +code blocks must appear verbatim between `START`/`END` markers in this repo's snippet +files. The only thing enforcing that today is the weekly +`test_llms_txt_code.py::test_llms_txt_snippets_are_covered` job, so a snippet change +here can break the published `llms.txt` days before anyone notices. + +This script answers one question at PR time: once this PR merges, which `llms.txt` code +blocks would that weekly coverage test no longer find? Those blocks, and only those, are +the ones `weaviate-io/static/llms.txt` has to update in lockstep. + +It is deliberately advisory: **once it runs it never fails the job**, because every +reporting path exits 0. (Only a malformed invocation, such as omitting `--base`, exits +non-zero, and that comes from argparse before any checking happens.) A legitimate snippet +PR cannot have a matching live `llms.txt` yet, because weaviate-io has not merged or +deployed, so a blocking check here would fail every honest PR and get routinely +overridden. Findings are reported as GitHub warning annotations plus a job summary +instead. + +Usage: + + python tests/check_llms_txt_drift.py --base + +The pre-change state is read with `git show :`; the post-change state is read +from the working tree, so the changed-file set is `git diff --name-only `. That also +makes the script easy to exercise locally: edit a snippet file and run it with +`--base HEAD`. + +Honors `LLMS_TXT_PATH` exactly like the guard tests do, so it can be pointed at a local +weaviate-io checkout instead of the live https://weaviate.io/llms.txt. +""" +import argparse +import os +import subprocess +import sys +from dataclasses import dataclass +from fnmatch import fnmatch +from typing import Optional + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +# Reuse the guard test's matching logic (SNIPPET_GLOBS, marker/fence regexes, +# normalization, llms.txt loading) so this check and the weekly job can never +# disagree about what "matches" means. +import test_llms_txt_code as guard # noqa: E402 + +LLMS_TXT_SOURCE = "weaviate-io/static/llms.txt" + +# guard._load_llms_txt() calls pytest.skip() when the fetch fails, and Skipped derives +# from BaseException, so a bare `except Exception` would let it through. +_LOAD_FAILURES = (Exception, guard.pytest.skip.Exception) + + +@dataclass +class Finding: + """A marked region that exists at the base ref but not after the change.""" + + path: str + language: str + marker: str + old_code: str + new_code: Optional[str] # same marker's code after the change, if it still exists + new_line: Optional[int] # 1-based line of the surviving `START` marker + + +def _git(*args): + return subprocess.run(["git", *args], capture_output=True, text=True, check=False) + + +def _repo_root(): + result = _git("rev-parse", "--show-toplevel") + return result.stdout.strip() if result.returncode == 0 else None + + +def _snippet_patterns(): + return [(lang, pattern) for lang, patterns in guard.SNIPPET_GLOBS.items() for pattern in patterns] + + +def _language_for(path): + for language, pattern in _snippet_patterns(): + if fnmatch(path, pattern): + return language + return None + + +def _changed_snippet_files(base): + """Paths matching SNIPPET_GLOBS that differ between `base` and the working tree.""" + result = _git("diff", "--name-only", base) + if result.returncode != 0: + return None, result.stderr.strip() or f"git diff against {base} failed" + changed = [path for path in result.stdout.splitlines() if _language_for(path)] + return sorted(changed), None + + +def _regions(text): + """[(marker, normalized code, 1-based START line)] for each region in `text`.""" + found = [] + for match in guard.MARKER_RE.finditer(text): + line = text.count("\n", 0, match.start()) + 1 + found.append((match.group(1), guard._normalize(match.group(2)), line)) + return found + + +def _blob_at(ref, path): + """File contents at `ref`, or None if the file did not exist there.""" + result = _git("show", f"{ref}:{path}") + return result.stdout if result.returncode == 0 else None + + +def _working_tree_text(path): + if not os.path.exists(path): + return "" # file deleted by the change + with open(path, encoding="utf-8") as handle: + return handle.read() + + +def _lost_regions(base, changed_paths): + """Regions present at `base` whose exact code no longer exists anywhere in the repo. + + Comparing against every snippet file (not just the changed one) means a region that + was merely moved or renamed is correctly treated as still covered. + """ + surviving = guard._collect_marked_regions() + lost = [] + for path in changed_paths: + language = _language_for(path) + base_text = _blob_at(base, path) + if base_text is None: + continue # new file: nothing can be stranded + head_regions = {marker: (code, line) for marker, code, line in _regions(_working_tree_text(path))} + for marker, code, _ in _regions(base_text): + if code in surviving[language]: + continue + new_code, new_line = head_regions.get(marker, (None, None)) + lost.append(Finding(path, language, marker, code, new_code, new_line)) + return lost + + +def _llms_txt_blocks(): + """({language: {normalized code}}, None), or (None, reason) if llms.txt is unreadable.""" + try: + content = guard._load_llms_txt() + except _LOAD_FAILURES as exc: + return None, str(exc) + blocks = {language: set() for language in guard.SNIPPET_GLOBS} + for match in guard.FENCE_RE.finditer(content): + language = guard.LANG_ALIASES.get(match.group(1).lower()) + if language is not None: + blocks[language].add(guard._normalize(match.group(2))) + return blocks, None + + +def _escape(value, is_property=False): + escaped = value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + if is_property: + escaped = escaped.replace(":", "%3A").replace(",", "%2C") + return escaped + + +def _annotate(level, message, path=None, line=None, title=None): + properties = [] + if path: + properties.append(f"file={_escape(path, True)}") + if line: + properties.append(f"line={line}") + if title: + properties.append(f"title={_escape(title, True)}") + joined = "," + ",".join(properties) if properties else "" + print(f"::{level}{joined}::{_escape(message)}") + + +def _write_summary(markdown): + print(markdown) + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + with open(summary_path, "a", encoding="utf-8") as handle: + handle.write(markdown + "\n") + + +def _report_in_sync(headline): + _write_summary(f"## llms.txt snippet sync\n\n{headline}\n") + + +def _report_drift(findings): + for finding in findings: + if finding.new_code is None: + detail = f"The `{finding.marker}` region was removed or renamed" + else: + detail = f"The `{finding.marker}` region changed" + _annotate( + "warning", + f"{detail}, but {LLMS_TXT_SOURCE} still publishes the previous version. " + f"Update {LLMS_TXT_SOURCE} in lockstep with this PR, or the weekly llms.txt " + f"coverage job will fail once this merges.", + path=finding.path, + line=finding.new_line, + title="llms.txt needs a matching update", + ) + + rows = "\n".join( + f"| `{finding.path}` | `{finding.marker}` | " + f"{'removed or renamed' if finding.new_code is None else 'changed'} |" + for finding in findings + ) + blocks = "\n\n".join( + f"
{finding.marker} " + f"({'no replacement in this repo' if finding.new_code is None else 'new block to copy into llms.txt'})" + f"\n\n```{finding.language}\n" + f"{finding.new_code if finding.new_code is not None else finding.old_code}\n```\n\n
" + for finding in findings + ) + plural = "block" if len(findings) == 1 else "blocks" + _write_summary( + "## llms.txt snippet sync\n\n" + f"This PR strands {len(findings)} code {plural} that `{LLMS_TXT_SOURCE}` publishes " + "verbatim. **`weaviate-io/static/llms.txt` must be updated in the same window**, " + "otherwise the weekly llms.txt coverage job starts failing once this merges.\n\n" + "| Snippet file | Marked region | What happened |\n" + "|---|---|---|\n" + f"{rows}\n\n" + f"{blocks}\n\n" + "This check never blocks the merge: weaviate-io cannot have shipped yet when a " + "legitimate snippet PR is opened.\n" + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Warn when a PR strands an llms.txt code block.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--base", + required=True, + help="git ref holding the pre-change state (on a PR, the base commit)", + ) + args = parser.parse_args() + + root = _repo_root() + if root is None: + _annotate("warning", "Not inside a git repository; skipping the llms.txt sync check.") + return 0 + os.chdir(root) + + changed, error = _changed_snippet_files(args.base) + if error is not None: + _annotate( + "warning", + f"Could not diff against {args.base} ({error}), so the llms.txt sync check was " + f"skipped. If this PR changes an llms.txt snippet, update {LLMS_TXT_SOURCE} too.", + title="llms.txt sync check could not run", + ) + return 0 + + if not changed: + _report_in_sync("No llms.txt-backed snippet files changed. Nothing to keep in sync.") + return 0 + + findings = _lost_regions(args.base, changed) + if not findings: + _report_in_sync( + f"{len(changed)} llms.txt snippet file(s) changed, but no `START`/`END` region " + f"lost its previous content, so `{LLMS_TXT_SOURCE}` stays valid." + ) + return 0 + + blocks, load_error = _llms_txt_blocks() + if blocks is None: + listed = ", ".join(f"`{path}`" for path in changed) + _annotate( + "warning", + f"Changed llms.txt snippet regions, but the published llms.txt could not be read " + f"({load_error}), so the comparison was skipped. Check {LLMS_TXT_SOURCE} by hand.", + title="llms.txt sync check could not run", + ) + _write_summary( + "## llms.txt snippet sync\n\n" + f"Could not read the published llms.txt ({load_error}). These files changed a " + f"marked region and may need a matching `{LLMS_TXT_SOURCE}` update: {listed}\n" + ) + return 0 + + stranded = [finding for finding in findings if finding.old_code in blocks[finding.language]] + if not stranded: + shipped = [ + finding + for finding in findings + if finding.new_code is not None and finding.new_code in blocks[finding.language] + ] + if shipped: + markers = ", ".join(f"`{finding.marker}`" for finding in shipped) + _report_in_sync( + f"`{LLMS_TXT_SOURCE}` already publishes the updated {markers} block(s). " + "weaviate-io shipped first, so there is nothing to do." + ) + else: + _report_in_sync( + f"Marked regions changed, but `{LLMS_TXT_SOURCE}` does not publish any of " + "their previous content, so nothing is stranded." + ) + return 0 + + _report_drift(stranded) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_docs_indexability.py b/tests/test_docs_indexability.py index 56b3af6c..ff5ad51d 100644 --- a/tests/test_docs_indexability.py +++ b/tests/test_docs_indexability.py @@ -31,15 +31,26 @@ ("/weaviate/search/hybrid", {"tabs", "code"}), ("/weaviate/connections/connect-cloud", {"tabs", "code"}), ("/weaviate/config-refs/collections", {"details", "table"}), - ("/weaviate/concepts/data-import", {"images"}), - ("/cloud/quickstart", {"code", "images"}), - ("/cloud/manage-clusters/create", {"images"}), + ("/weaviate/concepts/data-import", set()), + ("/cloud/quickstart", {"code"}), + ("/cloud/manage-clusters/create", set()), + ("/cloud/tools/query-tool", {"images"}), + ("/weaviate/manage-collections/tenant-states", {"images"}), ("/query-agent/recipes/query-agent-ecommerce-assistant", {"code"}), - ("/weaviate/search", set()), # landing page + ("/weaviate/search", set()), ] ALL_PATHS = [path for path, _ in TEST_PAGES] +# Pages that route readers onward instead of carrying content of their own, and +# so are exempt from the "content pages have h2 headings" rule. +# +# This is tracked separately from the feature sets because an empty feature set +# says only that a page has none of the structural features the checks below +# assert on. A page can be plain prose, with no tabs, code, details, table or +# image, and still be a content page that owes the reader h2 headings. +LANDING_PAGES = {"/weaviate/search"} + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -71,13 +82,6 @@ def _get_soup(path: str) -> BeautifulSoup: return BeautifulSoup(resp.text, "html.parser") -def _features_for(path: str) -> set[str]: - for p, features in TEST_PAGES: - if p == path: - return features - return set() - - # --------------------------------------------------------------------------- # Part 1: HTML Structure Tests (no API keys needed) # --------------------------------------------------------------------------- @@ -120,8 +124,7 @@ def test_heading_hierarchy(path): assert len(h1s) == 1, f"{path}: expected 1 h1, found {len(h1s)}" # Content pages (not landing pages) should have h2s - features = _features_for(path) - if features: # non-empty features means it's a content page + if path not in LANDING_PAGES: h2s = soup.find_all("h2") assert len(h2s) > 0, f"{path}: content page has no h2 headings" @@ -214,17 +217,16 @@ def test_details_content_present(path): ids=[p for p, f in TEST_PAGES if "images" in f], ) def test_images_have_alt_text(path): - """Content images have alt text (excludes SVG icons and badges).""" + """The page's own images have alt text (excludes SVG icons and badges).""" soup = _get_soup(path) + content_images = _content_images(soup) - # Find content images, excluding decorative ones - images = soup.find_all("img") - content_images = [ - img for img in images - if not _is_decorative_image(img) - ] - - assert len(content_images) > 0, f"{path}: no content images found" + if not content_images: + pytest.fail( + f"{path}: labelled 'images' but has no content image in the main " + "content region. Either the page lost its images, or the 'images' " + "label in TEST_PAGES is wrong." + ) missing_alt = [ img.get("src", "unknown") @@ -236,6 +238,26 @@ def test_images_have_alt_text(path): ) +def _main_content(soup): + """The page's main content region. + + Global navbar and footer chrome lives outside it. Scoping to this region + keeps site furniture (the site logo, the Ask AI button logo) from standing + in for the page's own images: that chrome is identical on every page, so + counting it made the alt-text check assert on the layout instead of the + page, and a single navbar change could flip it. + """ + return soup.find("main") or soup.find("article") or soup + + +def _content_images(soup): + """Images that carry the page's own content.""" + return [ + img for img in _main_content(soup).find_all("img") + if not _is_decorative_image(img) + ] + + def _is_decorative_image(img) -> bool: """Check if an image is decorative (SVG icon, badge, etc.).""" src = img.get("src", "") @@ -251,7 +273,17 @@ def _is_decorative_image(img) -> bool: # Skip language/site logo SVGs (e.g., /img/site/logo-py.svg) if src.endswith(".svg") and "/img/site/" in src: return True - # Skip analytics tracking pixels (e.g., Scarf) + # Skip images that carry no content by construction: analytics beacons, + # spacers, and anything hidden from both rendering and the accessibility + # tree. Such an image conveys nothing to a reader, a screen reader, or a + # crawler, so requiring alt text on it is meaningless. This is matched + # structurally rather than by hostname on purpose: the previous + # "static.scarf.sh" check went stale the moment the beacon moved to our own + # CNAME, and any hostname list will go stale again on the next move. + if _is_hidden_non_content(img): + return True + # Retained for the Scarf-hosted pixel, in case it is ever embedded without + # the hidden styling that the structural check above relies on. if "static.scarf.sh" in src: return True # Skip very small images (likely icons) @@ -262,6 +294,29 @@ def _is_decorative_image(img) -> bool: return False +def _is_hidden_non_content(img) -> bool: + """Check if an image is hidden from users and from the accessibility tree. + + Covers the ways a non-content image announces itself: + - inline `display: none` / `visibility: hidden` (removed from the render + tree, and therefore from the accessibility tree) + - `aria-hidden="true"` (explicitly withheld from assistive technology) + - 1x1 dimensions (a tracking pixel or a spacer) + """ + style = (img.get("style") or "").lower().replace(" ", "") + if "display:none" in style or "visibility:hidden" in style: + return True + + if str(img.get("aria-hidden", "")).lower() == "true": + return True + + dims = (img.get("width"), img.get("height")) + if all(d is not None and str(d).strip().isdigit() and int(d) <= 1 for d in dims): + return True + + return False + + @pytest.mark.indexability @pytest.mark.parametrize("path", ALL_PATHS) def test_llm_notice_present(path): diff --git a/tests/test_llms_txt_code.py b/tests/test_llms_txt_code.py index e89ca3c3..d0df9d7f 100644 --- a/tests/test_llms_txt_code.py +++ b/tests/test_llms_txt_code.py @@ -120,6 +120,62 @@ def test_llms_txt_snippets_are_covered(): ) +# The PR-time drift check only runs when the PR touches a path in this workflow's +# `paths:` filter, so a SNIPPET_GLOBS entry outside that filter disables the check +# for that language without any visible failure. Java and C# snippets already live +# outside `_includes/code/llms-txt/`, so a fifth language landing outside it is a +# realistic way to lose coverage silently. +SNIPPET_SYNC_WORKFLOW = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + ".github", "workflows", "llms_txt_snippet_sync.yml", +) + + +def _glob_to_regex(pattern): + """Compile a GitHub `paths:` filter, where `**` spans directories and `*` does not.""" + out, i = [], 0 + while i < len(pattern): + if pattern.startswith("**", i): + out.append(".*") + i += 2 + elif pattern[i] == "*": + out.append("[^/]*") + i += 1 + else: + out.append(re.escape(pattern[i])) + i += 1 + return re.compile("".join(out) + r"\Z") + + +@pytest.mark.llms_txt +def test_snippet_globs_are_covered_by_workflow_paths(): + """Every SNIPPET_GLOBS pattern must trigger the snippet-sync workflow.""" + # Imported here, not at module scope: check_llms_txt_drift.py imports this + # module in a workflow that installs pytest and nothing else. + import yaml + + with open(SNIPPET_SYNC_WORKFLOW, encoding="utf-8") as handle: + workflow = yaml.safe_load(handle) + + # PyYAML reads the bare `on:` key as the YAML 1.1 boolean True. + triggers = workflow.get("on", workflow.get(True)) + filters = [_glob_to_regex(path) for path in triggers["pull_request"]["paths"]] + + uncovered = [ + pattern + for patterns in SNIPPET_GLOBS.values() + for pattern in patterns + if not any(filt.match(pattern) for filt in filters) + ] + + assert not uncovered, ( + "SNIPPET_GLOBS pattern(s) not covered by the `paths:` filter in " + f"{os.path.basename(SNIPPET_SYNC_WORKFLOW)}: {uncovered}. A snippet change under " + "these paths would not trigger the sync check, so llms.txt could drift unnoticed. " + "Add a matching entry to the workflow's `paths:` list." + ) + + # Each library: (weaviate/* GitHub repo, regex matching the # "**Library**: vX.Y.Z+" bullet in llms.txt). The regex anchors to the bullet's # prefix so adding new libraries to llms.txt doesn't break it. diff --git a/uv.lock b/uv.lock index 3b19c9f5..b0f0e87f 100644 --- a/uv.lock +++ b/uv.lock @@ -1884,6 +1884,7 @@ dependencies = [ { name = "playwright" }, { name = "pytest" }, { name = "python-dotenv" }, + { name = "pyyaml" }, { name = "requests" }, { name = "tqdm" }, { name = "weaviate-agents" }, @@ -1903,6 +1904,7 @@ requires-dist = [ { name = "playwright", specifier = ">=1.40.0" }, { name = "pytest", specifier = ">=8.3.5" }, { name = "python-dotenv", specifier = ">=1.1.1" }, + { name = "pyyaml", specifier = ">=6.0" }, { name = "requests", specifier = ">=2.32.3" }, { name = "tqdm", specifier = ">=4.67.1" }, { name = "weaviate-agents", specifier = ">=1.7.0" },