From fcbf5c0d1e401213d056ca6bd2161f30569b07b0 Mon Sep 17 00:00:00 2001 From: SecureCloudGroup Date: Tue, 6 Oct 2026 08:20:14 -0400 Subject: [PATCH] Phase 0 gates: every row typed in answers-check, lint ratchet, unservable lookup rows, full offline suite in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check_sample now types every row's present cells and every column element (the first row still in full; a cell a row lacks is allowed, as the app's sparse-row binding shows "—" for it). answers-check --lint is a gate: kind lints minus the reasoned lint_exempt.json entries and utc lints fail; a keyword lint fails only when it is new against tests/fixtures/keyword_lint_baseline.json (474 held today), so the backlog can only shrink. evallookup reports rows whose every accepted source is refused, failed or missing as unservable instead of counting them, and an all-unservable set never passes. CI runs the lint gate and the whole offline suite after the build (the built pack's tests included), gates the development lookup set and reports held-out and sealed (held-out 36/38 on 2026-10-06). The lint test asserts the gate passes on the repo's own data; the column test helper types a day column as date. Docs: current counts (8,672 sources, 310 curated from 140 providers, 129 subcategories) and packs described as sha256-pinned until Ed25519 lands (PLAN R12); SCHEMA.md states the all-rows rule. Round 19 execution plan, Phase 0. --- .github/workflows/ci.yml | 13 +- CONTRIBUTING.md | 2 +- README.md | 6 +- SECURITY.md | 2 +- docs/PLAN.md | 6 +- docs/SCHEMA.md | 2 +- sourcetool/answers.py | 66 ++- sourcetool/evallookup.py | 27 +- tests/fixtures/keyword_lint_baseline.json | 476 ++++++++++++++++++++++ tests/test_answers.py | 2 +- tests/test_records_v12.py | 4 +- tests/test_tooling_v12.py | 3 +- 12 files changed, 584 insertions(+), 25 deletions(-) create mode 100644 tests/fixtures/keyword_lint_baseline.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae7c9f8..c9189f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,8 +21,8 @@ jobs: - run: pip install "httpx==0.28.1" "duckdb==1.5.4" "pytest==9.1.1" - name: schema check (every record + every answers/ file; no live fetch) run: python -m sourcetool check - - name: answers rules (schema, the sample check, build merge; offline) - run: python -m pytest tests/test_answers.py -q + - name: lints (kinds served minus reasoned exemptions, utc consistency, keyword baseline ratchet; offline) + run: python -m sourcetool answers-check --lint - name: probe sources changed in this PR (fail on regressions only) if: github.event_name == 'pull_request' run: | @@ -56,9 +56,12 @@ jobs: run: python -m sourcetool evalres resolution_asks resolution_holdout - name: build the DuckDB index run: python -m sourcetool build - - name: "source lookup (development + held-out; the first source offered must be right)" - run: | - python -m sourcetool evallookup lookup_asks lookup_holdout || echo "::warning::lookup below 100%, see output" + - name: offline suite (answers rules, recorded samples, asks overlap gate, taxonomy recall, resolvers, tooling; the built pack's tests included) + run: python -m pytest tests -q --ignore=tests/test_ingest.py + - name: "source lookup, development set (the first source offered must be right; rows no offerable source can serve are reported, not counted)" + run: python -m sourcetool evallookup lookup_asks + - name: "source lookup, held-out + sealed (measured, never tuned on; held-out stood at 36/38 on 2026-10-06)" + run: python -m sourcetool evallookup lookup_holdout lookup_sealed || echo "::warning::held-out or sealed lookup below 100%, see output" service: name: Library API + ingest tests runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0072c4a..1d7f937 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,6 +17,6 @@ You don't need Git or a GitHub account. Suggestions arrive through the Library A 1. Changes arrive through `sourcetool ingest`, `sourcetool harvest` or hand edits to `sources/curated/*.jsonl`. 2. Run `python -m sourcetool check` and `python -m sourcetool validate --id `. 3. Open a PR. CI schema-checks every record and probes the changed ones. -4. Only the maintainer merges into `main`. A tag publishes a new signed pack. +4. Only the maintainer merges into `main`. A tag publishes a new pack; each app release pins that pack's sha256 (Ed25519 signing is planned, see `docs/PLAN.md` R12). Please report security issues privately (see [SECURITY.md](SECURITY.md)). diff --git a/README.md b/README.md index cc8cb7c..8f2f672 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,10 @@ The SmartBrain Library is an open registry of **where the data lives**: weather, [SmartBrain](https://github.com/SecureCloudGroup/SmartBrain_3000)'s Neural Interface looks a request up in the Library **on the user's own machine**. That lookup takes milliseconds and uses no cloud search or AI service. SmartBrain then offers the best sources one at a time. When the user answers **Yes**, SmartBrain builds the card. -- **9,045 sources** (6,918 validated ok), US scope in v1: - - 269 hand-curated and validated sources from 131 providers; +- **8,672 sources** (6,531 validated ok), US scope in v1: + - 310 hand-curated sources (257 validated ok) from 140 providers, 254 of them with declared answers the app builds cards from; - harvested from open catalogs: US open-data portals, the Mobility Database, GBFS, Census, Wikidata, APIs.guru and public-apis. -- **18 categories, 127 subcategories**, in `taxonomy/taxonomy.json`. +- **18 categories, 129 subcategories**, in `taxonomy/taxonomy.json`. - Every source records: - who publishes it and how official it is; - its terms; diff --git a/SECURITY.md b/SECURITY.md index 6ce69d6..fdcb766 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,4 +6,4 @@ What this repo can affect: SmartBrain installs fetch data from the sources liste - only the maintainer merges changes; - CI schema checks: https only, public hosts, no credentials, no undeclared parameters; - the app runs the same validator, fetches only through its SSRF-guarded network layer, and treats every response as untrusted data; -- packs are signed and pinned on first use, and older packs are refused (no rollback). +- each app release pins one pack by tag and sha256 and installs nothing else (Ed25519 signing between app releases is planned, see `docs/PLAN.md` R12). diff --git a/docs/PLAN.md b/docs/PLAN.md index bbf7f26..fbd9699 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -108,7 +108,7 @@ Security: - Submissions go into a queue (SQLite on the VPS). - The only operator endpoint is authenticated (token held by the operator's machine) and reads that queue. -Flow: the queue feeds `sourcetool ingest` on the operator's side. It validates, aggregates votes per source, drafts a classification (R3) and opens a PR in this repo. The operator merges, a tag triggers a new signed pack, and the app fetches the manifest and installs the pack. +Flow: the queue feeds `sourcetool ingest` on the operator's side. It validates, aggregates votes per source, drafts a classification (R3) and opens a PR in this repo. The operator merges, a tag triggers a new pack release (sha256-pinned by the app; Ed25519 in v2), and the app fetches the manifest and installs the pack. ## 5. Distribution and trust @@ -118,7 +118,7 @@ Flow: the queue feeds `sourcetool ingest` on the operator's side. It validates, ## 6. In the app (R8, R9) -- **DuckDB tables:** `library_sources`, `library_source_categories`, `library_terms`, `library_taxonomy` and `library_meta`, loaded from the signed pack. They are plaintext: public catalog data holds no user content. +- **DuckDB tables:** `library_sources`, `library_source_categories`, `library_terms`, `library_taxonomy` and `library_meta`, loaded from the pinned pack. They are plaintext: public catalog data holds no user content. - **Local sources** are user data. They are sealed in the vault and indexed **in memory at unlock** (a user has tens or hundreds, not thousands). Scoring merges them with the Library results. So nothing about a user's interests sits in plaintext at rest. - **Lookup** uses the SQL in `sourcetool/build.py` (`_LOOKUP`): term weights plus a category match from the taxonomy, plus a prior. The prior covers tier, validation, authority, votes, terms and usage. - **The Library page** (`/ni/library`, R9): @@ -135,7 +135,7 @@ Flow: the queue feeds `sourcetool ingest` on the operator's side. It validates, |---|---|---| | A | Taxonomy v1, schema, `sourcetool` check, validate, harvest, build and lookup; curated US core; 7 harvesters; DuckDB build | **done** | | B | Repo public and protected; CI (schema check plus validation of changed records); scheduled re-validation | this change | -| C | App: signed pack install, DuckDB tables, lookup in the NI finder, Yes recorded locally, the Library page and form | next (SmartBrain_3000 PR) | +| C | App: pinned pack install, DuckDB tables, lookup in the NI finder, Yes recorded locally, the Library page and form | next (SmartBrain_3000 PR) | | D | The Library API on the VPS, `sourcetool ingest`, vote aggregation, and the Yes vote plus suggestions from the app | service and ingest built; VPS deploy and the app's calls next | | E | Model-drafted classification (R3) to fix keyword misplacements; more harvesters (state 511 and GTFS registries, NWS stations, NOAA stations as resolvers) | rolling | diff --git a/docs/SCHEMA.md b/docs/SCHEMA.md index 54f3ad4..cad6aea 100644 --- a/docs/SCHEMA.md +++ b/docs/SCHEMA.md @@ -26,7 +26,7 @@ Each source is one JSON object per line in `sources/**/*.jsonl`. Users' suggesti | `votes` | `{yes, no}`, aggregated from users' Yes/No answers through the Library API. | | `signals` | Harvest evidence: monthly views, last data update, column names. | | `origin` | Who or what created the record, and when. | -| `answers` | Curated sources: which paths of the response answer which questions, so the app builds cards without guessing paths. Authored in `answers/.json` (`{source_id, answers, sample_url, checked}`) and merged into the record by `sourcetool build`. A source has 1 to 12 answers, each `{name, label, words, primary?, kind: value\|list\|columns, ...}`; a path segment may be a whole `{param}` of the record, a key the plain names can't spell is quoted (`description["#cdata-section"]`), and a list may `filter` its rows by one or by a short fixed value the response uses (`"Final"`). A `time` whose values carry no zone is shown as the source's local clock; `"utc": true` says those zoneless values are UTC. Spec v1.2 adds what an answer delivers (below). The closed keys, types and limits are in `sourcetool/answers.py` (`answer_problems`), and every path must resolve in a real sample (`check_sample`, run live by `sourcetool answers-check`). | +| `answers` | Curated sources: which paths of the response answer which questions, so the app builds cards without guessing paths. Authored in `answers/.json` (`{source_id, answers, sample_url, checked}`) and merged into the record by `sourcetool build`. A source has 1 to 12 answers, each `{name, label, words, primary?, kind: value\|list\|columns, ...}`; a path segment may be a whole `{param}` of the record, a key the plain names can't spell is quoted (`description["#cdata-section"]`), and a list may `filter` its rows by one or by a short fixed value the response uses (`"Final"`). A `time` whose values carry no zone is shown as the source's local clock; `"utc": true` says those zoneless values are UTC. Spec v1.2 adds what an answer delivers (below). The closed keys, types and limits are in `sourcetool/answers.py` (`answer_problems`), and every path must resolve in a real sample (`check_sample`, run live by `sourcetool answers-check`): the first row of a list in full, and in every later row and every column element a present cell must hold its declared type while a cell the row lacks is allowed (the card shows "—" for it). | ### Answers spec v1.2: what an answer delivers diff --git a/sourcetool/answers.py b/sourcetool/answers.py index 629dd2a..62bdfc4 100644 --- a/sourcetool/answers.py +++ b/sourcetool/answers.py @@ -488,6 +488,41 @@ def _axis_check(w: str, cells: list) -> list[str]: return [] if bad is _MISSING else [f"{w}: axis cell {_show(bad)} is not an ISO date/time"] +_MAX_CHECKED_ROWS = 500 # the engine's series cap: every row a card could show is typed (spec v1.3, 2026-10-05) +# a cell a row lacks: the app's sparse-row rule (ni_flow._MISSING) shows "—" for these, never a wrong value +_SPARSE_TEXT = frozenset({"", "mm", "n/a", "na", "-", "--", "—", "null", "none", "missing"}) + + +def _sparse(v) -> bool: + return v is _MISSING or v is None or (isinstance(v, str) and v.strip().lower() in _SPARSE_TEXT) + + +def _check_rows(w: str, items: list, row: list, ex: dict) -> list[str]: + """Every row after the first (spec v1.3): a cell that is PRESENT must hold its declared type; a cell + the row lacks is allowed (the card shows "—" for it). The caller checks the first row in full. One + message per field, naming the first offending row; bounded by ``_MAX_CHECKED_ROWS``.""" + errs: list[str] = [] + for j, f in enumerate(row): + for i, it in enumerate(items[1:_MAX_CHECKED_ROWS], start=1): # rows 1..499: the first 500 rows with row 0 + v = resolve(it, f["path"], ex) + if _sparse(v): + continue + if not type_ok(v, f["type"]): + errs.append(f"{w} row[{j}] at item {i}: {f['path']} is {_show(v)}, not {f['type']}") + break + return errs + + +def _check_column(w: str, col: list, f: dict) -> list[str]: + """Every element of a column (spec v1.3): a present value holds the declared type; nulls are allowed.""" + for i, v in enumerate(col[:_MAX_CHECKED_ROWS]): + if _sparse(v): + continue + if not type_ok(v, f["type"]): + return [f"{w} at index {i}: {f['path']} is {_show(v)}, not {f['type']}"] + return [] + + def check_sample(answers: list, sample, examples: dict | None = None) -> list[str]: """Every path resolves in the sample with the type its answer promises. `examples` maps each of the record's params to its `example` (what a `{param}` segment or a filter stands for in the sample). @@ -520,6 +555,7 @@ def check_sample(answers: list, sample, examples: dict | None = None) -> list[st else: for j, f in enumerate(a["row"]): errs += _check_field(f"{w} row[{j}]", items[0], sample, f, ex) + errs += _check_rows(w, items, a["row"], ex) if "axis" in a: errs += _axis_check(w, [resolve(it, a["axis"]["cell"], ex) for it in items]) else: @@ -530,6 +566,7 @@ def check_sample(answers: list, sample, examples: dict | None = None) -> list[st errs.append(f"{w} columns[{j}]: {f['path']} is {_show(col)}, not a list") continue lengths.add(len(col)) + errs += _check_column(f"{w} columns[{j}]", col, f) if "unit_path" in f and not isinstance(resolve(sample, f["unit_path"], ex), str): errs.append(f"{w} columns[{j}]: unit_path {f['unit_path']} is not text") if len(lengths) > 1: @@ -674,14 +711,39 @@ def lint_report(recs: list[dict], tax: dict | None = None) -> list[str]: return out + keyword_lints(recs, tax) +def exempt_key(lint: str) -> str: + """The key a kind lint is exempted under in lint_exempt.json: ``": kind "`` — the lint message up to + its " has no answer" tail. One place for the rule, used by the gate and the test suite alike.""" + assert isinstance(lint, str), "lint message required" + return lint.split(" has no")[0] + + +KEYWORD_LINT_BASELINE = ROOT / "tests" / "fixtures" / "keyword_lint_baseline.json" +LINT_EXEMPT = ROOT / "tests" / "fixtures" / "records" / "lint_exempt.json" + + def cmd_lint() -> int: + """`answers-check --lint` (offline). Gating since Phase 0 (2026-10-05): kind lints (minus the reasoned + exemptions in lint_exempt.json, the same list the test suite applies) and utc lints fail; a keyword lint + fails only when it is NEW against the committed baseline — a ratchet, so the backlog never grows and + shrinks as sources are filed under their keywords.""" recs = _records() loaded, errs = load_answers({k: params_of(r) for k, r in recs.items()}) merged = [{**r, "answers": loaded[k]} if k in loaded else r for k, r in recs.items() if not r.get("replaced_by")] - problems = [f"INVALID {e}" for e in errs] + [f"LINT {p}" for p in lint_report(merged)] + exempt = json.loads(LINT_EXEMPT.read_text()) if LINT_EXEMPT.is_file() else {} + hard = [p for r in merged for p in answer_lints(r) if exempt_key(p) not in exempt] + hard += utc_consistency_lints(merged) + keyword = keyword_lints(merged) + baseline = set(json.loads(KEYWORD_LINT_BASELINE.read_text())) if KEYWORD_LINT_BASELINE.is_file() else set() + new_kw = [p for p in keyword if p not in baseline] + gone = sorted(baseline - set(keyword)) + problems = [f"INVALID {e}" for e in errs] + [f"LINT {p}" for p in hard] + [f"LINT NEW {p}" for p in new_kw] for p in problems: print(p) - print(f"{len(merged)} records, {len(loaded)} with answers: {len(problems)} lint problems") + if gone: + print(f"{len(gone)} baseline keyword lints no longer fire; drop them from {KEYWORD_LINT_BASELINE.name}") + print(f"{len(merged)} records, {len(loaded)} with answers: {len(problems)} lint problems " + f"({len(keyword) - len(new_kw)} keyword lints held by the baseline)") return 1 if problems else 0 diff --git a/sourcetool/evallookup.py b/sourcetool/evallookup.py index 5d34a1a..d7593ad 100644 --- a/sourcetool/evallookup.py +++ b/sourcetool/evallookup.py @@ -47,14 +47,29 @@ def _run_classify(name: str) -> int: return 1 if low else 0 +def _servable(ok_ids: list, recs: dict) -> bool: + """At least one accepted source exists and can be offered (not retired, failed or refused). A row whose + every accepted source is unofferable (the refused ESPN endpoints, an id that was never a record) cannot + pass or fail a lookup; it is reported as unservable (Phase 0, 2026-10-05).""" + for i in ok_ids: + r = recs.get(i) + if r and not r.get("replaced_by") and (r.get("validation") or {}).get("status") not in ("failed", "refused"): + return True + return False + + def run(name: str = "lookup_asks") -> int: if name.startswith("classify_"): return _run_classify(name) + from .answers import _records from .build import lookup spec = json.loads((ROOT / "tests" / f"{name}.json").read_text()) + recs = _records() + cases = [c for c in spec["cases"] if _servable(c["ok"], recs)] + unservable = [c for c in spec["cases"] if not _servable(c["ok"], recs)] top1 = top3 = 0 fails = [] - for c in spec["cases"]: + for c in cases: ids = [r["id"] for r in lookup(c["ask"], limit=3)] ok1 = bool(ids) and ids[0] in c["ok"] ok3 = any(i in c["ok"] for i in ids) @@ -62,7 +77,9 @@ def run(name: str = "lookup_asks") -> int: top3 += ok3 if not ok1: fails.append(f" {'top3' if ok3 else 'MISS'} {c['ask']!r}: got {ids}") - n = len(spec["cases"]) - print(f"{name}: top-1 {top1}/{n} ({100 * top1 // n}%), top-3 {top3}/{n} ({100 * top3 // n}%)") - print("\n".join(fails)) - return 0 if top1 == n else 1 + n = len(cases) + pct = (lambda k: 100 * k // n) if n else (lambda k: 0) + print(f"{name}: top-1 {top1}/{n} ({pct(top1)}%), top-3 {top3}/{n} ({pct(top3)}%); " + f"{len(unservable)} unservable (every accepted source refused, failed or missing)") + print("\n".join(fails + [f" UNSERVABLE {c['ask']!r}: {c['ok']}" for c in unservable])) + return 0 if n and top1 == n else 1 # an all-unservable file is a broken set, never a pass diff --git a/tests/fixtures/keyword_lint_baseline.json b/tests/fixtures/keyword_lint_baseline.json new file mode 100644 index 0000000..a52311d --- /dev/null +++ b/tests/fixtures/keyword_lint_baseline.json @@ -0,0 +1,476 @@ +[ + "civic/courts: no curated source filed here speaks to 'case'", + "civic/elections: no curated source filed here speaks to 'ballot'", + "civic/elections: no curated source filed here speaks to 'polls'", + "civic/elections: no curated source filed here speaks to 'primary'", + "civic/elections: no curated source filed here speaks to 'voter registration'", + "civic/gov_contracts: no curated source filed here speaks to 'contract award'", + "civic/gov_contracts: no curated source filed here speaks to 'grant'", + "civic/legislation: no curated source filed here speaks to 'vote'", + "civic/open_data: no curated source filed here speaks to 'dataset'", + "civic/open_data: no curated source filed here speaks to 'open data'", + "culture/books: no curated source filed here speaks to 'book release'", + "culture/books: no curated source filed here speaks to 'new book'", + "culture/events: no curated source filed here speaks to 'festival'", + "culture/events: no curated source filed here speaks to 'things to do'", + "culture/events: no curated source filed here speaks to 'tour dates'", + "culture/games: no curated source filed here speaks to 'video game'", + "culture/lottery: no curated source filed here speaks to 'cash4life'", + "culture/lottery: no curated source filed here speaks to 'win the lottery'", + "culture/lottery: no curated source filed here speaks to 'win the mega millions'", + "culture/lottery: no curated source filed here speaks to 'win the powerball'", + "culture/movies: no curated source filed here speaks to 'box office'", + "culture/movies: no curated source filed here speaks to 'new releases'", + "culture/museums_art: no curated source filed here speaks to 'exhibit'", + "culture/music: no curated source filed here speaks to 'billboard'", + "culture/music: no curated source filed here speaks to 'new album'", + "culture/trivia_fun: no curated source filed here speaks to 'fact'", + "culture/tv: no curated source filed here speaks to 'season'", + "culture/tv: no curated source filed here speaks to 'tv listings'", + "economy/economic_calendar: no curated source filed here speaks to 'economic calendar'", + "economy/economic_calendar: no curated source filed here speaks to 'fed meeting'", + "economy/economic_calendar: no curated source filed here speaks to 'fomc'", + "economy/economic_calendar: no curated source filed here speaks to 'release schedule'", + "economy/food_prices: no curated source filed here speaks to 'food price'", + "economy/food_prices: no curated source filed here speaks to 'grocery'", + "economy/fuel_prices: no curated source filed here speaks to 'cheapest gas'", + "economy/fuel_prices: no curated source filed here speaks to 'diesel'", + "economy/fuel_prices: no curated source filed here speaks to 'price of gas'", + "economy/gdp: no curated source filed here speaks to 'economic growth'", + "economy/housing: no curated source filed here speaks to 'house price index'", + "economy/housing: no curated source filed here speaks to 'rent'", + "economy/inflation: no curated source filed here speaks to 'ppi'", + "economy/inflation: no curated source filed here speaks to 'producer price'", + "economy/inflation: no curated source filed here speaks to 'producer prices'", + "economy/jobs: no curated source filed here speaks to 'labor'", + "economy/trade: no curated source filed here speaks to 'exports'", + "economy/trade: no curated source filed here speaks to 'imports'", + "economy/trade: no curated source filed here speaks to 'tariff'", + "energy/electricity_prices: no curated source filed here speaks to 'power price'", + "energy/electricity_prices: no curated source filed here speaks to 'utility rate'", + "energy/grid: no curated source filed here speaks to 'caiso'", + "energy/grid: no curated source filed here speaks to 'generation mix'", + "energy/grid: no curated source filed here speaks to 'iso'", + "energy/oil_gas_supply: no curated source filed here speaks to 'crude stocks'", + "energy/oil_gas_supply: no curated source filed here speaks to 'rig count'", + "hazards/earthquakes: no curated source filed here speaks to 'aftershock'", + "hazards/earthquakes: no curated source filed here speaks to 'aftershocks'", + "hazards/earthquakes: no curated source filed here speaks to 'richter'", + "hazards/earthquakes: no curated source filed here speaks to 'seismic'", + "hazards/earthquakes: no curated source filed here speaks to 'temblor'", + "hazards/earthquakes: no curated source filed here speaks to 'tremor'", + "hazards/floods_rivers: no curated source filed here speaks to 'creek'", + "hazards/floods_rivers: no curated source filed here speaks to 'crest forecast'", + "hazards/floods_rivers: no curated source filed here speaks to 'flood stage'", + "hazards/floods_rivers: no curated source filed here speaks to 'flooded'", + "hazards/floods_rivers: no curated source filed here speaks to 'flooding'", + "hazards/floods_rivers: no curated source filed here speaks to 'levee'", + "hazards/floods_rivers: no curated source filed here speaks to 'river crest'", + "hazards/floods_rivers: no curated source filed here speaks to 'river forecast'", + "hazards/floods_rivers: no curated source filed here speaks to 'river height'", + "hazards/space_weather: no curated source filed here speaks to 'aurora borealis'", + "hazards/space_weather: no curated source filed here speaks to 'cme'", + "hazards/space_weather: no curated source filed here speaks to 'coronal mass ejection'", + "hazards/space_weather: no curated source filed here speaks to 'solar activity'", + "hazards/space_weather: no curated source filed here speaks to 'solar wind'", + "hazards/space_weather: no curated source filed here speaks to 'sunspot'", + "hazards/space_weather: no curated source filed here speaks to 'sunspots'", + "hazards/space_weather: no curated source filed here speaks to 'x class flare'", + "hazards/tropical_storms: no curated source filed here speaks to 'hurricane center'", + "hazards/tropical_storms: no curated source filed here speaks to 'hurricane season'", + "hazards/tropical_storms: no curated source filed here speaks to 'in the atlantic'", + "hazards/tropical_storms: no curated source filed here speaks to 'in the gulf'", + "hazards/tropical_storms: no curated source filed here speaks to 'spaghetti models'", + "hazards/tropical_storms: no curated source filed here speaks to 'storm track'", + "hazards/tropical_storms: no curated source filed here speaks to 'tropical depression'", + "hazards/tropical_storms: no curated source filed here speaks to 'tropical storm watch'", + "hazards/tropical_storms: no curated source filed here speaks to 'typhoon'", + "hazards/tsunami: no curated source filed here speaks to 'tsunami advisory'", + "hazards/tsunami: no curated source filed here speaks to 'tsunami watch'", + "hazards/volcanoes: no curated source filed here speaks to 'active volcano'", + "hazards/volcanoes: no curated source filed here speaks to 'ash advisory'", + "hazards/volcanoes: no curated source filed here speaks to 'ashfall'", + "hazards/volcanoes: no curated source filed here speaks to 'kilauea'", + "hazards/volcanoes: no curated source filed here speaks to 'mauna loa'", + "hazards/volcanoes: no curated source filed here speaks to 'mount rainier'", + "hazards/volcanoes: no curated source filed here speaks to 'mount shasta'", + "hazards/volcanoes: no curated source filed here speaks to 'mount spurr'", + "hazards/volcanoes: no curated source filed here speaks to 'mount st helens'", + "hazards/volcanoes: no curated source filed here speaks to 'volcanic ash'", + "hazards/volcanoes: no curated source filed here speaks to 'volcanic'", + "hazards/volcanoes: no curated source filed here speaks to 'volcano alert level'", + "hazards/wildfires: no curated source filed here speaks to 'acres burned'", + "hazards/wildfires: no curated source filed here speaks to 'blaze'", + "hazards/wildfires: no curated source filed here speaks to 'brush fire'", + "hazards/wildfires: no curated source filed here speaks to 'burn'", + "hazards/wildfires: no curated source filed here speaks to 'evacuation'", + "hazards/wildfires: no curated source filed here speaks to 'evacuations'", + "hazards/wildfires: no curated source filed here speaks to 'fire danger'", + "hazards/wildfires: no curated source filed here speaks to 'fire map'", + "hazards/wildfires: no curated source filed here speaks to 'fire near'", + "hazards/wildfires: no curated source filed here speaks to 'fires near'", + "hazards/wildfires: no curated source filed here speaks to 'forest fire'", + "hazards/wildfires: no curated source filed here speaks to 'smoke'", + "health/hospitals: no curated source filed here speaks to 'clinic'", + "health/hospitals: no curated source filed here speaks to 'er wait'", + "markets/commodities: no curated source filed here speaks to 'bullion'", + "markets/commodities: no curated source filed here speaks to 'commodity'", + "markets/commodities: no curated source filed here speaks to 'copper'", + "markets/commodities: no curated source filed here speaks to 'corn'", + "markets/commodities: no curated source filed here speaks to 'lumber'", + "markets/commodities: no curated source filed here speaks to 'soybeans'", + "markets/commodities: no curated source filed here speaks to 'wheat'", + "markets/crypto: no curated source filed here speaks to 'altcoins'", + "markets/crypto: no curated source filed here speaks to 'cardano'", + "markets/crypto: no curated source filed here speaks to 'doge'", + "markets/crypto: no curated source filed here speaks to 'dogecoin'", + "markets/crypto: no curated source filed here speaks to 'eth'", + "markets/crypto: no curated source filed here speaks to 'litecoin'", + "markets/crypto: no curated source filed here speaks to 'memecoin'", + "markets/crypto: no curated source filed here speaks to 'stablecoin'", + "markets/crypto: no curated source filed here speaks to 'tether'", + "markets/crypto: no curated source filed here speaks to 'token'", + "markets/crypto: no curated source filed here speaks to 'tokens'", + "markets/crypto: no curated source filed here speaks to 'xrp'", + "markets/filings: no curated source filed here speaks to '10k'", + "markets/filings: no curated source filed here speaks to '8k'", + "markets/funds: no curated source filed here speaks to 'etf'", + "markets/funds: no curated source filed here speaks to 'mutual fund'", + "markets/funds: no curated source filed here speaks to 'nav'", + "markets/fx: no curated source filed here speaks to 'forex'", + "markets/indices: no curated source filed here speaks to 'futures'", + "markets/indices: no curated source filed here speaks to 'indexes'", + "markets/indices: no curated source filed here speaks to 'indices'", + "markets/indices: no curated source filed here speaks to 'market today'", + "markets/indices: no curated source filed here speaks to 'russell 2000'", + "markets/indices: no curated source filed here speaks to 'russell'", + "markets/indices: no curated source filed here speaks to 'small caps'", + "markets/indices: no curated source filed here speaks to 'sp500'", + "markets/indices: no curated source filed here speaks to 'the dow'", + "markets/indices: no curated source filed here speaks to 'the market'", + "markets/indices: no curated source filed here speaks to 'wall street'", + "markets/stocks: no curated source filed here speaks to 'after hours'", + "markets/stocks: no curated source filed here speaks to 'market cap'", + "markets/stocks: no curated source filed here speaks to 'nyse'", + "markets/stocks: no curated source filed here speaks to 'premarket'", + "markets/stocks: no curated source filed here speaks to 'share of'", + "markets/stocks: no curated source filed here speaks to 'share price'", + "markets/stocks: no curated source filed here speaks to 'share'", + "markets/stocks: no curated source filed here speaks to 'shares'", + "markets/stocks: no curated source filed here speaks to 'stock quote'", + "markets/stocks: no curated source filed here speaks to 'trading at'", + "news/headlines: no curated source filed here speaks to 'big news'", + "news/headlines: no curated source filed here speaks to 'current events'", + "news/headlines: no curated source filed here speaks to 'happening today'", + "news/headlines: no curated source filed here speaks to 'what happened'", + "news/headlines: no curated source filed here speaks to 'whats happening'", + "news/local_news: no curated source filed here speaks to 'in my area'", + "news/local_news: no curated source filed here speaks to 'my town'", + "news/podcasts: no curated source filed here speaks to 'radiolab'", + "news/podcasts: no curated source filed here speaks to 'the daily'", + "news/podcasts: no curated source filed here speaks to 'this american life'", + "news/podcasts: no curated source filed here speaks to 'up first'", + "personal/email_calendar: no curated source filed here speaks to 'inbox'", + "personal/email_calendar: no curated source filed here speaks to 'my calendar'", + "personal/home_devices: no curated source filed here speaks to 'sensor'", + "personal/home_devices: no curated source filed here speaks to 'smart home'", + "personal/knowledge_base: no curated source filed here speaks to 'my documents'", + "personal/schedules_runs: no curated source filed here speaks to 'my schedule'", + "science/environment_data: no curated source filed here speaks to 'carbon'", + "science/environment_data: no curated source filed here speaks to 'emissions'", + "science/geysers: no curated source filed here speaks to 'erupted'", + "science/geysers: no curated source filed here speaks to 'erupting'", + "shopping/app_prices: no curated source filed here speaks to 'app update'", + "shopping/app_prices: no curated source filed here speaks to 'software release'", + "shopping/availability: no curated source filed here speaks to 'back in stock'", + "shopping/product_prices: no curated source filed here speaks to 'cheapest'", + "shopping/product_prices: no curated source filed here speaks to 'deal'", + "shopping/product_prices: no curated source filed here speaks to 'price drop'", + "sky/iss: no curated source filed here speaks to 'pass over'", + "sky/launches: no curated source filed here speaks to 'artemis'", + "sky/launches: no curated source filed here speaks to 'falcon 9'", + "sky/near_earth: no curated source filed here speaks to 'neo'", + "sky/sky_events: no curated source filed here speaks to 'blood moon'", + "sky/sky_events: no curated source filed here speaks to 'comet'", + "sky/sky_events: no curated source filed here speaks to 'comets'", + "sky/sky_events: no curated source filed here speaks to 'conjunction'", + "sky/sky_events: no curated source filed here speaks to 'eclipse'", + "sky/sky_events: no curated source filed here speaks to 'geminids'", + "sky/sky_events: no curated source filed here speaks to 'jupiter'", + "sky/sky_events: no curated source filed here speaks to 'leonids'", + "sky/sky_events: no curated source filed here speaks to 'lyrids'", + "sky/sky_events: no curated source filed here speaks to 'mercury retrograde'", + "sky/sky_events: no curated source filed here speaks to 'meteor shower'", + "sky/sky_events: no curated source filed here speaks to 'meteor'", + "sky/sky_events: no curated source filed here speaks to 'meteors'", + "sky/sky_events: no curated source filed here speaks to 'orionids'", + "sky/sky_events: no curated source filed here speaks to 'perseids'", + "sky/sky_events: no curated source filed here speaks to 'planet'", + "sky/sky_events: no curated source filed here speaks to 'planets'", + "sky/sky_events: no curated source filed here speaks to 'quadrantids'", + "sky/sky_events: no curated source filed here speaks to 'retrograde'", + "sky/sky_events: no curated source filed here speaks to 'saturn'", + "sky/sky_events: no curated source filed here speaks to 'shooting stars'", + "sky/sky_events: no curated source filed here speaks to 'sky events'", + "sky/sky_events: no curated source filed here speaks to 'stargazing'", + "sky/sky_events: no curated source filed here speaks to 'venus'", + "sky/sky_events: no curated source filed here speaks to 'visible planets'", + "sky/space_images: no curated source filed here speaks to 'mars rover'", + "sky/sun_moon: no curated source filed here speaks to 'day length'", + "sky/sun_moon: no curated source filed here speaks to 'get dark'", + "sky/sun_moon: no curated source filed here speaks to 'gets dark'", + "sky/sun_moon: no curated source filed here speaks to 'golden hour'", + "sky/sun_moon: no curated source filed here speaks to 'sun come up'", + "sky/sun_moon: no curated source filed here speaks to 'sun go down'", + "sky/sun_moon: no curated source filed here speaks to 'sun set'", + "sports/college: no curated source filed here speaks to 'ap poll'", + "sports/college: no curated source filed here speaks to 'bowl game'", + "sports/college: no curated source filed here speaks to 'cfb'", + "sports/college: no curated source filed here speaks to 'college basketball'", + "sports/college: no curated source filed here speaks to 'crimson tide'", + "sports/college: no curated source filed here speaks to 'heisman'", + "sports/college: no curated source filed here speaks to 'march madness'", + "sports/college: no curated source filed here speaks to 'ncaa'", + "sports/college: no curated source filed here speaks to 'ncaab'", + "sports/college: no curated source filed here speaks to 'ncaaf'", + "sports/injuries_news: no curated source filed here speaks to 'injury report'", + "sports/injuries_news: no curated source filed here speaks to 'roster'", + "sports/injuries_news: no curated source filed here speaks to 'team news'", + "sports/motorsport_golf_other: no curated source filed here speaks to 'golf leaderboard'", + "sports/motorsport_golf_other: no curated source filed here speaks to 'nascar'", + "sports/motorsport_golf_other: no curated source filed here speaks to 'tennis'", + "sports/motorsport_golf_other: no curated source filed here speaks to 'ufc'", + "sports/results: no curated source filed here speaks to 'beat'", + "sports/results: no curated source filed here speaks to 'how did'", + "sports/results: no curated source filed here speaks to 'lose'", + "sports/results: no curated source filed here speaks to 'win loss record'", + "sports/schedules: no curated source filed here speaks to 'mnf'", + "sports/schedules: no curated source filed here speaks to 'monday night football'", + "sports/schedules: no curated source filed here speaks to 'play today'", + "sports/schedules: no curated source filed here speaks to 'play tonight'", + "sports/schedules: no curated source filed here speaks to 'playing'", + "sports/schedules: no curated source filed here speaks to 'puck drop'", + "sports/schedules: no curated source filed here speaks to 'sunday night football'", + "sports/schedules: no curated source filed here speaks to 'thursday night football'", + "sports/schedules: no curated source filed here speaks to 'upcoming matches'", + "sports/scores: no curated source filed here speaks to 'box score'", + "sports/scores: no curated source filed here speaks to 'game over'", + "sports/scores: no curated source filed here speaks to 'halftime'", + "sports/scores: no curated source filed here speaks to 'live score'", + "sports/scores: no curated source filed here speaks to 'live scores'", + "sports/scores: no curated source filed here speaks to 'up or down'", + "sports/scores: no curated source filed here speaks to 'vs'", + "sports/scores: no curated source filed here speaks to 'who s winning'", + "sports/scores: no curated source filed here speaks to 'whos winning'", + "sports/scores: no curated source filed here speaks to 'winning'", + "sports/standings: no curated source filed here speaks to 'clinched'", + "sports/standings: no curated source filed here speaks to 'elimination number'", + "sports/standings: no curated source filed here speaks to 'magic number'", + "sports/standings: no curated source filed here speaks to 'playoff picture'", + "sports/standings: no curated source filed here speaks to 'seeding'", + "sports/standings: no curated source filed here speaks to 'top 25'", + "tech/ai_models: no curated source filed here speaks to 'leaderboard'", + "tech/ai_models: no curated source filed here speaks to 'model release'", + "tech/domains_web: no curated source filed here speaks to 'domain expiry'", + "tech/domains_web: no curated source filed here speaks to 'ssl certificate'", + "tech/domains_web: no curated source filed here speaks to 'uptime'", + "tech/packages: no curated source filed here speaks to 'package version'", + "tech/repos_releases: no curated source filed here speaks to 'changelog'", + "tech/repos_releases: no curated source filed here speaks to 'kernel'", + "tech/repos_releases: no curated source filed here speaks to 'release notes'", + "tech/repos_releases: no curated source filed here speaks to 'version number'", + "tech/security_advisories: no curated source filed here speaks to 'exploit'", + "tech/security_advisories: no curated source filed here speaks to 'security advisory'", + "tech/security_advisories: no curated source filed here speaks to 'vulnerability'", + "tech/service_status: no curated source filed here speaks to 'degraded'", + "tech/service_status: no curated source filed here speaks to 'down right now'", + "tech/service_status: no curated source filed here speaks to 'having issues'", + "tech/service_status: no curated source filed here speaks to 'is down'", + "tech/service_status: no curated source filed here speaks to 'not loading'", + "tech/service_status: no curated source filed here speaks to 'not working'", + "tech/service_status: no curated source filed here speaks to 'outage right now'", + "tech/service_status: no curated source filed here speaks to 'server status'", + "tech/service_status: no curated source filed here speaks to 'servers down'", + "tech/service_status: no curated source filed here speaks to 'servers up'", + "tech/service_status: no curated source filed here speaks to 'servers'", + "tech/service_status: no curated source filed here speaks to 'status page'", + "tech/tech_news: no curated source filed here speaks to 'product hunt'", + "tech/tech_news: no curated source filed here speaks to 'reddit'", + "time/countdowns: no curated source filed here speaks to 'how long until'", + "time/holidays: no curated source filed here speaks to 'bank holiday'", + "time/holidays: no curated source filed here speaks to 'banks closed'", + "time/holidays: no curated source filed here speaks to 'columbus day'", + "time/holidays: no curated source filed here speaks to 'election day'", + "time/holidays: no curated source filed here speaks to 'fathers day'", + "time/holidays: no curated source filed here speaks to 'fourth of july'", + "time/holidays: no curated source filed here speaks to 'halloween'", + "time/holidays: no curated source filed here speaks to 'hanukkah'", + "time/holidays: no curated source filed here speaks to 'independence day'", + "time/holidays: no curated source filed here speaks to 'july 4th'", + "time/holidays: no curated source filed here speaks to 'juneteenth'", + "time/holidays: no curated source filed here speaks to 'labor day'", + "time/holidays: no curated source filed here speaks to 'mail delivery'", + "time/holidays: no curated source filed here speaks to 'memorial day'", + "time/holidays: no curated source filed here speaks to 'mlk day'", + "time/holidays: no curated source filed here speaks to 'mothers day'", + "time/holidays: no curated source filed here speaks to 'new years day'", + "time/holidays: no curated source filed here speaks to 'new years'", + "time/holidays: no curated source filed here speaks to 'post office closed'", + "time/holidays: no curated source filed here speaks to 'presidents day'", + "time/holidays: no curated source filed here speaks to 'st patricks day'", + "time/holidays: no curated source filed here speaks to 'valentines day'", + "time/holidays: no curated source filed here speaks to 'veterans day'", + "time/observances: no curated source filed here speaks to 'observance'", + "travel/airport_delays: no curated source filed here speaks to 'fog delays'", + "travel/airport_delays: no curated source filed here speaks to 'security line'", + "travel/airport_delays: no curated source filed here speaks to 'security wait'", + "travel/airport_delays: no curated source filed here speaks to 'tsa wait'", + "travel/border_waits: no curated source filed here speaks to 'crossing wait'", + "travel/flights: no curated source filed here speaks to 'arrival'", + "travel/flights: no curated source filed here speaks to 'delayed'", + "travel/flights: no curated source filed here speaks to 'departed'", + "travel/flights: no curated source filed here speaks to 'departure'", + "travel/flights: no curated source filed here speaks to 'flight status'", + "travel/flights: no curated source filed here speaks to 'landed'", + "travel/flights: no curated source filed here speaks to 'took off'", + "travel/rail: no curated source filed here speaks to 'train status'", + "travel/transit: no curated source filed here speaks to 'bus'", + "travel/transit: no curated source filed here speaks to 'light rail'", + "travel/transit: no curated source filed here speaks to 'metro'", + "travel/transit: no curated source filed here speaks to 'next arrival'", + "travel/transit: no curated source filed here speaks to 'transit'", + "travel/transit_alerts: no curated source filed here speaks to 'caltrain'", + "travel/transit_alerts: no curated source filed here speaks to 'dc metro'", + "travel/transit_alerts: no curated source filed here speaks to 'lirr'", + "travel/transit_alerts: no curated source filed here speaks to 'marta'", + "travel/transit_alerts: no curated source filed here speaks to 'metra'", + "travel/transit_alerts: no curated source filed here speaks to 'muni'", + "travel/transit_alerts: no curated source filed here speaks to 'nj transit'", + "travel/transit_alerts: no curated source filed here speaks to 'running late'", + "travel/transit_alerts: no curated source filed here speaks to 'septa'", + "travel/transit_alerts: no curated source filed here speaks to 'train delayed'", + "travel/transit_alerts: no curated source filed here speaks to 'train running'", + "travel/transit_alerts: no curated source filed here speaks to 'trains delayed'", + "travel/transit_alerts: no curated source filed here speaks to 'trains running'", + "travel/transit_alerts: no curated source filed here speaks to 'wmata'", + "water/currents: no curated source filed here speaks to 'flood current'", + "water/marine_forecast: no curated source filed here speaks to 'boat out'", + "water/marine_forecast: no curated source filed here speaks to 'boat'", + "water/marine_forecast: no curated source filed here speaks to 'boating forecast'", + "water/marine_forecast: no curated source filed here speaks to 'boating'", + "water/marine_forecast: no curated source filed here speaks to 'choppy'", + "water/marine_forecast: no curated source filed here speaks to 'coastal waters forecast'", + "water/marine_forecast: no curated source filed here speaks to 'coastal waters'", + "water/marine_forecast: no curated source filed here speaks to 'fishing forecast'", + "water/marine_forecast: no curated source filed here speaks to 'gale warning'", + "water/marine_forecast: no curated source filed here speaks to 'marine conditions'", + "water/marine_forecast: no curated source filed here speaks to 'marine weather'", + "water/marine_forecast: no curated source filed here speaks to 'nearshore forecast'", + "water/marine_forecast: no curated source filed here speaks to 'nearshore'", + "water/marine_forecast: no curated source filed here speaks to 'offshore forecast'", + "water/marine_forecast: no curated source filed here speaks to 'offshore'", + "water/marine_forecast: no curated source filed here speaks to 'sea conditions'", + "water/marine_forecast: no curated source filed here speaks to 'sea state'", + "water/marine_forecast: no curated source filed here speaks to 'seas'", + "water/surf_waves: no curated source filed here speaks to 'big wave'", + "water/surf_waves: no curated source filed here speaks to 'blown out'", + "water/surf_waves: no curated source filed here speaks to 'breakers'", + "water/surf_waves: no curated source filed here speaks to 'flat'", + "water/surf_waves: no curated source filed here speaks to 'glassy'", + "water/surf_waves: no curated source filed here speaks to 'head high'", + "water/surf_waves: no curated source filed here speaks to 'knee high'", + "water/surf_waves: no curated source filed here speaks to 'rideable'", + "water/surf_waves: no curated source filed here speaks to 'surf check'", + "water/surf_waves: no curated source filed here speaks to 'surfable'", + "water/surf_waves: no curated source filed here speaks to 'surfing'", + "water/surf_waves: no curated source filed here speaks to 'swell forecast'", + "water/surf_waves: no curated source filed here speaks to 'waist high'", + "water/surf_waves: no curated source filed here speaks to 'wave forecast'", + "water/tides: no curated source filed here speaks to 'tide table'", + "water/water_levels: no curated source filed here speaks to 'coastal flooding'", + "water/water_levels: no curated source filed here speaks to 'flood at high tide'", + "water/water_levels: no curated source filed here speaks to 'high tide flooding'", + "water/water_levels: no curated source filed here speaks to 'king tide'", + "water/water_levels: no curated source filed here speaks to 'king tides'", + "water/water_levels: no curated source filed here speaks to 'sea level'", + "water/water_levels: no curated source filed here speaks to 'storm surge forecast'", + "water/water_levels: no curated source filed here speaks to 'storm surge warning'", + "water/water_levels: no curated source filed here speaks to 'sunny day flooding'", + "water/water_levels: no curated source filed here speaks to 'tidal flooding'", + "water/water_levels: no curated source filed here speaks to 'tide gauge'", + "water/water_temperature: no curated source filed here speaks to 'cold is the water'", + "water/water_temperature: no curated source filed here speaks to 'cold to swim'", + "water/water_temperature: no curated source filed here speaks to 'lake temp'", + "water/water_temperature: no curated source filed here speaks to 'lake temperature'", + "water/water_temperature: no curated source filed here speaks to 'ocean temp'", + "water/water_temperature: no curated source filed here speaks to 'sea temp'", + "water/water_temperature: no curated source filed here speaks to 'swimming temperature'", + "water/water_temperature: no curated source filed here speaks to 'too cold to swim'", + "water/water_temperature: no curated source filed here speaks to 'warm enough to swim'", + "water/water_temperature: no curated source filed here speaks to 'warm is the lake'", + "water/water_temperature: no curated source filed here speaks to 'warm is the ocean'", + "water/water_temperature: no curated source filed here speaks to 'water cold'", + "water/water_temperature: no curated source filed here speaks to 'water warm'", + "weather/air_quality: no curated source filed here speaks to 'air pollution'", + "weather/air_quality: no curated source filed here speaks to 'pm25'", + "weather/air_quality: no curated source filed here speaks to 'smoky'", + "weather/air_quality: no curated source filed here speaks to 'wildfire smoke'", + "weather/alerts: no curated source filed here speaks to 'dense fog advisory'", + "weather/alerts: no curated source filed here speaks to 'excessive heat warning'", + "weather/alerts: no curated source filed here speaks to 'flash flood warning'", + "weather/alerts: no curated source filed here speaks to 'freeze warning'", + "weather/alerts: no curated source filed here speaks to 'frost advisory'", + "weather/alerts: no curated source filed here speaks to 'heat warning'", + "weather/alerts: no curated source filed here speaks to 'high wind warning'", + "weather/alerts: no curated source filed here speaks to 'high wind watch'", + "weather/alerts: no curated source filed here speaks to 'ice storm warning'", + "weather/alerts: no curated source filed here speaks to 'nws alerts'", + "weather/alerts: no curated source filed here speaks to 'severe thunderstorm watch'", + "weather/alerts: no curated source filed here speaks to 'special weather statement'", + "weather/alerts: no curated source filed here speaks to 'tornado sirens'", + "weather/alerts: no curated source filed here speaks to 'tornado watch'", + "weather/alerts: no curated source filed here speaks to 'tornadoes'", + "weather/alerts: no curated source filed here speaks to 'watches and warnings'", + "weather/alerts: no curated source filed here speaks to 'wind advisory'", + "weather/alerts: no curated source filed here speaks to 'winter storm watch'", + "weather/alerts: no curated source filed here speaks to 'winter weather advisory'", + "weather/climate_records: no curated source filed here speaks to 'drought'", + "weather/current: no curated source filed here speaks to 'current temp'", + "weather/current: no curated source filed here speaks to 'current weather'", + "weather/current: no curated source filed here speaks to 'currently'", + "weather/current: no curated source filed here speaks to 'how cold is it'", + "weather/current: no curated source filed here speaks to 'how hot is it'", + "weather/current: no curated source filed here speaks to 'how warm is it'", + "weather/current: no curated source filed here speaks to 'is it raining'", + "weather/current: no curated source filed here speaks to 'is it snowing'", + "weather/current: no curated source filed here speaks to 'snowing'", + "weather/current: no curated source filed here speaks to 'temp now'", + "weather/current: no curated source filed here speaks to 'temperature now'", + "weather/current: no curated source filed here speaks to 'visibility'", + "weather/forecast: no curated source filed here speaks to 'any lightning'", + "weather/forecast: no curated source filed here speaks to 'below freezing'", + "weather/forecast: no curated source filed here speaks to 'clear skies'", + "weather/forecast: no curated source filed here speaks to 'cold front'", + "weather/forecast: no curated source filed here speaks to 'foggy'", + "weather/forecast: no curated source filed here speaks to 'heat wave'", + "weather/forecast: no curated source filed here speaks to 'high temp'", + "weather/forecast: no curated source filed here speaks to 'jacket'", + "weather/forecast: no curated source filed here speaks to 'lightning expected'", + "weather/forecast: no curated source filed here speaks to 'lightning near'", + "weather/forecast: no curated source filed here speaks to 'lightning strikes'", + "weather/forecast: no curated source filed here speaks to 'low temp'", + "weather/forecast: no curated source filed here speaks to 'overcast'", + "weather/forecast: no curated source filed here speaks to 'precip'", + "weather/forecast: no curated source filed here speaks to 'rainy'", + "weather/forecast: no curated source filed here speaks to 'snowstorm'", + "weather/forecast: no curated source filed here speaks to 't storm'", + "weather/forecast: no curated source filed here speaks to 't storms'", + "weather/forecast: no curated source filed here speaks to 'tstorms'", + "weather/forecast: no curated source filed here speaks to 'wx'", + "weather/radar_imagery: no curated source filed here speaks to 'imagery'", + "weather/uv: no curated source filed here speaks to 'spf'", + "weather/uv: no curated source filed here speaks to 'strong is the sun'", + "weather/uv: no curated source filed here speaks to 'ultraviolet'", + "weather/uv: no curated source filed here speaks to 'uv forecast'", + "weather/uv: no curated source filed here speaks to 'uv level'" +] diff --git a/tests/test_answers.py b/tests/test_answers.py index 5bd9435..8f98c0e 100644 --- a/tests/test_answers.py +++ b/tests/test_answers.py @@ -29,7 +29,7 @@ def rows(**kw): def cols(**kw): a = {"name": "daily", "label": "Daily forecast", "kind": "columns", "primary": True, "words": ["forecast", "this week", "daily"], - "columns": [{"path": "daily.time", "label": "Day", "type": "time"}, + "columns": [{"path": "daily.time", "label": "Day", "type": "date"}, # a day column IS a date (v1.3 types every element) {"path": "daily.temperature_2m_max", "label": "High", "type": "number", "unit_path": "daily_units.temperature_2m_max"}]} a.update(kw) diff --git a/tests/test_records_v12.py b/tests/test_records_v12.py index ba1299e..eab7ae0 100644 --- a/tests/test_records_v12.py +++ b/tests/test_records_v12.py @@ -12,7 +12,7 @@ import pytest -from sourcetool.answers import (ANSWERS, AXIS_STEPS, MEASURES, WINDOWS, answer_lints, answer_problems, +from sourcetool.answers import (ANSWERS, AXIS_STEPS, MEASURES, WINDOWS, answer_lints, answer_problems, exempt_key, check_sample, params_of) from sourcetool.common import ROOT, SOURCES, read_jsonl from sourcetool.schema import resolver_host_value_problems, validate_record @@ -86,7 +86,7 @@ def test_answers_files_pass_the_schema_and_the_lints(): if sid not in RECORDS: continue # a harvested record's file: its own suite problems += [f"{sid}: {e}" for e in answer_problems(d["answers"], params_of(RECORDS[sid]))] - problems += [e for e in answer_lints(_merged(sid)) if e.split(" has no")[0] not in exempt] + problems += [e for e in answer_lints(_merged(sid)) if exempt_key(e) not in exempt] assert not problems, problems assert all(v for v in exempt.values()), "every exemption names its reason" diff --git a/tests/test_tooling_v12.py b/tests/test_tooling_v12.py index 52a3c4e..8e607c4 100644 --- a/tests/test_tooling_v12.py +++ b/tests/test_tooling_v12.py @@ -69,7 +69,8 @@ def no_fetch(*_a, **_k): monkeypatch.setattr(answers_mod, "fetch_sample", no_fetch) rc = answers_mod.cmd_check(["--lint"]) out = capsys.readouterr().out - assert rc in (0, 1) and "lint" in out.splitlines()[-1] + assert rc == 0, out # the gate: the repo's own data passes its lints (kinds served, utc, keyword baseline) + assert "lint" in out.splitlines()[-1] # --- the app's identity and decode ----------------------------------------------------------------------