Repository navigation
Expand file tree
/
Copy pathverbdb.py
More file actions
862 lines (756 loc) · 36 KB
/
Copy pathverbdb.py
File metadata and controls
862 lines (756 loc) · 36 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
#!/usr/bin/env python3
"""Verb record store + query API for the VirtualDJ VDJScript reference.
The store (`docs/vdjscript-verbs.json`) is the authored home for per-verb facts:
tier, aliases, surfaces, kind, doc coverage, and — the part that used to be
scattered across the tracker and topical docs — local-test status, confidence,
and evidence. It is hand-editable through `put`, so the record is edited once
instead of promoted into three Markdown files.
The storage format is deliberately behind this CLI. Agents use the stable verbs
(`get`, `put`, `next-incomplete`, `stats`, `search`, `check`); the JSON layout
underneath can change (e.g. to one file per verb) without retraining them.
Bootstrap seeds the store from the existing generated index plus the coverage
audit and the local-test tracker. It is merge-safe: it never overwrites a field
that already carries an authored value.
Usage:
python3 tools/verbdb.py get <name>
python3 tools/verbdb.py put <name> field=value [field=value ...]
python3 tools/verbdb.py next-incomplete
python3 tools/verbdb.py stats
python3 tools/verbdb.py search [term ...] [--surface=X] [--section=X]
[--tier=X] [--status=X] [--kind=X]
[--needs-test] [--format=json] [--limit=N]
python3 tools/verbdb.py bootstrap [--merge | --force]
python3 tools/verbdb.py check
"""
from __future__ import annotations
import difflib
import json
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
STORE = ROOT / "docs" / "vdjscript-verbs.json"
INDEX = ROOT / "docs" / "vdjscript-verb-index.json"
AUDIT = ROOT / "docs" / "Official VDJScript Coverage Audit.md"
TRACKER = ROOT / "docs" / "VDJScript Local Test Tracker.md"
# ---- schema -----------------------------------------------------------------
TEST_STATUSES = {"Untested", "Partial", "Pass", "Fail", "N/A", "Disproved"}
# `Disproved` is not a behaviour result: the name is not a verb on the
# anchored build (Evidence Standards rule 1b). The record is kept so the
# disproof stays addressable, but consumers that list or render verbs skip it.
LIST_FIELDS = {"aliases", "surfaces", "evidence"}
# Comma-splittable list fields. `evidence` is excluded on purpose: its entries are
# prose sentences that contain commas, so one `evidence=` is one entry.
CSV_FIELDS = {"aliases", "surfaces"}
# Fields that accumulate instead of replacing, so recording a second observation
# does not silently discard the first.
APPEND_FIELDS = {"evidence"}
# `kind` is which of onExecute / onQuery the verb's ACTION_ class implements
# (tests/action-contracts.json, cross-checked against the HTTP sweep): `Dual`
# both, `Action` execute only, `Query` query only. The sweep's own outcome
# words (query / action-only / needs-args) are joined at read time as
# `http_probe`, never stored here. `modifier` and `special-control` mark the
# grammar constructs that are not verbs and have no class.
KINDS = {"Action", "Query", "Dual", "modifier", "special-control"}
BOOL_FIELDS = {"official", "needs_test", "blocked"}
# Fields settable via `put`. `forms`/nested contract detail are hand-edited in JSON.
SETTABLE = {
"tier", "section", "canonical", "kind", "description", "example",
"confidence", "test_status", "note",
} | LIST_FIELDS | BOOL_FIELDS
# The 17 hardware-gated Needs-Local-Test names from the coverage audit; these are
# blocked (cannot be closed without hardware) and are skipped by next-incomplete.
HARDWARE_BLOCKED = {
"controllerscreen_deck", "controller_battery", "gemini_waveform_zoomlevel",
"phase_movement", "phase_position", "phase_active", "v7_status",
"rzx_touch", "rzx_touch_x", "rzx_touch_y",
"djc_shift", "djc_button", "djc_button_popup", "djc_button_slider",
"djc_button_select", "djc_panel", "denon_platter",
}
def default_record(name: str, *, official: bool = True) -> dict:
return {
# `official` defaults True because bootstrap only ever seeds names that
# came from the official appendix. A record minted by `put --new` is not
# in the index by definition, so it passes official=False.
"name": name,
"tier": "official-name-only" if official else "unofficial",
"official": official,
"aliases": [],
"surfaces": [],
"test_status": "Untested",
"needs_test": False,
"blocked": False,
"evidence": [],
}
# ---- store IO ---------------------------------------------------------------
def load_store() -> dict:
if not STORE.exists():
return {}
return json.loads(STORE.read_text())["verbs"]
def counts(store: dict) -> dict:
by_tier: dict[str, int] = {}
by_status: dict[str, int] = {}
documented = tested = needs = blocked = disproved = 0
for rec in store.values():
st = rec.get("test_status", "Untested")
if st == "Disproved":
disproved += 1 # not a verb; kept for the disproof, not counted as one
continue
by_tier[rec.get("tier", "?")] = by_tier.get(rec.get("tier", "?"), 0) + 1
by_status[st] = by_status.get(st, 0) + 1
if rec.get("description"):
documented += 1
if st == "Pass":
tested += 1
if rec.get("needs_test"):
needs += 1
if rec.get("blocked"):
blocked += 1
return {
"total": len(store) - disproved,
"disproved": disproved,
"by_tier": dict(sorted(by_tier.items())),
"by_test_status": dict(sorted(by_status.items())),
"documented": documented,
"tested_pass": tested,
"needs_test": needs,
"blocked": blocked,
"active_incomplete": len(audit_pool(store)),
}
def audit_pool(store: dict) -> list[dict]:
"""The coverage audit's own worklist: records it flagged `needs_test` that
are startable here and not already closed.
Counted by inclusion, never by subtracting `blocked` from `needs_test`: the
two sets overlap only partially (records are blocked without ever having
been flagged `needs_test`), so the subtraction crossed zero and reported a
negative backlog.
"""
return [
r for r in store.values()
if r.get("needs_test") and not r.get("blocked")
and r.get("test_status") != "Pass"
]
def save_store(store: dict) -> None:
payload = {
"_meta": {
"generated_by": "tools/verbdb.py",
"note": "Authoritative per-verb record store. Edit via `just put-verb`, "
"not by hand-writing tables. Seeded by `verbdb.py bootstrap`.",
"counts": counts(store),
},
"verbs": {k: store[k] for k in sorted(store)},
}
STORE.write_text(json.dumps(payload, indent=1, ensure_ascii=False) + "\n")
# ---- source parsers (bootstrap) ---------------------------------------------
def parse_audit_needs_test() -> set[str]:
text = AUDIT.read_text()
m = re.search(r"^## Needs Local Test\b.*?(?=^## )", text, re.S | re.M)
if not m:
return set()
return set(re.findall(r"`([^`]+)`", m.group(0)))
def parse_tracker_results() -> dict[str, dict]:
"""Map verb -> {result, note, build, hardware} from the tracker status tables.
Verb cells may hold several slash-separated backticked names; each maps to
the row's result. A later Pass row wins over an earlier non-Pass one.
The status tables share an 8-column layout ending `… | build | hardware |
Result | Notes |`, so build and hardware sit two and one cells left of the
result cell. Capturing them keeps the record's provenance (which build, what
hardware) instead of discarding it into free-text prose.
"""
rank = {"Untested": 0, "Partial": 1, "Fail": 1, "N/A": 1, "Disproved": 1, "Pass": 2}
out: dict[str, dict] = {}
for line in TRACKER.read_text().splitlines():
if not line.startswith("| `"):
continue
cells = [c.strip() for c in line.strip().strip("|").split("|")]
if len(cells) < 3:
continue
result_idx = next((i for i, c in enumerate(cells[1:], 1)
if c in TEST_STATUSES), None)
if result_idx is None:
continue
result = cells[result_idx]
note = cells[result_idx + 1] if result_idx + 1 < len(cells) else ""
# build/hardware are the two cells before result in the standard layout
build = cells[result_idx - 2] if result_idx >= 2 else ""
hardware = cells[result_idx - 1] if result_idx >= 1 else ""
names = re.findall(r"`([^`]+)`", cells[0])
for raw in names:
name = raw.split()[0] # strip arg examples like `sampler_loaded 8 'auto'`
prev = out.get(name)
if prev is None or rank[result] >= rank[prev["result"]]:
out[name] = {"result": result, "note": note,
"build": build, "hardware": hardware}
return out
def tracker_evidence(row: dict) -> str:
"""One provenance-carrying evidence sentence from a parsed tracker row."""
build = row.get("build") or ""
# the build column sometimes carries prose like "build not recorded"
where = build if build and build.upper() != "TBD" else "build not recorded"
hw = row.get("hardware") or ""
tag = f"Local test ({where}"
if hw and hw.lower() not in {"none required", "none recorded", "none", ""}:
tag += f", hardware: {hw}"
tag += ")"
note = row.get("note") or ""
return f"{tag}: {note}".strip() if note else tag
def bootstrap(mode: str) -> None:
if STORE.exists() and mode not in {"--merge", "--force"}:
sys.exit("store exists; pass --merge (safe: fill gaps only) or --force (rebuild)")
index = json.loads(INDEX.read_text())["verbs"]
needs = parse_audit_needs_test()
results = parse_tracker_results()
existing = {} if mode == "--force" else load_store()
store: dict[str, dict] = {}
for name, idx in index.items():
rec = existing.get(name) or default_record(name)
# merge-safe fill: set derived fields only when the current value is empty/default
def fill(field, value):
if value in (None, "", [], "official-name-only"):
return
cur = rec.get(field)
if cur in (None, "", [], "Untested", False, "official-name-only") \
or (field == "tier" and cur == "official-name-only"):
rec[field] = value
fill("tier", idx.get("tier"))
fill("section", idx.get("section"))
fill("description", idx.get("description"))
fill("example", idx.get("example"))
fill("kind", idx.get("kind"))
if idx.get("surfaces") and not rec.get("surfaces"):
rec["surfaces"] = idx["surfaces"]
if idx.get("aliases") and not rec.get("aliases"):
rec["aliases"] = idx["aliases"]
if idx.get("canonical"):
rec.setdefault("canonical", idx["canonical"])
rec["official"] = idx.get("official", rec.get("official", True))
if name in needs and not rec.get("_authored_needs_test"):
rec["needs_test"] = True
if name in HARDWARE_BLOCKED:
rec["blocked"] = True
if name in results:
row = results[name]
# never downgrade an authored Pass
if rec.get("test_status", "Untested") != "Pass":
rec["test_status"] = row["result"]
if row["note"] and "note" not in rec:
rec["note"] = row["note"][:400]
# Only an actually-tested row is evidence. A tracker row whose result
# is Untested/N/A is a test *plan*, not a result — stamping it
# `Local test` would be a false claim. A tested status must then
# carry structured evidence + confidence (the `just check` gate),
# filled only when empty so a hand-authored entry is never clobbered.
if row["result"] in {"Pass", "Fail", "Partial"}:
if not rec.get("evidence"):
rec["evidence"] = [tracker_evidence(row)[:400]]
if not rec.get("confidence"):
rec["confidence"] = "local_test"
store[name] = rec
# preserve any authored records not present in the index
for name, rec in existing.items():
store.setdefault(name, rec)
save_store(store)
c = counts(store)
print(f"bootstrapped {c['total']} records: "
f"needs_test={c['needs_test']} (blocked={c['blocked']}), "
f"documented={c['documented']}, tested_pass={c['tested_pass']}")
# ---- commands ---------------------------------------------------------------
def joined_view(name: str, rec: dict) -> dict:
"""One-stop contract answer: join the extraction/observation artifacts at
read time (never copied into the store — the artifacts stay authoritative).
Missing artifacts degrade silently to the bare store record."""
out = dict(rec)
def artifact(path):
try:
return json.load(open(ROOT / path))
except (OSError, json.JSONDecodeError):
return None
table = artifact("tests/verb-table.json")
if table:
t = table["verbs"].get(name)
if t:
siblings = [n for n, r in table["verbs"].items()
if r["id"] == t["id"] and n != name]
out["verb_table"] = {**t, "canonical_spelling": t["flags"] != 1,
"editor_hidden": t["flags"] == 256,
**({"same_id_as": siblings} if siblings else {})}
else:
out["verb_table"] = {"present": False,
"meaning": "NOT a verb on this build (rule 1b)"}
contracts = artifact("tests/action-contracts.json")
if contracts and name in contracts["verbs"]:
out["contract"] = contracts["verbs"][name]
sweep = artifact("tests/verb-existence-sweep.json")
if sweep and name in sweep["verbs"]:
s = sweep["verbs"][name]
out["http_probe"] = {k: s[k] for k in ("status", "kind") if k in s}
rtypes = artifact("tests/verb-return-types.json")
if rtypes and name in rtypes["verbs"]:
r = rtypes["verbs"][name]
out["observed_return"] = {"type": r["observed_type"],
"sample": next(iter(r["samples"].values()), None)}
# Where Atomix compiled the verb, from the unstripped build's STABS entries.
# The section backfill copied `clean` modules' sections first, then the
# majority section of `mixed` ones; the grade says how much the section
# owes to the module rather than to the manual's own taxonomy.
modules = artifact("tests/action-modules-9246.json")
if modules:
for module, verbs in modules.get("modules", {}).items():
if name in verbs:
grade = (modules.get("sections", {}).get(module) or {}).get("grade")
out["module"] = {"name": module,
"build": modules["summary"]["build"],
**({"section_grade": grade} if grade else {})}
break
# Sampled compatibility history from the historical installers: which
# sampled builds carried the name, and where its flags or alias peers moved.
# Brackets appearance between samples; it is never a release date.
history = artifact("tests/build-history-2026-09-06/summary.json")
if history and name in history.get("verb_history", {}).get("verbs", {}):
out["history"] = {"samples": history["verb_history"]["samples"],
**history["verb_history"]["verbs"][name]}
vendor = artifact("tests/build-history-2026-09-06/vendor-text-diff.json")
if vendor:
v = {}
if name in vendor["described_only_historically"]:
v["described_only_historically"] = vendor["described_only_historically"][name]
if name in vendor["description_changed"]:
v["description_changed_in"] = sorted(
b for b in vendor["description_changed"][name] if b != "current")
if name in vendor["documented_parameters_lost"]:
v["documented_parameters_lost"] = vendor["documented_parameters_lost"][name]
if name in vendor["skin_usages_lost"]:
v["shipped_skin_usages_lost"] = vendor["skin_usages_lost"][name]
if v:
out["vendor_history"] = v
return out
def cmd_get(args):
if not args:
sys.exit("usage: get <name> [--raw]")
raw = "--raw" in args
args = [a for a in args if a != "--raw"]
store = load_store()
name = args[0]
rec = store.get(name)
view = (lambda n, r: r) if raw else joined_view
# An alias record carries little but a pointer; follow it to the canonical.
if rec is not None and rec.get("tier") == "alias" and rec.get("canonical"):
canon = store.get(rec["canonical"])
if canon:
print(f"# '{name}' is an alias of '{rec['canonical']}'")
print(json.dumps(view(rec["canonical"], canon), indent=1,
ensure_ascii=False))
return
if rec is not None:
print(json.dumps(view(name, rec), indent=1, ensure_ascii=False))
return
# not a record name: maybe it is listed as someone's alias
for n, r in store.items():
if name in r.get("aliases", []):
print(f"# '{name}' is an alias of '{n}'")
print(json.dumps(view(n, r), indent=1, ensure_ascii=False))
return
pool = set(store) | {a for r in store.values() for a in r.get("aliases", [])}
near = difflib.get_close_matches(name, sorted(pool), n=5, cutoff=0.6)
msg = f"no record for '{name}'"
if near:
msg += "\ndid you mean: " + ", ".join(near)
msg += f"\nor try: just list-verbs {name}"
sys.exit(msg)
def coerce(field: str, value: str):
if field in BOOL_FIELDS:
return value.lower() in {"1", "true", "yes", "on"}
if field in CSV_FIELDS:
return [v.strip() for v in value.split(",") if v.strip()]
if field in LIST_FIELDS:
return [value.strip()] if value.strip() else []
if field == "test_status" and value not in TEST_STATUSES:
sys.exit(f"test_status must be one of {sorted(TEST_STATUSES)}")
if field == "kind" and value not in KINDS:
sys.exit(f"kind must be one of {sorted(KINDS)}")
return value
def cmd_put(args):
args = list(args)
allow_new = "--new" in args
if allow_new:
args.remove("--new")
if len(args) < 2:
sys.exit("usage: put <name> field=value [field=value ...] [--new]")
store = load_store()
name = args[0]
rec = store.get(name)
if rec is None:
# Minting a record for an unknown name must be deliberate: otherwise a
# typo silently becomes a junk record claiming to be an official verb.
if not allow_new:
msg = (f"'{name}' has no record and is not in the official index.\n"
"If that is a typo, fix it. If the name is real but unofficial "
"(a scope wrapper, a hidden candidate), record it explicitly:\n"
f" just put-verb {name} --new field=value ...")
near = difflib.get_close_matches(name, list(store), n=5, cutoff=0.6)
if near:
msg += "\ndid you mean: " + ", ".join(near)
sys.exit(msg)
rec = default_record(name, official=False)
for pair in args[1:]:
if "=" not in pair:
sys.exit(f"bad assignment '{pair}' (want field=value)")
field, _, value = pair.partition("=")
if field not in SETTABLE:
sys.exit(f"field '{field}' not settable; allowed: {sorted(SETTABLE)}")
new = coerce(field, value)
if field in APPEND_FIELDS:
kept = [v for v in rec.get(field, []) if v not in new]
rec[field] = kept + new
else:
rec[field] = new
if field == "needs_test":
rec["_authored_needs_test"] = True
store[name] = rec
save_store(store)
print(json.dumps(rec, indent=1, ensure_ascii=False))
def priority(rec: dict) -> tuple:
# lower sorts first: prefer documented, higher-tier, non-blocked
tier_rank = {"curated": 0, "catalog": 1, "alias": 2, "official-name-only": 3}
return (
0 if rec.get("description") else 1,
tier_rank.get(rec.get("tier"), 9),
rec.get("name", ""),
)
# Tier-1 contract artifacts, joined at read time like every other artifact so
# nothing derived from them is ever written into the store.
CONTRACT_ARTIFACTS = ("tests/verb-return-types.json", "tests/verb-arg-forms.json")
def contract_names(paths=CONTRACT_ARTIFACTS) -> set[str]:
"""Names carrying Tier-1 contract evidence: a return-type row, an
argument-form row, or both."""
names: set[str] = set()
for rel in paths:
try:
names |= set(json.load(open(ROOT / rel))["verbs"])
except (OSError, KeyError, json.JSONDecodeError):
continue
return names
def contract_gap_pool(store: dict, known: set[str]) -> list[dict]:
"""Records with no Tier-1 contract evidence at all — task 10's worklist.
An alias inherits its canonical's contract, and hardware-blocked names are
skipped for the same reason the audit pool skips them.
"""
return [
r for r in store.values()
if r["name"] not in known
and (r.get("canonical") or r["name"]) not in known
and not r.get("blocked")
and r["name"] not in HARDWARE_BLOCKED
]
def select_next(store: dict, known: set[str]) -> tuple[list[dict], str]:
"""The audit pool first; once it is empty, fall through to the contract
gap. Returns the sorted pool and its label, so a reader can tell an
'audit gap' pick from a 'contract gap' one instead of reading an empty
selector as completion."""
pool, label = audit_pool(store), "audit"
if not pool:
pool, label = contract_gap_pool(store, known), "contract-gap"
return sorted(pool, key=priority), label
POOL_DESC = {
"audit": "coverage-audit needs_test",
"contract-gap": "no return-type or argument-form row (task 10)",
}
def cmd_next_incomplete(args):
store = load_store()
candidates, pool = select_next(store, contract_names())
if not candidates:
print("no active incomplete verbs in either pool; "
"check `stats` for the blocked/hardware queue")
return
print(json.dumps(candidates[0], indent=1, ensure_ascii=False))
print(f"\n# pool: {pool} ({POOL_DESC[pool]}), {len(candidates)} item(s)",
file=sys.stderr)
if len(candidates) > 1:
rest = ", ".join(r["name"] for r in candidates[1:8])
print(f"# next up: {rest}"
f"{' ...' if len(candidates) > 9 else ''}", file=sys.stderr)
def cmd_stats(args):
store = load_store()
out = counts(store)
# The pool split is a query, not stored state: the contract gap is derived
# from artifacts that regenerate, so it is reported and never written.
out["incomplete_pools"] = {
"audit": len(audit_pool(store)),
"contract_gap": len(contract_gap_pool(store, contract_names())),
}
print(json.dumps(out, indent=1, ensure_ascii=False))
def cmd_uncategorized(args):
"""Records with no `section`, with the b9246 module where one exists.
Aliases are skipped: `get` follows them to the canonical, whose section is
the one that counts. This is a query on the store; nothing is written."""
as_json = "--format=json" in args
store = load_store()
modules = module_map()
rows = []
for name, rec in sorted(store.items()):
if rec.get("section") or rec.get("tier") == "alias":
continue
rows.append({"name": name, "tier": rec.get("tier"),
"kind": rec.get("kind"), "module": modules.get(name)})
if as_json:
print(json.dumps(rows, indent=1, ensure_ascii=False))
return
if not rows:
print("every non-alias record has a section")
return
width = max(len(r["name"]) for r in rows)
for r in rows:
print(f"{r['name']:{width}} {r['tier'] or '-':20} {r['kind'] or '-':8} "
f"{r['module'] or '(no ACTION_ class on b9246)'}")
print(f"\n# {len(rows)} uncategorized non-alias record(s); "
"set one with `just put-verb <name> section='<Section>'`", file=sys.stderr)
def cmd_sections(args):
"""The section vocabulary, as a query: each section with how many non-alias
records it holds and how many of those have a tested status. Aliases are
left out because they carry no section of their own. Uncategorized records
appear last as `(none)` so the gap is part of the same view."""
as_json = "--format=json" in args
store = load_store()
rows: dict[str, dict] = {}
for rec in store.values():
if rec.get("tier") == "alias" or rec.get("test_status") == "Disproved":
continue
key = rec.get("section") or "(none)"
row = rows.setdefault(key, {"section": key, "verbs": 0, "tested": 0})
row["verbs"] += 1
if rec.get("test_status") not in (None, "", "Untested"):
row["tested"] += 1
ordered = sorted(rows.values(), key=lambda r: (r["section"] == "(none)", -r["verbs"], r["section"]))
if as_json:
print(json.dumps(ordered, indent=1, ensure_ascii=False))
return
width = max(len(r["section"]) for r in ordered)
print(f"{'section':{width}} verbs tested")
for r in ordered:
print(f"{r['section']:{width}} {r['verbs']:5d} {r['tested']:6d}")
print(f"\n# {len(ordered) - ('(none)' in rows)} sections; filter with "
"`just list-verbs --section='<name>'`", file=sys.stderr)
FILTERS = {"surface", "section", "tier", "status", "kind", "module"}
def module_map() -> dict[str, str]:
"""verb -> module, joined at read time like every other artifact."""
try:
art = json.load(open(ROOT / "tests/action-modules-9246.json"))
except (OSError, json.JSONDecodeError):
return {}
return {v: module for module, verbs in art.get("modules", {}).items() for v in verbs}
def cmd_search(args):
"""Filtered query. Terms AND with filters; category listings are just an
unfiltered query, so no grouped dump needs to exist on disk."""
terms, opts = [], {}
fmt, limit = "table", 0
for a in args:
if a.startswith("--"):
key, _, val = a[2:].partition("=")
if key == "format":
fmt = val
elif key == "limit":
limit = int(val) if val else 0
elif key == "needs-test":
opts["needs_test"] = True
elif key in FILTERS:
opts[key] = val
else:
sys.exit(f"unknown option --{key}; filters: "
f"{sorted(FILTERS)} + --needs-test, --format, --limit")
else:
terms.append(a.lower())
store = load_store()
modules = module_map() if "module" in opts else {}
hits = []
for name, rec in store.items():
# Disproved names are not verbs; list them only when asked for by status.
if rec.get("test_status") == "Disproved" and \
"disproved" not in opts.get("status", "").lower():
continue
if opts.get("needs_test") and not rec.get("needs_test"):
continue
if "module" in opts and opts["module"].lower() not in modules.get(name, "").lower():
continue
if "surface" in opts and opts["surface"].lower() not in \
[s.lower() for s in rec.get("surfaces", [])]:
continue
for key, field in (("section", "section"), ("tier", "tier"),
("status", "test_status"), ("kind", "kind")):
if key in opts and opts[key].lower() not in \
str(rec.get(field, "")).lower():
break
else:
hay = " ".join([
name, rec.get("description", ""),
" ".join(rec.get("aliases", [])), rec.get("section", ""),
]).lower()
if all(t in hay for t in terms):
hits.append(rec)
hits.sort(key=priority)
shown = hits[:limit] if limit else hits
if fmt == "json":
print(json.dumps(shown, indent=1, ensure_ascii=False))
else:
# name, section, [tier/status/kind], description, aliases. The section
# column shows `-` for an uncategorized record so the gap is visible in
# any listing, not only in `uncategorized`.
width = max((len(r.get("section") or "-") for r in shown), default=1)
for rec in shown:
al = f" aliases={','.join(rec['aliases'])}" if rec.get("aliases") else ""
tag = f"{rec.get('tier','?')}/{rec.get('test_status','?')}"
if rec.get("kind"):
tag += f"/{rec['kind']}"
print(f"{rec['name']:<28} {rec.get('section') or '-':<{width}} [{tag}] "
f"{rec.get('description','')}{al}")
where = " ".join(terms) + " " + " ".join(f"--{k}={v}" for k, v in opts.items())
note = f"showing {len(shown)} of {len(hits)}" if len(shown) != len(hits) \
else f"{len(hits)} match(es)"
print(f"\n{note} for [{where.strip()}]", file=sys.stderr)
SELECTION_FIXTURES = ROOT / "tests" / "verbdb-selection"
def selection_selftest() -> list[str]:
"""Replay the selector over the recorded stores.
Both failures this guards against were silent: a backlog that went negative
and read as done, and a selector that printed nothing while most of the
store had no contract evidence at all.
"""
expectations = json.loads((SELECTION_FIXTURES / "expected.json").read_text())
failures: list[str] = []
for fname, want in sorted(expectations.items()):
fixture = json.loads((SELECTION_FIXTURES / fname).read_text())
store, known = fixture["store"], set(fixture["contract_names"])
got_counts = counts(store)
if got_counts["active_incomplete"] < 0:
failures.append(f"{fname}: active_incomplete is negative "
f"({got_counts['active_incomplete']})")
if got_counts["active_incomplete"] != want["active_incomplete"]:
failures.append(f"{fname}: active_incomplete "
f"{got_counts['active_incomplete']}, "
f"expected {want['active_incomplete']}")
got_pools = {"audit": len(audit_pool(store)),
"contract_gap": len(contract_gap_pool(store, known))}
if got_pools != want["pools"]:
failures.append(f"{fname}: pools {got_pools}, expected {want['pools']}")
pool_recs, pool = select_next(store, known)
got_next = pool_recs[0]["name"] if pool_recs else None
if got_next != want["next"]:
failures.append(f"{fname}: selected {got_next!r}, "
f"expected {want['next']!r}")
if pool != want["pool"]:
failures.append(f"{fname}: picked from pool {pool!r}, "
f"expected {want['pool']!r}")
return failures
def cmd_check(args):
if not STORE.exists():
sys.exit("store missing; run `python3 tools/verbdb.py bootstrap`")
store = load_store()
index = json.loads(INDEX.read_text())["verbs"]
errors = []
# coverage: every official index name has a record
missing = set(index) - set(store)
if missing:
errors.append(f"{len(missing)} index names missing from store: "
f"{sorted(missing)[:5]}...")
# coverage: every verb-table name has a record, and every record absent
# from the verb table must say why it is in a *verb* store at all
# (disproven, or a non-verb construct kept for discoverability).
table_path = ROOT / "tests" / "verb-table.json"
if table_path.exists():
table = set(json.loads(table_path.read_text())["verbs"])
known = set(store) | {a for r in store.values() for a in r.get("aliases", [])}
gap = table - known
if gap:
errors.append(f"{len(gap)} verb-table names missing from store: "
f"{sorted(gap)[:5]}...")
for name in sorted(set(store) - table):
rec = store[name]
if rec.get("test_status") == "Disproved":
continue # disproven names stay, carrying their disproof
if rec.get("kind") in {"modifier", "special-control"}:
continue # grammar/mapper constructs, not verbs
if "scope wrapper" in (rec.get("note") or ""):
continue
errors.append(f"{name}: not in the verb table and not marked as a "
f"disproven name or non-verb construct (rule 1b)")
for name, rec in store.items():
if rec.get("name") != name:
errors.append(f"{name}: record 'name' mismatch ({rec.get('name')})")
st = rec.get("test_status", "Untested")
if st not in TEST_STATUSES:
errors.append(f"{name}: invalid test_status '{st}'")
if rec.get("kind") is not None and rec["kind"] not in KINDS:
errors.append(f"{name}: invalid kind '{rec['kind']}' "
f"(one of {sorted(KINDS)})")
# A tested status must carry its proof as structured evidence, not just a
# free-text note. This is the gate that keeps status and evidence from
# drifting apart the way the bootstrap-seeded records once did.
if st in {"Pass", "Fail", "Partial", "Disproved"} and not rec.get("evidence"):
errors.append(f"{name}: test_status '{st}' but no evidence "
f"(record the proof with `just put-verb {name} "
f"evidence=\"…\"`)")
canon = rec.get("canonical")
if canon and canon not in store:
errors.append(f"{name}: canonical '{canon}' has no record")
for al in rec.get("aliases", []):
if al not in store and al not in index:
errors.append(f"{name}: alias '{al}' unresolved")
errors.extend(f"selection selftest: {f}" for f in selection_selftest())
# staleness: store counts must match what save would write
payload = json.loads(STORE.read_text())
if payload["_meta"]["counts"] != counts(store):
errors.append("stored _meta.counts is stale; re-save via any put or bootstrap --merge")
if errors:
print("verbdb check FAILED:")
for e in errors[:40]:
print(f" - {e}")
sys.exit(1)
c = counts(store)
gap = len(contract_gap_pool(store, contract_names()))
print(f"verbdb check passed: {c['total']} records, "
f"needs_test={c['needs_test']} (blocked={c['blocked']}), "
f"tested_pass={c['tested_pass']}, "
f"incomplete: {c['active_incomplete']} audit / {gap} contract-gap")
COMMANDS = {
"get": cmd_get,
"put": cmd_put,
"next-incomplete": cmd_next_incomplete,
"stats": cmd_stats,
"uncategorized": cmd_uncategorized,
"sections": cmd_sections,
"search": cmd_search,
"check": cmd_check,
}
USAGE = """usage: verbdb.py <command> ... | verbdb.py <verb-name>
<verb-name> shorthand for `get <verb-name>`
get <name> one record (follows aliases)
put <name> f=v ... set fields
search [term] [--surface= --section= --tier= --status= --kind= --module=
--needs-test --format=json --limit=N]
next-incomplete next active (non-hardware-blocked) work item
stats counts by tier / test status
sections [--format=json]
the section vocabulary with per-section counts
uncategorized [--format=json]
non-alias records with no section, with their b9246 module
check validate the store
bootstrap [--merge|--force]
Note: `search` is also a VDJScript verb name. Commands win; use
`verbdb.py get search` for the verb record."""
def main(argv):
if not argv:
sys.exit(USAGE)
cmd, rest = argv[0], argv[1:]
if cmd in {"-h", "--help", "help"}:
print(USAGE)
return
if cmd == "bootstrap":
bootstrap(rest[0] if rest else "")
return
if cmd in COMMANDS:
COMMANDS[cmd](rest)
return
# Not a command: treat a bare argument as a verb lookup, for direct
# `python3 tools/verbdb.py leftdeck` use. The `just` recipes are flat
# (get-verb / list-verbs / ...) and always pass an explicit subcommand.
cmd_get(argv)
if __name__ == "__main__":
main(sys.argv[1:])