From ea6baaa60a9c30da2066002efc82d496930700cc Mon Sep 17 00:00:00 2001 From: Matt <47545907+SoundMatt@users.noreply.github.com> Date: Fri, 21 Aug 2026 07:02:24 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20continuous=20example=20verification=20(?= =?UTF-8?q?=C2=A717=20Requirement=2019)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New §20.7 "Continuous example verification" and §17 Requirement 19: every JSON example this document marks normative ( marker convention) and every literal, executable command in an implementation's README `## RELAY conformance` section MUST be machine-extracted and checked in CI -- not merely true when someone last looked. Extends §13.8's existing "the demo must work" and §19.5's existing "version literals must match" to cover example *content* more broadly, closing the same class of gap THEME-J and REL-SPEC-10 independently documented. applied to all 7 of this document's existing normative JSON examples (§12.1 version, §12.2 capabilities x2, §12.3 status, §15.8 vectors-manifest, §17.2 manifest, §20.6 attestation). Each is run through the exact same validator relay conform applies to a real binary's output (cmd/relay/doctest.go's extractDoctestExamples + cmd/relay/doctest_test.go's TestSpecDoctestExamplesValidate). Three real bugs found and fixed while building this: - Three of this document's own examples used truncated placeholder hex ("816b402d...") instead of a real 64-char SHA-256. Replaced with genuine values -- two are the actual hashes of the referenced golden vectors. - RELAY's own JSON Schema validator never implemented the `pattern` keyword at all (silently ignored per its own documented keyword list) -- every schema constraining a SHA-256 field with "^[0-9a-f]{64}$" was unenforced. relay conform would have accepted a malformed hash in any capabilities/manifest/attestation/ vectors-manifest document without a single finding. Found because the truncated placeholders above initially validated cleanly -- a false pass this doctest mechanism exists to prevent. Fixed with a regexp-based pattern check in validateSchema; mutation-tested. - README.md's own "Current" version line was six MINOR releases stale (v2.0 while spec/version.json was at v2.8) -- the exact class of drift §19.5 already guards against in relay-spec.md, just not in README.md. Fixed, and TestReadmeVersionMatchesVersionJSON extends the §19.5 mechanism so it can't silently recur (README.md now embedded evidence). New REQ-RELAY-102. Deliberately out of scope: network-fetching commands (go get/go install) and commands needing a published container image are exempt from README example execution -- a hermetic runner for those is a reasonable future addition, not mandated here. Mutation-tested the doctest mechanism itself: corrupted a hash pattern and a required commands entry in two separate examples, confirmed both caught with a precise violation message, restored and reconfirmed green. SpecVersion 2.8 -> 2.9 (MINOR, new §17 requirement). Closes #146 Signed-off-by: Matt <47545907+SoundMatt@users.noreply.github.com> --- .fusa-reqs.json | 10 ++ README.md | 2 +- cmd/relay/conform.go | 1 + cmd/relay/conform_test.go | 14 +-- cmd/relay/doctest.go | 54 +++++++++++ cmd/relay/doctest_test.go | 93 +++++++++++++++++++ cmd/relay/jsonschema.go | 12 ++- cmd/relay/jsonschema_test.go | 24 +++++ cmd/relay/main.go | 2 +- evidence.go | 4 +- evidence_test.go | 1 + spec/CHANGELOG.md | 62 +++++++++++++ spec/relay-spec.md | 112 +++++++++++++++++++---- spec/schemas/relay-conform-manifest.json | 2 +- spec/version.json | 2 +- version.go | 2 +- version_consistency_test.go | 26 ++++++ version_test.go | 4 +- 18 files changed, 392 insertions(+), 35 deletions(-) create mode 100644 cmd/relay/doctest.go create mode 100644 cmd/relay/doctest_test.go diff --git a/.fusa-reqs.json b/.fusa-reqs.json index e321d24..3b1ff5c 100644 --- a/.fusa-reqs.json +++ b/.fusa-reqs.json @@ -961,6 +961,16 @@ "verification": "test", "safety_goal": "SG-006", "asil": "ASIL-B" + }, + { + "id": "REQ-RELAY-102", + "title": "Continuous example verification (doctest, spec sec-20.7)", + "text": "A fenced json code block in the specification document immediately preceded by an HTML comment on its own line MUST be extracted by this repository's own CI and validated against KIND's rules (version/capabilities/status document rules, or the relay-conform-manifest/relay-conform-attestation/vectors-manifest schema) using the same validator relay conform applies to a real binary's output. A build MUST fail if a marked example violates its validator, or if a KIND has zero examples found. Per spec sec-13.8, an implementation's README RELAY conformance section MUST similarly have every literal, executable command example machine-extracted and executed in its own CI, excluding network-fetching commands (go get/go install) and commands requiring a published container image, which are explicitly out of this requirement's scope.", + "category": "functional", + "criticality": "medium", + "verification": "test", + "safety_goal": "SG-006", + "asil": "ASIL-B" } ] } diff --git a/README.md b/README.md index 8ff41ed..bb7cb72 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Full specification: [`spec/relay-spec.md`](spec/relay-spec.md) Machine-readable version: [`spec/version.json`](spec/version.json) Change history: [`spec/CHANGELOG.md`](spec/CHANGELOG.md) -Current: **v2.0 (stable)** +Current: **v2.9 (stable)** ## Install diff --git a/cmd/relay/conform.go b/cmd/relay/conform.go index 4ea7195..9208775 100644 --- a/cmd/relay/conform.go +++ b/cmd/relay/conform.go @@ -336,6 +336,7 @@ func buildManifest(binary string) conformManifest { {16, statusNotObservable, verifierCI}, {17, req17, verifierRelayConform}, {18, statusNotObservable, verifierCI}, + {19, statusNotObservable, verifierCI}, } m.Overall = statusPass diff --git a/cmd/relay/conform_test.go b/cmd/relay/conform_test.go index 706dc4e..2efa4a9 100644 --- a/cmd/relay/conform_test.go +++ b/cmd/relay/conform_test.go @@ -382,8 +382,8 @@ func TestBuildManifestSelf(t *testing.T) { if m.SpecVersion == "" { t.Error("SpecVersion is empty") } - if len(m.Requirements) != 18 { - t.Fatalf("len(Requirements) = %d, want 18", len(m.Requirements)) + if len(m.Requirements) != 19 { + t.Fatalf("len(Requirements) = %d, want 19", len(m.Requirements)) } for i, r := range m.Requirements { if r.ID != i+1 { @@ -414,7 +414,7 @@ func TestBuildManifestNotObservableDefaults(t *testing.T) { bin := buildTestBinary(t) m := buildManifest(bin) - wantNotObservable := []int{2, 3, 4, 5, 8, 9, 10, 11, 13, 14, 15, 16, 18} + wantNotObservable := []int{2, 3, 4, 5, 8, 9, 10, 11, 13, 14, 15, 16, 18, 19} // Requirement 17 is fully observable (see TestBuildManifestSelf's // observable set), so it is deliberately excluded from this list. // Requirement 18 (persisted interop matrix) is verified by a @@ -513,8 +513,8 @@ func TestRunConformManifestFlag(t *testing.T) { if m.Overall != statusPass { t.Errorf("Overall = %s, want %s", m.Overall, statusPass) } - if len(m.Requirements) != 18 { - t.Errorf("len(Requirements) = %d, want 18", len(m.Requirements)) + if len(m.Requirements) != 19 { + t.Errorf("len(Requirements) = %d, want 19", len(m.Requirements)) } } @@ -575,8 +575,8 @@ func TestBuildAttestationSelf(t *testing.T) { if a.Predicate.ConformanceManifest.Kind != "relay-conform-manifest" { t.Errorf("Predicate.ConformanceManifest.Kind = %q, want relay-conform-manifest", a.Predicate.ConformanceManifest.Kind) } - if len(a.Predicate.ConformanceManifest.Requirements) != 18 { - t.Errorf("len(Predicate.ConformanceManifest.Requirements) = %d, want 18", len(a.Predicate.ConformanceManifest.Requirements)) + if len(a.Predicate.ConformanceManifest.Requirements) != 19 { + t.Errorf("len(Predicate.ConformanceManifest.Requirements) = %d, want 19", len(a.Predicate.ConformanceManifest.Requirements)) } // §20.6: MUST be false — this type has no signing mechanism at all. if a.Predicate.Signed { diff --git a/cmd/relay/doctest.go b/cmd/relay/doctest.go new file mode 100644 index 0000000..ed2e961 --- /dev/null +++ b/cmd/relay/doctest.go @@ -0,0 +1,54 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package main + +import ( + "fmt" + "regexp" + "strings" +) + +// doctestMarkerRe matches a `` comment line marking the +// fenced ```json block that immediately follows as a normative example this +// document commits to keeping genuinely valid (spec §20.7, §17 Requirement 19). +var doctestMarkerRe = regexp.MustCompile(``) + +// extractDoctestExamples scans a spec document line-by-line for +// `` markers and returns the JSON bytes of the fenced +// ```json block that follows each one (skipping blank lines in between). A +// marker not followed by a properly-terminated fence is reported as an +// error — a silently-skipped marker would defeat the whole point of this +// mechanism: an example that's supposed to be checked but never actually is. +// +//fusa:req REQ-RELAY-102 +func extractDoctestExamples(spec string) (map[string][][]byte, error) { + lines := strings.Split(spec, "\n") + out := map[string][][]byte{} + for i := 0; i < len(lines); i++ { + m := doctestMarkerRe.FindStringSubmatch(lines[i]) + if m == nil { + continue + } + kind := m[1] + j := i + 1 + for j < len(lines) && strings.TrimSpace(lines[j]) == "" { + j++ + } + if j >= len(lines) || strings.TrimSpace(lines[j]) != "```json" { + return nil, fmt.Errorf("doctest:%s marker at line %d is not followed by a ```json fence", kind, i+1) + } + j++ + start := j + for j < len(lines) && strings.TrimSpace(lines[j]) != "```" { + j++ + } + if j >= len(lines) { + return nil, fmt.Errorf("doctest:%s marker at line %d: unterminated json fence", kind, i+1) + } + out[kind] = append(out[kind], []byte(strings.Join(lines[start:j], "\n"))) + i = j + } + return out, nil +} diff --git a/cmd/relay/doctest_test.go b/cmd/relay/doctest_test.go new file mode 100644 index 0000000..59806da --- /dev/null +++ b/cmd/relay/doctest_test.go @@ -0,0 +1,93 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package main + +import ( + "encoding/json" + "testing" + + relay "github.com/SoundMatt/RELAY/v2" +) + +// TestSpecDoctestExamplesValidate is spec §20.7's mechanism: every JSON +// example marked `` in relay-spec.md is extracted and +// run through the exact same validator relay conform itself uses on a real +// binary's output — not a relaxed or hand-copied check. A stale or +// hand-broken example (the class of drift that produced NEW-SPEC-6's "0.1" +// incident) fails this test, not just a human reader's attention. +// +//fusa:test REQ-RELAY-102 +func TestSpecDoctestExamplesValidate(t *testing.T) { + specRaw, err := relay.Evidence("specification") + if err != nil { + t.Fatalf("Evidence(specification): %v", err) + } + examples, err := extractDoctestExamples(string(specRaw)) + if err != nil { + t.Fatal(err) + } + + // Sanity: catches the marker convention silently falling out of sync + // with the document (all markers removed, or the kind renamed) rather + // than the loop below quietly validating zero examples and passing. + for _, kind := range []string{"version", "capabilities", "status", "manifest", "attestation", "vectors-manifest"} { + if len(examples[kind]) == 0 { + t.Errorf("no doctest:%s example found in spec/relay-spec.md", kind) + } + } + + for _, ex := range examples["version"] { + assertDoctestNoFail(t, "version", ex, validateVersionDoc(ex)) + } + for _, ex := range examples["capabilities"] { + assertDoctestNoFail(t, "capabilities", ex, validateCapabilitiesDoc(ex)) + } + for _, ex := range examples["status"] { + assertDoctestNoFail(t, "status", ex, validateStatusDoc(ex)) + } + for _, ex := range examples["manifest"] { + assertDoctestSchemaValid(t, "manifest", "relay-conform-manifest", ex) + } + for _, ex := range examples["attestation"] { + assertDoctestSchemaValid(t, "attestation", "relay-conform-attestation", ex) + } + for _, ex := range examples["vectors-manifest"] { + assertDoctestSchemaValid(t, "vectors-manifest", "vectors-manifest", ex) + } +} + +//fusa:test REQ-RELAY-102 +func TestExtractDoctestExamplesUnterminatedFence(t *testing.T) { + if _, err := extractDoctestExamples("\n```json\n{\"a\":1}\n"); err == nil { + t.Error("expected an error for an unterminated json fence") + } + if _, err := extractDoctestExamples("\nnot a fence\n"); err == nil { + t.Error("expected an error when the marker isn't followed by a ```json fence") + } +} + +func assertDoctestNoFail(t *testing.T, kind string, example []byte, findings []conformFinding) { + t.Helper() + if hasFail(findings) { + t.Errorf("doctest:%s example failed validation: %+v\n%s", kind, findings, example) + } +} + +func assertDoctestSchemaValid(t *testing.T, kind, schemaName string, example []byte) { + t.Helper() + var asAny interface{} + if err := json.Unmarshal(example, &asAny); err != nil { + t.Errorf("doctest:%s example is not valid JSON: %v\n%s", kind, err, example) + return + } + schemaJSON, err := relay.Schema(schemaName) + if err != nil { + t.Fatalf("doctest:%s: no embedded schema %q: %v", kind, schemaName, err) + } + violations := validateSchema(schemaJSON, asAny) + if len(violations) > 0 { + t.Errorf("doctest:%s example violates %s schema: %v\n%s", kind, schemaName, violations, example) + } +} diff --git a/cmd/relay/jsonschema.go b/cmd/relay/jsonschema.go index 009cccb..77f15bf 100644 --- a/cmd/relay/jsonschema.go +++ b/cmd/relay/jsonschema.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "math" + "regexp" "sort" "strings" ) @@ -19,7 +20,7 @@ import ( // // type (string or []string), properties, required, additionalProperties:false, // enum, const, minimum, maximum, items (single schema), minItems, maxItems, -// contains (single schema). +// contains (single schema), pattern (string values, RE2 syntax). // // Unsupported keywords are ignored (they do not cause false failures). @@ -75,6 +76,15 @@ func validateValue(schema map[string]interface{}, v interface{}, path string, ou if max, ok := numberOf(schema["maximum"]); ok && val > max { *out = append(*out, fmt.Sprintf("%s: %v > maximum %v", loc, val, max)) } + case string: + if pat, ok := schema["pattern"].(string); ok { + re, err := regexp.Compile(pat) + if err != nil { + *out = append(*out, fmt.Sprintf("%s: schema pattern %q does not compile: %v", loc, pat, err)) + } else if !re.MatchString(val) { + *out = append(*out, fmt.Sprintf("%s: value %q does not match pattern %q", loc, val, pat)) + } + } case map[string]interface{}: validateObject(schema, val, path, out) case []interface{}: diff --git a/cmd/relay/jsonschema_test.go b/cmd/relay/jsonschema_test.go index 61c3551..6cc4491 100644 --- a/cmd/relay/jsonschema_test.go +++ b/cmd/relay/jsonschema_test.go @@ -89,6 +89,30 @@ func TestSchemaMinMax(t *testing.T) { } } +//fusa:test REQ-RELAY-058 +func TestSchemaPattern(t *testing.T) { + // SHA-256 hex, the pattern every schema in spec/schemas/ that declares a + // digest field uses. Found genuinely unenforced (silently ignored) while + // building the §20.7 doctest mechanism: a document with a truncated or + // non-hex hash passed schema validation with zero violations before this. + schema := `{"type":"string","pattern":"^[0-9a-f]{64}$"}` + if v := validateSchema([]byte(schema), "816b402dca5070ec011863b8d8eb3b97abbf2ea8cf264706113e06fcceb4fe27"); len(v) != 0 { + t.Errorf("a genuine 64-char lowercase hex string should satisfy the pattern, got %v", v) + } + if v := validateSchema([]byte(schema), "816b402d..."); len(v) == 0 { + t.Error("a truncated hash with a literal ellipsis should violate the pattern") + } + if v := validateSchema([]byte(schema), "816B402DCA5070EC011863B8D8EB3B97ABBF2EA8CF264706113E06FCCEB4FE27"); len(v) == 0 { + t.Error("uppercase hex should violate a lowercase-only pattern") + } + // An invalid regex must be reported as a violation, not silently ignored + // or panic — a malformed schema is a real (if different) defect. + bad := `{"type":"string","pattern":"("}` + if v := validateSchema([]byte(bad), "anything"); len(v) == 0 { + t.Error("an uncompilable pattern should itself be reported as a violation") + } +} + //fusa:test REQ-RELAY-058 func TestSchemaArrayItemsAndContains(t *testing.T) { schema := `{"type":"array","items":{"type":"string"},"contains":{"const":"version"}}` diff --git a/cmd/relay/main.go b/cmd/relay/main.go index d9d40a5..11a44de 100644 --- a/cmd/relay/main.go +++ b/cmd/relay/main.go @@ -18,7 +18,7 @@ import ( relay "github.com/SoundMatt/RELAY/v2" ) -const toolVersion = "2.8.0" +const toolVersion = "2.9.0" func main() { if err := run(os.Stdout, os.Stderr, os.Args[1:]); err != nil { diff --git a/evidence.go b/evidence.go index cc66d81..282a21b 100644 --- a/evidence.go +++ b/evidence.go @@ -27,13 +27,14 @@ import ( //fusa:req REQ-RELAY-093 //fusa:req REQ-RELAY-094 //fusa:req REQ-RELAY-100 +//fusa:req REQ-RELAY-102 //fusa:req REQ-RELAY-074 //fusa:req REQ-RELAY-075 //fusa:req REQ-RELAY-076 //go:embed .fusa-reqs.json .fusa-hara.json .fusa-tara.json //go:embed spec/version.json docs/tool-safety-manual.md //go:embed docs/formal/RelayLifecycle.tla docs/formal/README.md -//go:embed docs/asil-d-uplift.md spec/relay-spec.md +//go:embed docs/asil-d-uplift.md spec/relay-spec.md README.md var evidenceFS embed.FS // evidencePaths maps the logical evidence name to its embedded path. @@ -47,6 +48,7 @@ var evidencePaths = map[string]string{ "formal-model-doc": "docs/formal/README.md", "asil-d-uplift": "docs/asil-d-uplift.md", "specification": "spec/relay-spec.md", + "readme": "README.md", } // Evidence returns the raw bytes of a named safety-evidence artifact, e.g. diff --git a/evidence_test.go b/evidence_test.go index 50a0174..6b28a95 100644 --- a/evidence_test.go +++ b/evidence_test.go @@ -19,6 +19,7 @@ func TestEvidenceNames(t *testing.T) { "version": true, "tool-safety-manual": true, "formal-model": true, "formal-model-doc": true, "asil-d-uplift": true, "specification": true, + "readme": true, } if len(names) != len(want) { t.Fatalf("EvidenceNames = %v, want %d entries", names, len(want)) diff --git a/spec/CHANGELOG.md b/spec/CHANGELOG.md index 71d6def..103ae1a 100644 --- a/spec/CHANGELOG.md +++ b/spec/CHANGELOG.md @@ -1,5 +1,67 @@ # RELAY Spec Changelog +## v2.9 — 2026-08-21 (MINOR — continuous example verification, §17 Requirement 19) + +- **New §20.7 "Continuous example verification"** and **new §17 + Requirement 19**. §13.8 already required a README's documented CLI + round-trip to work; §19.5 already required this document's own + version literals to stay in sync. Neither stopped a *content* example + (not just a version number) from silently rotting once nobody was + reading closely — exactly the failure mode THEME-J and REL-SPEC-10 + independently documented. This closes it the same way §19.5 did: no + example ships unverified. +- **`` marker convention** for this document's own + normative JSON examples. `KIND` ∈ `{version, capabilities, status, + manifest, attestation, vectors-manifest}` selects which validator a + marked example is run through — the *same* validator `relay conform` + applies to a real binary's output, not a separate, looser check + invented for the doctest alone. Applied to all 7 existing normative + examples (§12.1, §12.2 ×2, §12.3, §15.8, §17.2, §20.6). +- **README `## RELAY conformance` sections** must now have every + literal, executable command example machine-extracted and executed + in CI (§13.8 extended). Network-fetching commands (`go get`/`go + install`) and commands needing a published container image are + explicitly out of scope — mandating a hermetic runner for those isn't + this section's job. +- **Real bugs found while building this, fixed along the way**: + - Three of this document's own examples (§15.8's two vector hashes, + §17.2's `capabilities_sha256`, §20.6's subject digest) used + truncated placeholder hex (`"816b402d..."`) instead of a real + 64-character SHA-256. Replaced with genuine values — two are the + *actual* hashes of the referenced golden vectors; the other two are + real SHA-256 outputs (of `""` and `"test"` respectively), chosen so + the "before" value was already a recognizable prefix of the "after" + one. + - **RELAY's own JSON Schema validator never implemented the + `pattern` keyword at all** — silently ignored, per its own + documented keyword list. Every schema in `spec/schemas/` that + constrains a SHA-256 field with `"pattern": "^[0-9a-f]{64}$"` was + unenforced: `relay conform` would have accepted a malformed hash in + any capabilities/manifest/attestation/vectors-manifest document + without a single finding. Found because the truncated placeholder + examples above initially validated cleanly — a false pass this + doctest mechanism was specifically built to prevent. Fixed with a + `regexp`-based `pattern` check in `validateSchema`; new + `TestSchemaPattern` unit test; mutation-tested (confirmed the fix + catches both the truncated hash and case-sensitivity violations, + and that an uncompilable pattern is itself reported rather than + silently ignored or panicking). + - **README.md's own "Current" version line was six MINOR releases + stale** (`v2.0`, while `spec/version.json` was already at `v2.8`) — + the exact class of drift §19.5 exists to prevent, just in a file + §19.5 doesn't cover. Fixed to the version this release ships, and + `TestReadmeVersionMatchesVersionJSON` extends the §19.5 mechanism + (new `REQ-RELAY-102`, `README.md` now embedded evidence) so it + can't silently recur. +- **Reference implementation**: `extractDoctestExamples` + (`cmd/relay/doctest.go`), `TestSpecDoctestExamplesValidate` + (`cmd/relay/doctest_test.go`). Mutation-tested: corrupted a hash + pattern and a required `commands` entry in two separate examples, + confirmed both were caught with a precise violation message, restored + and reconfirmed green. +- `SpecVersion` bumped `2.8` → `2.9` (MINOR, new §17 requirement). + Closes [NEW-SPEC-8]. + ## v2.8 — 2026-08-21 (MINOR — persisted, regression-gated interop matrix, §17 Requirement 18) - **New §17 Requirement 18 — persisted, regression-gated interop matrix.** diff --git a/spec/relay-spec.md b/spec/relay-spec.md index 18c249e..ec3a627 100644 --- a/spec/relay-spec.md +++ b/spec/relay-spec.md @@ -1,4 +1,4 @@ -# RELAY Specification — v2.8 +# RELAY Specification — v2.9 **Real-time Embedded Link Abstraction Yoke** @@ -1233,13 +1233,14 @@ failure rather than a skip. Exit: `0` all equivalent, `1` any mismatch/error, ` version --format json`: + ```json { "tool": "go-can", "protocol": "CAN", "protocol_int": 1, "version": "1.2.3", - "spec_version": "2.8", + "spec_version": "2.9", "language": "go", "runtime": "go1.25.0", "commit": "a1b2c3d4" @@ -1257,6 +1258,7 @@ conformance failure, but §17.2's conformance manifest cannot populate its own ` capabilities`: + ```json { "kind": "capabilities", @@ -1264,7 +1266,7 @@ conformance failure, but §17.2's conformance manifest cannot populate its own "protocol": "CAN", "protocol_int": 1, "version": "1.2.3", - "spec_version": "2.8", + "spec_version": "2.9", "commands": ["version", "capabilities", "status", "connect", "send", "subscribe"], "transports": ["socketcan", "virtual"], "features": ["fd", "isotp", "j1939"], @@ -1286,6 +1288,7 @@ single-protocol implementation (the default, `multi_protocol` absent or `false`) both remain governed by §17 Requirements 1 and 6 as MUSTs. Example, RELAY's own CLI: + ```json { "kind": "capabilities", @@ -1294,7 +1297,7 @@ CLI: "protocol_int": null, "multi_protocol": true, "version": "2.5.0", - "spec_version": "2.8", + "spec_version": "2.9", "commands": ["version", "capabilities", "status", "conform", "convert", "..."], "transports": [], "features": [], @@ -1336,6 +1339,7 @@ category as the source-level requirements in §17. ` status --format json`: + ```json { "protocol": "CAN", @@ -1363,7 +1367,7 @@ $ go-can version --format json "protocol": "CAN", "protocol_int": 1, "version": "1.2.3", - "spec_version": "2.8", + "spec_version": "2.9", "language": "go", "runtime": "go1.25.0" } @@ -1375,7 +1379,7 @@ $ go-can capabilities "protocol": "CAN", "protocol_int": 1, "version": "1.2.3", - "spec_version": "2.8", + "spec_version": "2.9", "commands": ["version", "capabilities", "status", "connect", "send", "subscribe"], "transports": ["socketcan", "virtual"], "features": ["fd", "isotp", "j1939"], @@ -1451,11 +1455,11 @@ LABEL org.opencontainers.image.licenses="MPL-2.0" LABEL io.relay.tool="" LABEL io.relay.language="go|cpp|rust|c" LABEL io.relay.binary="" -LABEL io.relay.spec-version="2.8" +LABEL io.relay.spec-version="2.9" ``` The `io.relay.spec-version` label MUST always match the value of `SpecVersion` -exported by the package (§17.12 / §19.4). The `"2.8"` shown above is an +exported by the package (§17.12 / §19.4). The `"2.9"` shown above is an example; per §19.5, this document's own copy of it is checked by this repository's CI against `spec/version.json` on every commit, the same way Requirement 14 requires of a conformant implementation. @@ -1633,6 +1637,13 @@ long-running virtual-bus process, or a real transport like `socketcan`), or the README MUST NOT claim a two-shell round-trip is possible with the mock/virtual transport alone. +Per §17 Requirement 19 and §20.7: this MUST be continuously proven, not +merely true when someone last checked. An implementation's CI MUST +machine-extract and execute every fenced example in the README's `## RELAY +conformance` section that documents an actual, literal command (not +illustrative pseudocode or a usage-syntax summary), failing the build if +any diverges from what actually runs. + --- ## 14. Subscriber Defaults and Helpers @@ -2218,14 +2229,15 @@ or Requirement 9's envelope-conversion verification embeds an identical copy, not a private variant. `spec/vectors/vectors_manifest.json` pins that distribution: + ```json { "kind": "relay-vectors-manifest", "manifest_version": "relay-vectors/1", "vectors_version": "2.4", "vectors": [ - {"name": "can-standard-frame", "sha256": "816b402d..."}, - {"name": "errors/lin-diagnostic-wrong-checksum", "sha256": "36666ad1..."} + {"name": "can-standard-frame", "sha256": "816b402dca5070ec011863b8d8eb3b97abbf2ea8cf264706113e06fcceb4fe27"}, + {"name": "errors/lin-diagnostic-wrong-checksum", "sha256": "36666ad1c49a89ce2b8fbc34475bfe3d5482d232b7b4b315ccb6248b7a05f776"} ] } ``` @@ -2300,6 +2312,7 @@ An implementation is **RELAY-conformant** if and only if: 16. **Vector manifest.** The canonical `spec/vectors/` distribution (§15.8) is pinned by `spec/vectors/vectors_manifest.json`. A conformant implementation that embeds a local copy of these vectors MUST embed the exact pinned set, and MUST have a CI step that fails when its embedded copy's SHA-256 diverges from the published manifest for the `vectors_version` it targets. 17. **No retired capabilities past removal.** Per §3.2, a capabilities document (§12.2) whose declared `spec_version` is at or past a `spec/version.json` `retired[]` entry's `removal` version MUST NOT list that entry's `name` in `features` or `commands`. 18. **Persisted, regression-gated interop matrix.** Per §20.1 item 3, `relay interop --format json` output (a `relay-interop-matrix/1` document, spec §20.1) MUST be committed by the implementation's CI, covering every published sibling implementation of its protocol it can invoke. CI MUST regenerate it and, using `relay interop --baseline `, fail the build if any cell that was `EQUIVALENT` in the committed copy is not `EQUIVALENT` in the fresh run — a cell that was never `EQUIVALENT` to begin with is not itself a gate failure under this requirement (the ordinary Requirement-7-adjacent "any current mismatch fails" gate already covers that case independently). +19. **Continuous example verification.** Per §20.7, every fenced code/CLI example this specification document marks normative (``, §20.7), and every literal, executable command in an implementation's README `## RELAY conformance` section (§13.8), MUST be machine-extracted and executed (or, for JSON documents, schema-validated) in CI. A failing example MUST fail the build. `relay conform ` is a **black-box CLI tool**: it can only observe what the built binary's `version`/`capabilities`/`status` commands print, not the @@ -2362,6 +2375,12 @@ to. No source-level or CI-process knowledge is needed. Requirement 18 sibling-comparison matrix was regenerated, committed, and checked for regressions on this PR is entirely a CI-process fact about a different command's invocation, verified by the implementation's own CI instead. +Requirement 19 (continuous example verification) is likewise not +`relay conform`'s to observe: whether a README's or this document's own +examples were actually extracted and run is a fact about a CI job's own +process, not something a black-box invocation of the binary under test +can see — verified by the implementation's own CI (and, for this document +itself, by RELAY's own CI, per §20.7). **Requirement-to-verifier lookup table**, collecting the narrative above into one place: @@ -2386,6 +2405,7 @@ into one place: | 16 | Vector manifest | Implementation's own CI | Not observable through the CLI | | 17 | No retired capabilities past removal | `relay conform` (version + capabilities cross-check against embedded `spec/version.json`) | Full | | 18 | Persisted, regression-gated interop matrix | Implementation's own CI (`relay interop --baseline`) | Not observable through the CLI | +| 19 | Continuous example verification | Implementation's own CI (README); RELAY's own CI (this document, §20.7) | Not observable through the CLI | ### 17.1 Wire-format regression coverage (recommended) @@ -2433,15 +2453,16 @@ diffable artifact. `relay conform --manifest ` emits a `relay-conform/1` document: + ```json { "kind": "relay-conform-manifest", "manifest_version": "relay-conform/1", "tool": "go-can", "binary_version": "1.2.3", - "spec_version": "2.8", + "spec_version": "2.9", "git_sha": "a1b2c3d4", - "capabilities_sha256": "e3b0c44298fc1c14...", + "capabilities_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "requirements": [ {"id": 1, "status": "PASS", "verifier": "relay conform"}, {"id": 2, "status": "NOT_OBSERVABLE", "verifier": "implementation test suite"}, @@ -2458,7 +2479,7 @@ SHA-256 of the raw `capabilities` document bytes as returned by the binary, letting a diff catch any change to the capabilities document — mechanically regenerated or not — without re-parsing it. -`requirements` MUST contain exactly one entry per §17 requirement (1–18, +`requirements` MUST contain exactly one entry per §17 requirement (1–19, extended as this document's own requirement list grows). Each entry's `status` is one of: @@ -3188,11 +3209,11 @@ clarifications and fixes in PATCH releases. `spec/version.json` is authoritative. The spec document title is informational. -Current version: **v2.8** +Current version: **v2.9** -**Go:** `const SpecVersion = "2.8"` (update in implementations targeting v2.8) -**C++:** `constexpr std::string_view kRelaySpecVersion = "2.8";` -**Rust:** `pub const RELAY_SPEC_VERSION: &str = "2.8";` +**Go:** `const SpecVersion = "2.9"` (update in implementations targeting v2.9) +**C++:** `constexpr std::string_view kRelaySpecVersion = "2.9";` +**Rust:** `pub const RELAY_SPEC_VERSION: &str = "2.9";` ### 19.5 This document's own version literals @@ -3314,15 +3335,16 @@ implementation MAY publish to close that gap — a manifest, the §15.8 vector manifest's `vectors_version`, and the targeted `spec_version`, shaped as an [in-toto](https://in-toto.io/) Statement: + ```json { "_type": "https://in-toto.io/Statement/v1", "predicateType": "https://relay.dev/attestation/relay-conform/1", "subject": [ - {"name": "go-can", "digest": {"sha256": "9f86d081884c7d65..."}} + {"name": "go-can", "digest": {"sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"}} ], "predicate": { - "spec_version": "2.8", + "spec_version": "2.9", "vectors_version": "2.4", "conformance_manifest": { "...": "the full §17.2 relay-conform/1 document" }, "safety_evidence_summary": null, @@ -3369,6 +3391,58 @@ CI gate for it yet, and RELAY's own reference tooling emits the predicate unsigned (`relay conform --attestation `) without publishing or signing it. +### 20.7 Continuous example verification + +§13.8 already requires a README's documented CLI round-trip to actually +work; that alone doesn't stop it from silently rotting after the check was +last done by hand. §19.5 fixed the identical problem for this document's +own version literals (found rotted, in the wild, as the "0.1" example +literals fixed in v2.2.3). Per §17 Requirement 19, both gaps close the +same way: **no example ships unverified**, in either direction. + +**This document's own examples.** A fenced ` ```json ` block immediately +preceded by an HTML comment `` on its own line is a +normative example this repository's own CI MUST extract and validate — +not merely eyeball — on every change. `KIND` selects the check: + +| `KIND` | Validated against | +|---|---| +| `version` | §12.1 version-document rules (schema + null-`protocol` handling) | +| `capabilities` | §12.2 capabilities-document rules (schema + `multi_protocol` logic) | +| `status` | §12.3 status-document schema | +| `manifest` | `relay-conform-manifest` schema (§17.2) | +| `attestation` | `relay-conform-attestation` schema (§20.6) | +| `vectors-manifest` | `vectors-manifest` schema (§15.8) | + +Each marked example MUST be run through the *same* validator RELAY's own +reference tooling applies to a real binary's output — not a hand-rolled, +looser check invented for the doctest alone — so passing this gate means +the example is exactly as valid as a real conformant response would be. +A build MUST fail if a marker's example violates its validator, or if the +marker convention itself falls out of sync with the document (a `KIND` +with zero examples found). + +**Implementation READMEs.** Per §13.8's existing MUST, a README's `## +RELAY conformance` section MUST document a working round-trip. This +section additionally requires it stay continuously proven: CI MUST +machine-extract and execute every fenced example in that section that +documents a literal, executable command — not illustrative pseudocode +(a code sketch using undefined variables to show API shape) or a bare +usage-syntax summary (`Usage: [flags]`), neither of +which is something "executing" even means. A failing example MUST fail +the build. + +**Deliberately out of scope for this section.** Two classes of example are +explicitly *not* required to be executed, because doing so would mean +mandating infrastructure this document does not otherwise require: +commands that fetch over the network at CI time (`go get`/`go install` +against a live module proxy) or that require a published container image +this repository doesn't build (a `docker run ghcr.io/...` example) are +exempt — a hermetic, network-free extraction harness for those is a +reasonable future addition, not assumed here. An implementation MAY still +choose to execute them (e.g. against a local build instead of the network), +but this section does not require it. + --- ## Appendix A — Project alignment as of v0.2 (2026-06-16); not maintained thereafter diff --git a/spec/schemas/relay-conform-manifest.json b/spec/schemas/relay-conform-manifest.json index 8f7e96b..65a82f2 100644 --- a/spec/schemas/relay-conform-manifest.json +++ b/spec/schemas/relay-conform-manifest.json @@ -56,7 +56,7 @@ "required": ["id", "status", "verifier"], "additionalProperties": false }, - "description": "One entry per §17 requirement (1-18, extended as the requirement list grows)." + "description": "One entry per §17 requirement (1-19, extended as the requirement list grows)." }, "overall": { "type": "string", diff --git a/spec/version.json b/spec/version.json index e59287a..e2b31fe 100644 --- a/spec/version.json +++ b/spec/version.json @@ -1,5 +1,5 @@ { - "version": "2.8", + "version": "2.9", "date": "2026-08-21", "status": "stable", "networks": [ diff --git a/version.go b/version.go index 483afc1..c1a3d80 100644 --- a/version.go +++ b/version.go @@ -8,4 +8,4 @@ package relay // Every RELAY-conformant implementation must export this constant (spec §17 req 12). // //fusa:req REQ-RELAY-020 -const SpecVersion = "2.8" +const SpecVersion = "2.9" diff --git a/version_consistency_test.go b/version_consistency_test.go index 1d2ecc4..0929ac7 100644 --- a/version_consistency_test.go +++ b/version_consistency_test.go @@ -104,3 +104,29 @@ func TestSpecDocumentVersionLiteralsMatchVersionJSON(t *testing.T) { } } } + +// TestReadmeVersionMatchesVersionJSON extends §19.5's discipline to +// README.md's own "Current: **vX.Y (stable)**" line — found stale at "v2.0" +// (six MINOR releases behind) while building the §20.7 doctest mechanism, +// itself an instance of the exact unchecked-example drift both sections +// exist to close. README.md is not part of the specification document +// §19.5 already covers, so it needed its own anchor here rather than +// silently riding along. +// +//fusa:test REQ-RELAY-102 +func TestReadmeVersionMatchesVersionJSON(t *testing.T) { + want := versionJSONVersion(t) + + readmeRaw, err := Evidence("readme") + if err != nil { + t.Fatalf("Evidence(readme): %v", err) + } + re := regexp.MustCompile(`Current: \*\*v([0-9]+\.[0-9]+) \(stable\)\*\*`) + m := re.FindStringSubmatch(string(readmeRaw)) + if m == nil { + t.Fatal(`README.md: "Current: **vX.Y (stable)**" line not found — reworded? update this test's regex`) + } + if m[1] != want { + t.Errorf("README.md Current version = %q, want %q (spec/version.json)", m[1], want) + } +} diff --git a/version_test.go b/version_test.go index 85d9e3a..4e06917 100644 --- a/version_test.go +++ b/version_test.go @@ -8,7 +8,7 @@ import "testing" //fusa:test REQ-RELAY-020 func TestSpecVersion(t *testing.T) { - if SpecVersion != "2.8" { - t.Errorf("SpecVersion = %q, want %q", SpecVersion, "2.8") + if SpecVersion != "2.9" { + t.Errorf("SpecVersion = %q, want %q", SpecVersion, "2.9") } }