feat: continuous example verification (§17 Requirement 19)

.fusa-reqs.json

@@ -961,6 +961,16 @@
       "verification": "test",
       "safety_goal": "SG-006",
       "asil": "ASIL-B"
+    },
+    {
+      "id": "REQ-RELAY-102",
+      "title": "Continuous example verification (doctest, spec sec-20.7)",
+      "text": "A fenced json code block in the specification document immediately preceded by an HTML comment <!-- doctest:KIND --> on its own line MUST be extracted by this repository's own CI and validated against KIND's rules (version/capabilities/status document rules, or the relay-conform-manifest/relay-conform-attestation/vectors-manifest schema) using the same validator relay conform applies to a real binary's output. A build MUST fail if a marked example violates its validator, or if a KIND has zero examples found. Per spec sec-13.8, an implementation's README RELAY conformance section MUST similarly have every literal, executable command example machine-extracted and executed in its own CI, excluding network-fetching commands (go get/go install) and commands requiring a published container image, which are explicitly out of this requirement's scope.",
+      "category": "functional",
+      "criticality": "medium",
+      "verification": "test",
+      "safety_goal": "SG-006",
+      "asil": "ASIL-B"
     }
   ]
 }

README.md

@@ -29,7 +29,7 @@ Full specification: [`spec/relay-spec.md`](spec/relay-spec.md)
 Machine-readable version: [`spec/version.json`](spec/version.json)  
 Change history: [`spec/CHANGELOG.md`](spec/CHANGELOG.md)
 
-Current: **v2.0 (stable)**
+Current: **v2.9 (stable)**
 
 ## Install
 

cmd/relay/conform.go

@@ -336,6 +336,7 @@ func buildManifest(binary string) conformManifest {
 		{16, statusNotObservable, verifierCI},
 		{17, req17, verifierRelayConform},
 		{18, statusNotObservable, verifierCI},
+		{19, statusNotObservable, verifierCI},
 	}
 
 	m.Overall = statusPass

cmd/relay/conform_test.go

@@ -382,8 +382,8 @@ func TestBuildManifestSelf(t *testing.T) {
 	if m.SpecVersion == "" {
 		t.Error("SpecVersion is empty")
 	}
-	if len(m.Requirements) != 18 {
-		t.Fatalf("len(Requirements) = %d, want 18", len(m.Requirements))
+	if len(m.Requirements) != 19 {
+		t.Fatalf("len(Requirements) = %d, want 19", len(m.Requirements))
 	}
 	for i, r := range m.Requirements {
 		if r.ID != i+1 {
@@ -414,7 +414,7 @@ func TestBuildManifestNotObservableDefaults(t *testing.T) {
 	bin := buildTestBinary(t)
 	m := buildManifest(bin)
 
-	wantNotObservable := []int{2, 3, 4, 5, 8, 9, 10, 11, 13, 14, 15, 16, 18}
+	wantNotObservable := []int{2, 3, 4, 5, 8, 9, 10, 11, 13, 14, 15, 16, 18, 19}
 	// Requirement 17 is fully observable (see TestBuildManifestSelf's
 	// observable set), so it is deliberately excluded from this list.
 	// Requirement 18 (persisted interop matrix) is verified by a
@@ -513,8 +513,8 @@ func TestRunConformManifestFlag(t *testing.T) {
 	if m.Overall != statusPass {
 		t.Errorf("Overall = %s, want %s", m.Overall, statusPass)
 	}
-	if len(m.Requirements) != 18 {
-		t.Errorf("len(Requirements) = %d, want 18", len(m.Requirements))
+	if len(m.Requirements) != 19 {
+		t.Errorf("len(Requirements) = %d, want 19", len(m.Requirements))
 	}
 }
 
@@ -575,8 +575,8 @@ func TestBuildAttestationSelf(t *testing.T) {
 	if a.Predicate.ConformanceManifest.Kind != "relay-conform-manifest" {
 		t.Errorf("Predicate.ConformanceManifest.Kind = %q, want relay-conform-manifest", a.Predicate.ConformanceManifest.Kind)
 	}
-	if len(a.Predicate.ConformanceManifest.Requirements) != 18 {
-		t.Errorf("len(Predicate.ConformanceManifest.Requirements) = %d, want 18", len(a.Predicate.ConformanceManifest.Requirements))
+	if len(a.Predicate.ConformanceManifest.Requirements) != 19 {
+		t.Errorf("len(Predicate.ConformanceManifest.Requirements) = %d, want 19", len(a.Predicate.ConformanceManifest.Requirements))
 	}
 	// §20.6: MUST be false — this type has no signing mechanism at all.
 	if a.Predicate.Signed {

cmd/relay/doctest.go

@@ -0,0 +1,54 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+// doctestMarkerRe matches a `<!-- doctest:KIND -->` comment line marking the
+// fenced ```json block that immediately follows as a normative example this
+// document commits to keeping genuinely valid (spec §20.7, §17 Requirement 19).
+var doctestMarkerRe = regexp.MustCompile(`<!-- doctest:([a-z-]+) -->`)
+
+// extractDoctestExamples scans a spec document line-by-line for
+// `<!-- doctest:KIND -->` markers and returns the JSON bytes of the fenced
+// ```json block that follows each one (skipping blank lines in between). A
+// marker not followed by a properly-terminated fence is reported as an
+// error — a silently-skipped marker would defeat the whole point of this
+// mechanism: an example that's supposed to be checked but never actually is.
+//
+//fusa:req REQ-RELAY-102
+func extractDoctestExamples(spec string) (map[string][][]byte, error) {
+	lines := strings.Split(spec, "\n")
+	out := map[string][][]byte{}
+	for i := 0; i < len(lines); i++ {
+		m := doctestMarkerRe.FindStringSubmatch(lines[i])
+		if m == nil {
+			continue
+		}
+		kind := m[1]
+		j := i + 1
+		for j < len(lines) && strings.TrimSpace(lines[j]) == "" {
+			j++
+		}
+		if j >= len(lines) || strings.TrimSpace(lines[j]) != "```json" {
+			return nil, fmt.Errorf("doctest:%s marker at line %d is not followed by a ```json fence", kind, i+1)
+		}
+		j++
+		start := j
+		for j < len(lines) && strings.TrimSpace(lines[j]) != "```" {
+			j++
+		}
+		if j >= len(lines) {
+			return nil, fmt.Errorf("doctest:%s marker at line %d: unterminated json fence", kind, i+1)
+		}
+		out[kind] = append(out[kind], []byte(strings.Join(lines[start:j], "\n")))
+		i = j
+	}
+	return out, nil
+}

cmd/relay/doctest_test.go

@@ -0,0 +1,93 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/json"
+	"testing"
+
+	relay "github.com/SoundMatt/RELAY/v2"
+)
+
+// TestSpecDoctestExamplesValidate is spec §20.7's mechanism: every JSON
+// example marked `<!-- doctest:KIND -->` in relay-spec.md is extracted and
+// run through the exact same validator relay conform itself uses on a real
+// binary's output — not a relaxed or hand-copied check. A stale or
+// hand-broken example (the class of drift that produced NEW-SPEC-6's "0.1"
+// incident) fails this test, not just a human reader's attention.
+//
+//fusa:test REQ-RELAY-102
+func TestSpecDoctestExamplesValidate(t *testing.T) {
+	specRaw, err := relay.Evidence("specification")
+	if err != nil {
+		t.Fatalf("Evidence(specification): %v", err)
+	}
+	examples, err := extractDoctestExamples(string(specRaw))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Sanity: catches the marker convention silently falling out of sync
+	// with the document (all markers removed, or the kind renamed) rather
+	// than the loop below quietly validating zero examples and passing.
+	for _, kind := range []string{"version", "capabilities", "status", "manifest", "attestation", "vectors-manifest"} {
+		if len(examples[kind]) == 0 {
+			t.Errorf("no doctest:%s example found in spec/relay-spec.md", kind)
+		}
+	}
+
+	for _, ex := range examples["version"] {
+		assertDoctestNoFail(t, "version", ex, validateVersionDoc(ex))
+	}
+	for _, ex := range examples["capabilities"] {
+		assertDoctestNoFail(t, "capabilities", ex, validateCapabilitiesDoc(ex))
+	}
+	for _, ex := range examples["status"] {
+		assertDoctestNoFail(t, "status", ex, validateStatusDoc(ex))
+	}
+	for _, ex := range examples["manifest"] {
+		assertDoctestSchemaValid(t, "manifest", "relay-conform-manifest", ex)
+	}
+	for _, ex := range examples["attestation"] {
+		assertDoctestSchemaValid(t, "attestation", "relay-conform-attestation", ex)
+	}
+	for _, ex := range examples["vectors-manifest"] {
+		assertDoctestSchemaValid(t, "vectors-manifest", "vectors-manifest", ex)
+	}
+}
+
+//fusa:test REQ-RELAY-102
+func TestExtractDoctestExamplesUnterminatedFence(t *testing.T) {
+	if _, err := extractDoctestExamples("<!-- doctest:version -->\n```json\n{\"a\":1}\n"); err == nil {
+		t.Error("expected an error for an unterminated json fence")
+	}
+	if _, err := extractDoctestExamples("<!-- doctest:version -->\nnot a fence\n"); err == nil {
+		t.Error("expected an error when the marker isn't followed by a ```json fence")
+	}
+}
+
+func assertDoctestNoFail(t *testing.T, kind string, example []byte, findings []conformFinding) {
+	t.Helper()
+	if hasFail(findings) {
+		t.Errorf("doctest:%s example failed validation: %+v\n%s", kind, findings, example)
+	}
+}
+
+func assertDoctestSchemaValid(t *testing.T, kind, schemaName string, example []byte) {
+	t.Helper()
+	var asAny interface{}
+	if err := json.Unmarshal(example, &asAny); err != nil {
+		t.Errorf("doctest:%s example is not valid JSON: %v\n%s", kind, err, example)
+		return
+	}
+	schemaJSON, err := relay.Schema(schemaName)
+	if err != nil {
+		t.Fatalf("doctest:%s: no embedded schema %q: %v", kind, schemaName, err)
+	}
+	violations := validateSchema(schemaJSON, asAny)
+	if len(violations) > 0 {
+		t.Errorf("doctest:%s example violates %s schema: %v\n%s", kind, schemaName, violations, example)
+	}
+}

cmd/relay/jsonschema.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"math"
+	"regexp"
 	"sort"
 	"strings"
 )
@@ -19,7 +20,7 @@ import (
 //
 //	type (string or []string), properties, required, additionalProperties:false,
 //	enum, const, minimum, maximum, items (single schema), minItems, maxItems,
-//	contains (single schema).
+//	contains (single schema), pattern (string values, RE2 syntax).
 //
 // Unsupported keywords are ignored (they do not cause false failures).
 
@@ -75,6 +76,15 @@ func validateValue(schema map[string]interface{}, v interface{}, path string, ou
 		if max, ok := numberOf(schema["maximum"]); ok && val > max {
 			*out = append(*out, fmt.Sprintf("%s: %v > maximum %v", loc, val, max))
 		}
+	case string:
+		if pat, ok := schema["pattern"].(string); ok {
+			re, err := regexp.Compile(pat)
+			if err != nil {
+				*out = append(*out, fmt.Sprintf("%s: schema pattern %q does not compile: %v", loc, pat, err))
+			} else if !re.MatchString(val) {
+				*out = append(*out, fmt.Sprintf("%s: value %q does not match pattern %q", loc, val, pat))
+			}
+		}
 	case map[string]interface{}:
 		validateObject(schema, val, path, out)
 	case []interface{}:

cmd/relay/jsonschema_test.go

@@ -89,6 +89,30 @@ func TestSchemaMinMax(t *testing.T) {
 	}
 }
 
+//fusa:test REQ-RELAY-058
+func TestSchemaPattern(t *testing.T) {
+	// SHA-256 hex, the pattern every schema in spec/schemas/ that declares a
+	// digest field uses. Found genuinely unenforced (silently ignored) while
+	// building the §20.7 doctest mechanism: a document with a truncated or
+	// non-hex hash passed schema validation with zero violations before this.
+	schema := `{"type":"string","pattern":"^[0-9a-f]{64}$"}`
+	if v := validateSchema([]byte(schema), "816b402dca5070ec011863b8d8eb3b97abbf2ea8cf264706113e06fcceb4fe27"); len(v) != 0 {
+		t.Errorf("a genuine 64-char lowercase hex string should satisfy the pattern, got %v", v)
+	}
+	if v := validateSchema([]byte(schema), "816b402d..."); len(v) == 0 {
+		t.Error("a truncated hash with a literal ellipsis should violate the pattern")
+	}
+	if v := validateSchema([]byte(schema), "816B402DCA5070EC011863B8D8EB3B97ABBF2EA8CF264706113E06FCCEB4FE27"); len(v) == 0 {
+		t.Error("uppercase hex should violate a lowercase-only pattern")
+	}
+	// An invalid regex must be reported as a violation, not silently ignored
+	// or panic — a malformed schema is a real (if different) defect.
+	bad := `{"type":"string","pattern":"("}`
+	if v := validateSchema([]byte(bad), "anything"); len(v) == 0 {
+		t.Error("an uncompilable pattern should itself be reported as a violation")
+	}
+}
+
 //fusa:test REQ-RELAY-058
 func TestSchemaArrayItemsAndContains(t *testing.T) {
 	schema := `{"type":"array","items":{"type":"string"},"contains":{"const":"version"}}`

cmd/relay/main.go

@@ -18,7 +18,7 @@ import (
 	relay "github.com/SoundMatt/RELAY/v2"
 )
 
-const toolVersion = "2.8.0"
+const toolVersion = "2.9.0"
 
 func main() {
 	if err := run(os.Stdout, os.Stderr, os.Args[1:]); err != nil {

evidence.go

@@ -27,13 +27,14 @@ import (
 //fusa:req REQ-RELAY-093
 //fusa:req REQ-RELAY-094
 //fusa:req REQ-RELAY-100
+//fusa:req REQ-RELAY-102
 //fusa:req REQ-RELAY-074
 //fusa:req REQ-RELAY-075
 //fusa:req REQ-RELAY-076
 //go:embed .fusa-reqs.json .fusa-hara.json .fusa-tara.json
 //go:embed spec/version.json docs/tool-safety-manual.md
 //go:embed docs/formal/RelayLifecycle.tla docs/formal/README.md
-//go:embed docs/asil-d-uplift.md spec/relay-spec.md
+//go:embed docs/asil-d-uplift.md spec/relay-spec.md README.md
 var evidenceFS embed.FS
 
 // evidencePaths maps the logical evidence name to its embedded path.
@@ -47,6 +48,7 @@ var evidencePaths = map[string]string{
 	"formal-model-doc":   "docs/formal/README.md",
 	"asil-d-uplift":      "docs/asil-d-uplift.md",
 	"specification":      "spec/relay-spec.md",
+	"readme":             "README.md",
 }
 
 // Evidence returns the raw bytes of a named safety-evidence artifact, e.g.

evidence_test.go

@@ -19,6 +19,7 @@ func TestEvidenceNames(t *testing.T) {
 		"version": true, "tool-safety-manual": true,
 		"formal-model": true, "formal-model-doc": true,
 		"asil-d-uplift": true, "specification": true,
+		"readme": true,
 	}
 	if len(names) != len(want) {
 		t.Fatalf("EvidenceNames = %v, want %d entries", names, len(want))

spec/CHANGELOG.md

@@ -1,5 +1,67 @@
 # RELAY Spec Changelog
 
+## v2.9 — 2026-08-21 (MINOR — continuous example verification, §17 Requirement 19)
+
+- **New §20.7 "Continuous example verification"** and **new §17
+  Requirement 19**. §13.8 already required a README's documented CLI
+  round-trip to work; §19.5 already required this document's own
+  version literals to stay in sync. Neither stopped a *content* example
+  (not just a version number) from silently rotting once nobody was
+  reading closely — exactly the failure mode THEME-J and REL-SPEC-10
+  independently documented. This closes it the same way §19.5 did: no
+  example ships unverified.
+- **`<!-- doctest:KIND -->` marker convention** for this document's own
+  normative JSON examples. `KIND` ∈ `{version, capabilities, status,
+  manifest, attestation, vectors-manifest}` selects which validator a
+  marked example is run through — the *same* validator `relay conform`
+  applies to a real binary's output, not a separate, looser check
+  invented for the doctest alone. Applied to all 7 existing normative
+  examples (§12.1, §12.2 ×2, §12.3, §15.8, §17.2, §20.6).
+- **README `## RELAY conformance` sections** must now have every
+  literal, executable command example machine-extracted and executed
+  in CI (§13.8 extended). Network-fetching commands (`go get`/`go
+  install`) and commands needing a published container image are
+  explicitly out of scope — mandating a hermetic runner for those isn't
+  this section's job.
+- **Real bugs found while building this, fixed along the way**:
+  - Three of this document's own examples (§15.8's two vector hashes,
+    §17.2's `capabilities_sha256`, §20.6's subject digest) used
+    truncated placeholder hex (`"816b402d..."`) instead of a real
+    64-character SHA-256. Replaced with genuine values — two are the
+    *actual* hashes of the referenced golden vectors; the other two are
+    real SHA-256 outputs (of `""` and `"test"` respectively), chosen so
+    the "before" value was already a recognizable prefix of the "after"
+    one.
+  - **RELAY's own JSON Schema validator never implemented the
+    `pattern` keyword at all** — silently ignored, per its own
+    documented keyword list. Every schema in `spec/schemas/` that
+    constrains a SHA-256 field with `"pattern": "^[0-9a-f]{64}$"` was
+    unenforced: `relay conform` would have accepted a malformed hash in
+    any capabilities/manifest/attestation/vectors-manifest document
+    without a single finding. Found because the truncated placeholder
+    examples above initially validated cleanly — a false pass this
+    doctest mechanism was specifically built to prevent. Fixed with a
+    `regexp`-based `pattern` check in `validateSchema`; new
+    `TestSchemaPattern` unit test; mutation-tested (confirmed the fix
+    catches both the truncated hash and case-sensitivity violations,
+    and that an uncompilable pattern is itself reported rather than
+    silently ignored or panicking).
+  - **README.md's own "Current" version line was six MINOR releases
+    stale** (`v2.0`, while `spec/version.json` was already at `v2.8`) —
+    the exact class of drift §19.5 exists to prevent, just in a file
+    §19.5 doesn't cover. Fixed to the version this release ships, and
+    `TestReadmeVersionMatchesVersionJSON` extends the §19.5 mechanism
+    (new `REQ-RELAY-102`, `README.md` now embedded evidence) so it
+    can't silently recur.
+- **Reference implementation**: `extractDoctestExamples`
+  (`cmd/relay/doctest.go`), `TestSpecDoctestExamplesValidate`
+  (`cmd/relay/doctest_test.go`). Mutation-tested: corrupted a hash
+  pattern and a required `commands` entry in two separate examples,
+  confirmed both were caught with a precise violation message, restored
+  and reconfirmed green.
+- `SpecVersion` bumped `2.8` → `2.9` (MINOR, new §17 requirement).
+  Closes [NEW-SPEC-8].
+
 ## v2.8 — 2026-08-21 (MINOR — persisted, regression-gated interop matrix, §17 Requirement 18)
 
 - **New §17 Requirement 18 — persisted, regression-gated interop matrix.**