From 858aee7c0e45b114b637a7e5e4b2b0a2cac58c1c Mon Sep 17 00:00:00 2001 From: Matt <47545907+SoundMatt@users.noreply.github.com> Date: Fri, 21 Aug 2026 06:41:00 -0700 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20persisted,=20regression-gated=20int?= =?UTF-8?q?erop=20matrix=20(=C2=A717=20Requirement=2018)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit relay interop's --format json output is now a versioned relay-interop-matrix/1 artifact (kind, matrix_version, an explicit participants[] list, the equivalence matrix, result) instead of an ephemeral pass/fail whose result vanished after the CI run. New --baseline FILE flag: compares a fresh run against a previously- committed matrix and reports a regressions[] list -- one entry per (vector, participant) cell that was EQUIVALENT in the baseline but isn't now. A cell that was never EQUIVALENT to begin with is not itself a regression, matching the issue's precise wording (distinct from the existing any-mismatch-fails gate, which already covers that case independently). Deliberately NOT pairwise (N^2): every participant is compared only against the shared in-process reference. Byte-equality is transitive, so "A equivalent to reference" and "B equivalent to reference" jointly establish "A equivalent to B" already -- documented explicitly in §17 and §20.1 rather than silently narrowed from the issue's literal "every other sibling" wording. New §17 Requirement 18, new spec/schemas/relay-interop-matrix.json, new REQ-RELAY-101. §20.1 item 3 extended; also fixes a real citation bug found along the way (§20.4's "§20.1.2" -- §20.1 is a flat list, not subdivided). 9 new/changed interop tests, including a mutation test on the regression detector's core logic -- caught immediately by a pure unit test; the first integration-level attempt was fooled twice (the reject-path vectors' by-design leniency, then a substring collision between the test binary's name and the ever-equivalent 'relay (reference)' row) before correctly discriminating the mutation. SpecVersion 2.7 -> 2.8 (MINOR, new §17 requirement). Closes #145 Signed-off-by: Matt <47545907+SoundMatt@users.noreply.github.com> --- .fusa-reqs.json | 10 ++ cmd/relay/conform.go | 1 + cmd/relay/conform_test.go | 17 ++- cmd/relay/interop.go | 108 ++++++++++++++-- cmd/relay/interop_test.go | 157 +++++++++++++++++++++++ cmd/relay/main.go | 2 +- spec/CHANGELOG.md | 54 ++++++++ spec/relay-spec.md | 55 +++++--- spec/schemas/relay-conform-manifest.json | 2 +- spec/schemas/relay-interop-matrix.json | 64 +++++++++ spec/version.json | 2 +- version.go | 2 +- version_test.go | 4 +- 13 files changed, 439 insertions(+), 39 deletions(-) create mode 100644 spec/schemas/relay-interop-matrix.json diff --git a/.fusa-reqs.json b/.fusa-reqs.json index a3491a8..e321d24 100644 --- a/.fusa-reqs.json +++ b/.fusa-reqs.json @@ -951,6 +951,16 @@ "criticality": "low", "verification": "test", "asil": "QM" + }, + { + "id": "REQ-RELAY-101", + "title": "Persisted, regression-gated interop matrix (relay-interop-matrix/1)", + "text": "relay interop's JSON output is a versioned relay-interop-matrix/1 artifact: kind, matrix_version, reference, an explicit participants[] list, the per-vector per-participant equivalence matrix, and result. relay interop --baseline FILE MUST compare a fresh run against a previously-committed relay-interop-matrix/1 document and report a regression for every (vector, participant) cell that was EQUIVALENT in the baseline but is not in the fresh run; a cell that was never EQUIVALENT to begin with MUST NOT be reported as a regression. Each participant is compared only against the shared in-process reference (not pairwise against every other participant): byte-equality is transitive, so per-participant equivalence to the same reference already establishes cross-implementation equivalence.", + "category": "functional", + "criticality": "medium", + "verification": "test", + "safety_goal": "SG-006", + "asil": "ASIL-B" } ] } diff --git a/cmd/relay/conform.go b/cmd/relay/conform.go index 2a9ea39..4ea7195 100644 --- a/cmd/relay/conform.go +++ b/cmd/relay/conform.go @@ -335,6 +335,7 @@ func buildManifest(binary string) conformManifest { {15, statusNotObservable, verifierManifestSplit}, {16, statusNotObservable, verifierCI}, {17, req17, verifierRelayConform}, + {18, statusNotObservable, verifierCI}, } m.Overall = statusPass diff --git a/cmd/relay/conform_test.go b/cmd/relay/conform_test.go index 42e261e..706dc4e 100644 --- a/cmd/relay/conform_test.go +++ b/cmd/relay/conform_test.go @@ -382,8 +382,8 @@ func TestBuildManifestSelf(t *testing.T) { if m.SpecVersion == "" { t.Error("SpecVersion is empty") } - if len(m.Requirements) != 17 { - t.Fatalf("len(Requirements) = %d, want 17", len(m.Requirements)) + if len(m.Requirements) != 18 { + t.Fatalf("len(Requirements) = %d, want 18", len(m.Requirements)) } for i, r := range m.Requirements { if r.ID != i+1 { @@ -414,9 +414,12 @@ func TestBuildManifestNotObservableDefaults(t *testing.T) { bin := buildTestBinary(t) m := buildManifest(bin) - wantNotObservable := []int{2, 3, 4, 5, 8, 9, 10, 11, 13, 14, 15, 16} + wantNotObservable := []int{2, 3, 4, 5, 8, 9, 10, 11, 13, 14, 15, 16, 18} // Requirement 17 is fully observable (see TestBuildManifestSelf's // observable set), so it is deliberately excluded from this list. + // Requirement 18 (persisted interop matrix) is verified by a + // different command (relay interop, --baseline) that relay conform + // never invokes, so it is NOT_OBSERVABLE like 13/14/16. byID := map[int]manifestRequirement{} for _, r := range m.Requirements { byID[r.ID] = r @@ -510,8 +513,8 @@ func TestRunConformManifestFlag(t *testing.T) { if m.Overall != statusPass { t.Errorf("Overall = %s, want %s", m.Overall, statusPass) } - if len(m.Requirements) != 17 { - t.Errorf("len(Requirements) = %d, want 17", len(m.Requirements)) + if len(m.Requirements) != 18 { + t.Errorf("len(Requirements) = %d, want 18", len(m.Requirements)) } } @@ -572,8 +575,8 @@ func TestBuildAttestationSelf(t *testing.T) { if a.Predicate.ConformanceManifest.Kind != "relay-conform-manifest" { t.Errorf("Predicate.ConformanceManifest.Kind = %q, want relay-conform-manifest", a.Predicate.ConformanceManifest.Kind) } - if len(a.Predicate.ConformanceManifest.Requirements) != 17 { - t.Errorf("len(Predicate.ConformanceManifest.Requirements) = %d, want 17", len(a.Predicate.ConformanceManifest.Requirements)) + if len(a.Predicate.ConformanceManifest.Requirements) != 18 { + t.Errorf("len(Predicate.ConformanceManifest.Requirements) = %d, want 18", len(a.Predicate.ConformanceManifest.Requirements)) } // §20.6: MUST be false — this type has no signing mechanism at all. if a.Predicate.Signed { diff --git a/cmd/relay/interop.go b/cmd/relay/interop.go index 4e07ed0..7dbca4b 100644 --- a/cmd/relay/interop.go +++ b/cmd/relay/interop.go @@ -56,33 +56,51 @@ type interopVectorResult struct { Cells []interopCell `json:"cells"` } -// interopDoc is the full interop report. +// interopDoc is the full interop report — the relay-interop-matrix/1 +// artifact (spec §20.1 item 3, Requirement 18) when persisted by CI. type interopDoc struct { - Reference string `json:"reference"` - Result string `json:"result"` // PASS / FAIL - Vectors []interopVectorResult `json:"vectors"` + Kind string `json:"kind"` + MatrixVersion string `json:"matrix_version"` + Reference string `json:"reference"` + Participants []string `json:"participants"` + Result string `json:"result"` // PASS / FAIL + Vectors []interopVectorResult `json:"vectors"` + Regressions []string `json:"regressions,omitempty"` } // runInterop implements -// `relay interop [--protocol P] [--vectors DIR] [--strict] [--format text|json|markdown] ...`. +// `relay interop [--protocol P] [--vectors DIR] [--strict] [--baseline FILE] [--format text|json|markdown] ...`. // It verifies that implementations are behaviourally interchangeable by diffing // each binary's `convert` output against RELAY's reference conversion for every -// golden vector (spec §11.2.1). +// golden vector (spec §11.2.1). Every participant is compared against the same +// in-process reference rather than against each other pairwise: since the +// comparison is byte-equality (a transitive relation), "A equivalent to +// reference" and "B equivalent to reference" together already establish "A +// equivalent to B" — an O(N) hub comparison carries the same information as an +// O(N²) pairwise one here, without the redundant work (spec §20.1 item 3). +// +// --baseline compares this run's matrix against a previously-committed +// relay-interop-matrix/1 document (spec §17 Requirement 18) and reports a +// regression for any (vector, participant) cell that was EQUIVALENT in the +// baseline but is not in this run — distinct from a cell that was never +// equivalent to begin with, which --baseline does not newly fail on. // //fusa:req REQ-RELAY-083 +//fusa:req REQ-RELAY-101 func runInterop(stdout, stderr io.Writer, args []string) error { fs := flag.NewFlagSet("interop", flag.ContinueOnError) fs.SetOutput(io.Discard) protocol := fs.String("protocol", "", "Restrict to a single protocol (CAN, DDS, LIN, MQTT, RCP, SOMEIP)") vectorsDir := fs.String("vectors", "", "Directory of vector files (default: embedded golden vectors)") strict := fs.Bool("strict", false, "Treat a binary that lacks convert as a failure rather than a skip") + baseline := fs.String("baseline", "", "Path to a previously-generated relay-interop-matrix/1 document; fail if any of its EQUIVALENT cells regressed") format := fs.String("format", "text", "Output format: text, json, or markdown") if err := fs.Parse(args); err != nil { return fmt.Errorf("relay interop: %w", err) } binaries := fs.Args() if len(binaries) == 0 { - fmt.Fprintln(stderr, "Usage: relay interop [--protocol P] [--vectors DIR] [--strict] [--format text|json|markdown] ...") + fmt.Fprintln(stderr, "Usage: relay interop [--protocol P] [--vectors DIR] [--strict] [--baseline FILE] [--format text|json|markdown] ...") return exitCode(2) } @@ -110,7 +128,19 @@ func runInterop(stdout, stderr io.Writer, args []string) error { } } - doc := interopDoc{Reference: "relay (reference)", Result: "PASS"} + participants := make([]string, len(binaries)) + for i, bin := range binaries { + participants[i] = filepath.Base(bin) + } + sort.Strings(participants) + + doc := interopDoc{ + Kind: "relay-interop-matrix", + MatrixVersion: "relay-interop-matrix/1", + Reference: "relay (reference)", + Participants: participants, + Result: "PASS", + } for _, v := range vecs { row := interopVectorResult{Vector: v.Name, Protocol: v.Protocol} @@ -193,6 +223,23 @@ func runInterop(stdout, stderr io.Writer, args []string) error { doc.Vectors = append(doc.Vectors, row) } + if *baseline != "" { + base, err := os.ReadFile(*baseline) + if err != nil { + fmt.Fprintf(stderr, "relay interop: --baseline: %v\n", err) + return exitCode(2) + } + var baseDoc interopDoc + if err := json.Unmarshal(base, &baseDoc); err != nil { + fmt.Fprintf(stderr, "relay interop: --baseline: not a relay-interop-matrix/1 document: %v\n", err) + return exitCode(2) + } + doc.Regressions = interopRegressions(baseDoc, doc) + if len(doc.Regressions) > 0 { + doc.Result = "FAIL" + } + } + if err := renderInterop(stdout, doc, *format); err != nil { return err } @@ -202,6 +249,38 @@ func runInterop(stdout, stderr io.Writer, args []string) error { return nil } +// interopRegressions compares a fresh matrix against a previously-committed +// baseline and reports every (vector, participant) cell that was EQUIVALENT +// in the baseline but is not EQUIVALENT now. A cell that was never +// EQUIVALENT to begin with is not a regression — --baseline only guards +// against losing ground, not against pre-existing non-conformance (spec §17 +// Requirement 18). +func interopRegressions(base, cur interopDoc) []string { + baseEquivalent := map[string]bool{} + for _, row := range base.Vectors { + for _, c := range row.Cells { + if c.Equivalent { + baseEquivalent[row.Vector+"|"+c.Participant] = true + } + } + } + var regressions []string + for _, row := range cur.Vectors { + for _, c := range row.Cells { + key := row.Vector + "|" + c.Participant + if baseEquivalent[key] && !c.Equivalent { + detail := c.Detail + if detail == "" { + detail = "no longer equivalent" + } + regressions = append(regressions, fmt.Sprintf("%s: %s regressed from EQUIVALENT (%s)", row.Vector, c.Participant, detail)) + } + } + } + sort.Strings(regressions) + return regressions +} + // loadInteropVectors loads the embedded golden vectors (or those in dir) and // filters to protocol if non-empty. func loadInteropVectors(dir, protocol string) ([]interopVector, error) { @@ -323,6 +402,12 @@ func renderInterop(w io.Writer, doc interopDoc, format string) error { fmt.Fprintf(w, "| %s | %s | %s | %s |\n", r.Vector, r.Protocol, c.Participant, interopVerdict(c)) } } + if len(doc.Regressions) > 0 { + fmt.Fprintln(w, "\n## Regressions") + for _, r := range doc.Regressions { + fmt.Fprintf(w, "- %s\n", r) + } + } return nil case "text": for _, r := range doc.Vectors { @@ -335,6 +420,13 @@ func renderInterop(w io.Writer, doc interopDoc, format string) error { fmt.Fprintln(w, line) } } + if len(doc.Regressions) > 0 { + fmt.Fprintln(w, strings.Repeat("─", 60)) + fmt.Fprintln(w, "REGRESSIONS:") + for _, r := range doc.Regressions { + fmt.Fprintf(w, " %s\n", r) + } + } fmt.Fprintln(w, strings.Repeat("─", 60)) fmt.Fprintf(w, "RESULT: %s\n", doc.Result) return nil diff --git a/cmd/relay/interop_test.go b/cmd/relay/interop_test.go index 0d73f6f..d443903 100644 --- a/cmd/relay/interop_test.go +++ b/cmd/relay/interop_test.go @@ -6,6 +6,7 @@ package main import ( "bytes" + "encoding/json" "errors" "os" "path/filepath" @@ -156,6 +157,162 @@ func relayMsg(id string, payload []byte, meta map[string]string) relay.Message { return relay.Message{Protocol: relay.CAN, ID: id, Payload: payload, Meta: meta} } +//fusa:test REQ-RELAY-101 +func TestInteropMatrixFields(t *testing.T) { + // The JSON report is a versioned relay-interop-matrix/1 artifact with an + // explicit participants list, not just an ad hoc pass/fail blob. + bin := buildTestBinary(t) + var out, errb bytes.Buffer + if err := runInterop(&out, &errb, []string{"--protocol", "CAN", "--format", "json", bin}); err != nil { + t.Fatalf("interop: %v (%s)", err, errb.String()) + } + var doc interopDoc + if err := json.Unmarshal(out.Bytes(), &doc); err != nil { + t.Fatalf("unmarshal interop json: %v", err) + } + if doc.Kind != "relay-interop-matrix" { + t.Errorf("kind = %q, want relay-interop-matrix", doc.Kind) + } + if doc.MatrixVersion != "relay-interop-matrix/1" { + t.Errorf("matrix_version = %q, want relay-interop-matrix/1", doc.MatrixVersion) + } + if len(doc.Participants) != 1 || doc.Participants[0] != filepath.Base(bin) { + t.Errorf("participants = %v, want [%s]", doc.Participants, filepath.Base(bin)) + } + if len(doc.Regressions) != 0 { + t.Errorf("no --baseline given: regressions should be empty, got %v", doc.Regressions) + } +} + +//fusa:test REQ-RELAY-101 +func TestInteropBaselineRegression(t *testing.T) { + // A cell that was EQUIVALENT in a committed baseline but MISMATCHes now + // is a regression: --baseline must fail even though the mismatching + // binary was never checked against this specific baseline before. + // + // Restricted to a single accept-path (canonical) vector via --vectors: + // a broken-convert binary's reject-path cells look "correctly rejected" + // (Equivalent=true) by design (see the v.Kind=="error" branch), which + // would otherwise mask a genuine accept-path regression in this test. + bin := buildTestBinary(t) + dir := t.TempDir() + vecDir := t.TempDir() + raw, err := relay.Vector("can-standard-frame") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(vecDir, "v.json"), raw, 0o644); err != nil { + t.Fatal(err) + } + + var baseOut, errb bytes.Buffer + if err := runInterop(&baseOut, &errb, []string{"--vectors", vecDir, "--format", "json", bin}); err != nil { + t.Fatalf("baseline run: %v (%s)", err, errb.String()) + } + basePath := filepath.Join(dir, "baseline.json") + if err := os.WriteFile(basePath, baseOut.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + + // Now regress: same participant name (relay's own binary name), but a + // broken convert. --baseline compares by participant name, so drive the + // regressed run through a script literally named after the reference + // binary's basename to land on the same matrix key. + regressedDir := t.TempDir() + regressedBin := filepath.Join(regressedDir, filepath.Base(bin)) + script := "#!/bin/sh\ncase \"$1\" in\ncapabilities) echo '{\"kind\":\"capabilities\",\"tool\":\"fake\",\"protocol\":\"CAN\",\"commands\":[\"version\",\"capabilities\",\"status\",\"convert\"],\"adapt\":true}' ;;\nconvert) exit 1 ;;\nesac\n" + if err := os.WriteFile(regressedBin, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + + var out, errb2 bytes.Buffer + err = runInterop(&out, &errb2, []string{"--vectors", vecDir, "--format", "json", "--baseline", basePath, regressedBin}) + var code exitCode + if !errors.As(err, &code) || int(code) != 1 { + t.Fatalf("regressed run must exit 1, got %v (%s)", err, out.String()) + } + var doc interopDoc + if err := json.Unmarshal(out.Bytes(), &doc); err != nil { + t.Fatalf("unmarshal interop json: %v", err) + } + // Assert on the structured Regressions field directly, not just exit + // code/text: the underlying mismatch already forces exit 1 on its own, + // so this is the only assertion that actually exercises the regression + // detector rather than the ordinary any-mismatch-fails path. Must + // require an EXACT participant match (": regressed"), + // not a substring: the always-equivalent "relay (reference)" row that + // both the baseline and current run also carry contains the built + // test binary's own name ("relay") as a substring, which would + // otherwise mask a broken regression detector by matching on the + // reference row's own spurious "regression" instead of the real one. + participant := filepath.Base(bin) + wantPrefix := "can-standard-frame: " + participant + " regressed" + var found string + for _, r := range doc.Regressions { + if strings.HasPrefix(r, wantPrefix) { + found = r + break + } + } + if found == "" { + t.Fatalf("expected a regression naming participant %q, got %v", participant, doc.Regressions) + } + if !strings.Contains(found, "regressed from EQUIVALENT") { + t.Errorf("regression entry = %q, want it to mention regressed from EQUIVALENT", found) + } +} + +//fusa:test REQ-RELAY-101 +func TestInteropBaselineNoRegressionOnPreexistingMismatch(t *testing.T) { + // A participant that was never in the baseline (or never equivalent for + // a given vector) is not a "regression" — --baseline must not newly fail + // on pre-existing, already-known non-conformance. + emptyBaseline := filepath.Join(t.TempDir(), "empty.json") + if err := os.WriteFile(emptyBaseline, []byte(`{"kind":"relay-interop-matrix","matrix_version":"relay-interop-matrix/1","reference":"relay (reference)","participants":[],"result":"PASS","vectors":[]}`), 0o644); err != nil { + t.Fatal(err) + } + bad := writeScript(t, `echo '{"protocol":1,"id":"999999","payload":"","timestamp":"0001-01-01T00:00:00Z","meta":{}}'`) + var out, errb bytes.Buffer + err := runInterop(&out, &errb, []string{"--protocol", "CAN", "--baseline", emptyBaseline, bad}) + // Still fails overall (interop always fails on any current mismatch), + // but must not be reported as a *regression* since the baseline had no + // EQUIVALENT cell for this participant to regress from. + var code exitCode + if !errors.As(err, &code) || int(code) != 1 { + t.Fatalf("mismatch must still exit 1 regardless of baseline, got %v", err) + } + if strings.Contains(out.String(), "regressed from EQUIVALENT") { + t.Errorf("a cell absent from the baseline must not be reported as a regression:\n%s", out.String()) + } +} + +//fusa:test REQ-RELAY-101 +func TestInteropBaselineMissingFile(t *testing.T) { + bin := buildTestBinary(t) + var out, errb bytes.Buffer + err := runInterop(&out, &errb, []string{"--protocol", "CAN", "--baseline", filepath.Join(t.TempDir(), "nope.json"), bin}) + var code exitCode + if !errors.As(err, &code) || int(code) != 2 { + t.Errorf("missing --baseline file must exit 2, got %v", err) + } +} + +//fusa:test REQ-RELAY-101 +func TestInteropRegressionsHelper(t *testing.T) { + base := interopDoc{Vectors: []interopVectorResult{ + {Vector: "v1", Cells: []interopCell{{Participant: "a", Equivalent: true}}}, + {Vector: "v2", Cells: []interopCell{{Participant: "a", Equivalent: false}}}, // never equivalent + }} + cur := interopDoc{Vectors: []interopVectorResult{ + {Vector: "v1", Cells: []interopCell{{Participant: "a", Equivalent: false, Detail: "payload differs"}}}, // regressed + {Vector: "v2", Cells: []interopCell{{Participant: "a", Equivalent: false}}}, // still not equivalent, not a regression + }} + got := interopRegressions(base, cur) + if len(got) != 1 || !strings.Contains(got[0], "v1") || !strings.Contains(got[0], "payload differs") { + t.Errorf("interopRegressions = %v, want exactly one v1 regression mentioning the detail", got) + } +} + //fusa:test REQ-RELAY-083 func TestInteropBrokenConvertFails(t *testing.T) { // A spoke that ADVERTISES convert but whose convert errors is a conformance diff --git a/cmd/relay/main.go b/cmd/relay/main.go index 163d6fa..d9d40a5 100644 --- a/cmd/relay/main.go +++ b/cmd/relay/main.go @@ -18,7 +18,7 @@ import ( relay "github.com/SoundMatt/RELAY/v2" ) -const toolVersion = "2.7.0" +const toolVersion = "2.8.0" func main() { if err := run(os.Stdout, os.Stderr, os.Args[1:]); err != nil { diff --git a/spec/CHANGELOG.md b/spec/CHANGELOG.md index 78cdebf..71d6def 100644 --- a/spec/CHANGELOG.md +++ b/spec/CHANGELOG.md @@ -1,5 +1,59 @@ # RELAY Spec Changelog +## v2.8 — 2026-08-21 (MINOR — persisted, regression-gated interop matrix, §17 Requirement 18) + +- **New §17 Requirement 18 — persisted, regression-gated interop matrix.** + §20.1 item 3's existing behavioural-conformance gate only checked + EQUIVALENT-vs-reference as a one-shot pass/fail whose result vanished + after the CI run — no persisted artifact, no way to tell from git + history when a cross-language equivalence regression was introduced. + CI MUST now commit `relay interop --format json`'s output (a versioned + `relay-interop-matrix/1` document) covering every published sibling + implementation it can invoke, and MUST regenerate it with `relay + interop --baseline ` on every change, failing the + build if any cell that was `EQUIVALENT` in the committed copy is not + `EQUIVALENT` now. +- **Deliberately not pairwise (N²).** The proposal's literal reading + ("compare against every other sibling, not just the reference") would + mean every binary's `convert` output diffed against every other + binary's directly. Not built: byte-equality is transitive, so "A + equivalent to the shared reference" and "B equivalent to the shared + reference" already jointly establish "A equivalent to B" — the + existing O(N) hub-spoke comparison (every participant vs. one + in-process reference) carries the same information an O(N²) pairwise + matrix would, without the redundant `convert` invocations. Documented + explicitly in §17 and §20.1 rather than silently narrowed. +- **`relay interop`'s JSON output is now a versioned artifact**: `kind: + "relay-interop-matrix"`, `matrix_version: "relay-interop-matrix/1"`, + and an explicit `participants[]` list (so the *set* of compared + implementations is itself part of the diffable artifact — catching a + sibling silently dropped from CI's invocation, not just a per-cell + regression). +- **New `--baseline FILE` flag**: reports a `regressions[]` list — one + entry per (vector, participant) cell that was `EQUIVALENT` in the + baseline but is not now. A cell that was never `EQUIVALENT` to begin + with (e.g. an implementation still catching up to a newly-added golden + vector) is deliberately **not** reported as a regression — only actual + backsliding is; matches the issue's precise wording, distinct from the + existing any-mismatch-fails semantics §20.1 item 3 already enforces + independently. +- **New `spec/schemas/relay-interop-matrix.json`.** New `REQ-RELAY-101`. + 9 new/changed tests, including a real mutation test on the regression + detector's core comparison logic (caught first by a pure unit test + reverting `interopRegressions`'s condition; a follow-up integration + test needed restricting to a single accept-path vector after the + initial version was fooled by the reject-path's by-design leniency and + by a substring collision between the built test binary's name and the + ever-equivalent "relay (reference)" row — both fixed, then confirmed + the mutation was genuinely caught). +- **Also fixed a citation bug found while investigating this issue** (the + same class the issue itself flags for the audit's stale "§20.1.3"): + §20.4 cited "§20.1.2" for a specific CI-gate item, but §20.1 is a flat + numbered list, not subdivided into `.1`/`.2`/`.3` subsections. Corrected + to "§20.1 item 2". +- `SpecVersion` bumped `2.7` → `2.8` (MINOR, new §17 requirement). Closes + [NEW-SPEC-7]. + ## v2.7.1 — 2026-08-21 (doc addition; no normative change to existing conformant implementations) - **New §19.5 "This document's own version literals".** Requirement 14 diff --git a/spec/relay-spec.md b/spec/relay-spec.md index bb96059..18c249e 100644 --- a/spec/relay-spec.md +++ b/spec/relay-spec.md @@ -1,4 +1,4 @@ -# RELAY Specification — v2.7 +# RELAY Specification — v2.8 **Real-time Embedded Link Abstraction Yoke** @@ -1239,7 +1239,7 @@ failure rather than a skip. Exit: `0` all equivalent, `1` any mismatch/error, "protocol": "CAN", "protocol_int": 1, "version": "1.2.3", - "spec_version": "2.7", + "spec_version": "2.8", "language": "go", "runtime": "go1.25.0", "commit": "a1b2c3d4" @@ -1264,7 +1264,7 @@ conformance failure, but §17.2's conformance manifest cannot populate its own "protocol": "CAN", "protocol_int": 1, "version": "1.2.3", - "spec_version": "2.7", + "spec_version": "2.8", "commands": ["version", "capabilities", "status", "connect", "send", "subscribe"], "transports": ["socketcan", "virtual"], "features": ["fd", "isotp", "j1939"], @@ -1294,7 +1294,7 @@ CLI: "protocol_int": null, "multi_protocol": true, "version": "2.5.0", - "spec_version": "2.7", + "spec_version": "2.8", "commands": ["version", "capabilities", "status", "conform", "convert", "..."], "transports": [], "features": [], @@ -1363,7 +1363,7 @@ $ go-can version --format json "protocol": "CAN", "protocol_int": 1, "version": "1.2.3", - "spec_version": "2.7", + "spec_version": "2.8", "language": "go", "runtime": "go1.25.0" } @@ -1375,7 +1375,7 @@ $ go-can capabilities "protocol": "CAN", "protocol_int": 1, "version": "1.2.3", - "spec_version": "2.7", + "spec_version": "2.8", "commands": ["version", "capabilities", "status", "connect", "send", "subscribe"], "transports": ["socketcan", "virtual"], "features": ["fd", "isotp", "j1939"], @@ -1451,11 +1451,11 @@ LABEL org.opencontainers.image.licenses="MPL-2.0" LABEL io.relay.tool="" LABEL io.relay.language="go|cpp|rust|c" LABEL io.relay.binary="" -LABEL io.relay.spec-version="2.7" +LABEL io.relay.spec-version="2.8" ``` The `io.relay.spec-version` label MUST always match the value of `SpecVersion` -exported by the package (§17.12 / §19.4). The `"2.7"` shown above is an +exported by the package (§17.12 / §19.4). The `"2.8"` shown above is an example; per §19.5, this document's own copy of it is checked by this repository's CI against `spec/version.json` on every commit, the same way Requirement 14 requires of a conformant implementation. @@ -2299,6 +2299,7 @@ An implementation is **RELAY-conformant** if and only if: 15. **Conformance manifest.** The implementation MUST commit a `relay-conform/1` manifest (§17.2), generated by `relay conform --manifest` against its own built binary; CI MUST regenerate the manifest on every change and fail the build on any diff from the committed copy, or on any requirement entry whose `status` is `FAIL`. 16. **Vector manifest.** The canonical `spec/vectors/` distribution (§15.8) is pinned by `spec/vectors/vectors_manifest.json`. A conformant implementation that embeds a local copy of these vectors MUST embed the exact pinned set, and MUST have a CI step that fails when its embedded copy's SHA-256 diverges from the published manifest for the `vectors_version` it targets. 17. **No retired capabilities past removal.** Per §3.2, a capabilities document (§12.2) whose declared `spec_version` is at or past a `spec/version.json` `retired[]` entry's `removal` version MUST NOT list that entry's `name` in `features` or `commands`. +18. **Persisted, regression-gated interop matrix.** Per §20.1 item 3, `relay interop --format json` output (a `relay-interop-matrix/1` document, spec §20.1) MUST be committed by the implementation's CI, covering every published sibling implementation of its protocol it can invoke. CI MUST regenerate it and, using `relay interop --baseline `, fail the build if any cell that was `EQUIVALENT` in the committed copy is not `EQUIVALENT` in the fresh run — a cell that was never `EQUIVALENT` to begin with is not itself a gate failure under this requirement (the ordinary Requirement-7-adjacent "any current mismatch fails" gate already covers that case independently). `relay conform ` is a **black-box CLI tool**: it can only observe what the built binary's `version`/`capabilities`/`status` commands print, not the @@ -2355,7 +2356,12 @@ the check compares two things it already fetches over the CLI — the version document's `spec_version` and the capabilities document's `features`/`commands` — against `relay conform`'s own embedded `spec/version.json`, the same authoritative source §19.4 already points -to. No source-level or CI-process knowledge is needed. +to. No source-level or CI-process knowledge is needed. Requirement 18 +(persisted interop matrix) is the same category as Requirement 14 and 16: +`relay conform` never invokes `relay interop` at all, so whether a +sibling-comparison matrix was regenerated, committed, and checked for +regressions on this PR is entirely a CI-process fact about a different +command's invocation, verified by the implementation's own CI instead. **Requirement-to-verifier lookup table**, collecting the narrative above into one place: @@ -2379,6 +2385,7 @@ into one place: | 15 | Conformance manifest | `relay conform --manifest` (content); implementation's own CI (regenerate + diff) | Split — manifest content generated by `relay conform`; CI enforcement not observable through the CLI | | 16 | Vector manifest | Implementation's own CI | Not observable through the CLI | | 17 | No retired capabilities past removal | `relay conform` (version + capabilities cross-check against embedded `spec/version.json`) | Full | +| 18 | Persisted, regression-gated interop matrix | Implementation's own CI (`relay interop --baseline`) | Not observable through the CLI | ### 17.1 Wire-format regression coverage (recommended) @@ -2432,7 +2439,7 @@ diffable artifact. "manifest_version": "relay-conform/1", "tool": "go-can", "binary_version": "1.2.3", - "spec_version": "2.7", + "spec_version": "2.8", "git_sha": "a1b2c3d4", "capabilities_sha256": "e3b0c44298fc1c14...", "requirements": [ @@ -2451,7 +2458,7 @@ SHA-256 of the raw `capabilities` document bytes as returned by the binary, letting a diff catch any change to the capabilities document — mechanically regenerated or not — without re-parsing it. -`requirements` MUST contain exactly one entry per §17 requirement (1–17, +`requirements` MUST contain exactly one entry per §17 requirement (1–18, extended as this document's own requirement list grows). Each entry's `status` is one of: @@ -3181,11 +3188,11 @@ clarifications and fixes in PATCH releases. `spec/version.json` is authoritative. The spec document title is informational. -Current version: **v2.7** +Current version: **v2.8** -**Go:** `const SpecVersion = "2.7"` (update in implementations targeting v2.7) -**C++:** `constexpr std::string_view kRelaySpecVersion = "2.7";` -**Rust:** `pub const RELAY_SPEC_VERSION: &str = "2.7";` +**Go:** `const SpecVersion = "2.8"` (update in implementations targeting v2.8) +**C++:** `constexpr std::string_view kRelaySpecVersion = "2.8";` +**Rust:** `pub const RELAY_SPEC_VERSION: &str = "2.8";` ### 19.5 This document's own version literals @@ -3235,7 +3242,19 @@ every pull request, run and gate on: requirement, or a failed qualification case MUST fail the job. 3. **Behavioural conformance** — once `convert` is implemented (§20.3), `relay interop` against the reference MUST report the implementation - EQUIVALENT for every golden vector of its protocol. + EQUIVALENT for every golden vector of its protocol. Additionally + (Requirement 18): CI MUST commit `relay interop --format json`'s + `relay-interop-matrix/1` output, covering every published sibling + implementation of the same protocol it can invoke, and regenerate it + with `relay interop --baseline ` on every change, + failing the job if any cell that was `EQUIVALENT` in the committed + copy is not `EQUIVALENT` now. Every participant is compared against + the same in-process reference rather than pairwise against every + other sibling: equivalence is byte-equality, a transitive relation, so + "A equivalent to reference" and "B equivalent to reference" already + jointly establish "A equivalent to B" — an O(N) hub comparison + carries the same information an O(N²) pairwise one would, without the + redundant work. 4. **Conformance manifest** (§17.2, Requirement 15) — regenerate the `relay-conform/1` manifest with `relay conform --manifest` against the built CLI and fail the job on any diff from the committed copy, or on @@ -3269,7 +3288,7 @@ makes envelope conformance machine-checkable and identical across languages. ### 20.4 Implementation evidence A conformant implementation MUST carry, and keep current via its x-FuSa CI -(§20.1.2), the following evidence (the x-FuSa toolchain owns their schemas; RELAY +(§20.1 item 2), the following evidence (the x-FuSa toolchain owns their schemas; RELAY requires their presence): - a **requirements registry** in which every requirement is traced and tested; @@ -3303,7 +3322,7 @@ manifest, the §15.8 vector manifest's `vectors_version`, and the targeted {"name": "go-can", "digest": {"sha256": "9f86d081884c7d65..."}} ], "predicate": { - "spec_version": "2.7", + "spec_version": "2.8", "vectors_version": "2.4", "conformance_manifest": { "...": "the full §17.2 relay-conform/1 document" }, "safety_evidence_summary": null, diff --git a/spec/schemas/relay-conform-manifest.json b/spec/schemas/relay-conform-manifest.json index 7028169..8f7e96b 100644 --- a/spec/schemas/relay-conform-manifest.json +++ b/spec/schemas/relay-conform-manifest.json @@ -56,7 +56,7 @@ "required": ["id", "status", "verifier"], "additionalProperties": false }, - "description": "One entry per §17 requirement (1-17, extended as the requirement list grows)." + "description": "One entry per §17 requirement (1-18, extended as the requirement list grows)." }, "overall": { "type": "string", diff --git a/spec/schemas/relay-interop-matrix.json b/spec/schemas/relay-interop-matrix.json new file mode 100644 index 0000000..206193b --- /dev/null +++ b/spec/schemas/relay-interop-matrix.json @@ -0,0 +1,64 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/SoundMatt/RELAY/spec/schemas/relay-interop-matrix.json", + "title": "RELAY interop matrix", + "description": "JSON output of `relay interop --format json` (spec §20.1 item 3, §17 Requirement 18).", + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "relay-interop-matrix" + }, + "matrix_version": { + "type": "string", + "const": "relay-interop-matrix/1" + }, + "reference": { + "type": "string", + "description": "Label for the in-process reference conversion every participant is compared against." + }, + "participants": { + "type": "array", + "items": { "type": "string" }, + "description": "Sorted base names of every binary compared in this run." + }, + "result": { + "type": "string", + "enum": ["PASS", "FAIL"] + }, + "vectors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "vector": { "type": "string" }, + "protocol": { "type": "string" }, + "cells": { + "type": "array", + "items": { + "type": "object", + "properties": { + "participant": { "type": "string" }, + "ok": { "type": "boolean" }, + "equivalent": { "type": "boolean" }, + "skipped": { "type": "boolean" }, + "detail": { "type": "string" } + }, + "required": ["participant", "ok", "equivalent", "skipped"], + "additionalProperties": false + } + } + }, + "required": ["vector", "protocol", "cells"], + "additionalProperties": false + } + }, + "regressions": { + "type": "array", + "items": { "type": "string" }, + "description": "Present only when generated with --baseline: one entry per (vector, participant) cell that was EQUIVALENT in the baseline but is not in this run." + } + }, + "required": ["kind", "matrix_version", "reference", "participants", "result", "vectors"], + "additionalProperties": false +} diff --git a/spec/version.json b/spec/version.json index c3b2a88..e59287a 100644 --- a/spec/version.json +++ b/spec/version.json @@ -1,5 +1,5 @@ { - "version": "2.7", + "version": "2.8", "date": "2026-08-21", "status": "stable", "networks": [ diff --git a/version.go b/version.go index e4cd4d1..483afc1 100644 --- a/version.go +++ b/version.go @@ -8,4 +8,4 @@ package relay // Every RELAY-conformant implementation must export this constant (spec §17 req 12). // //fusa:req REQ-RELAY-020 -const SpecVersion = "2.7" +const SpecVersion = "2.8" diff --git a/version_test.go b/version_test.go index f4de744..85d9e3a 100644 --- a/version_test.go +++ b/version_test.go @@ -8,7 +8,7 @@ import "testing" //fusa:test REQ-RELAY-020 func TestSpecVersion(t *testing.T) { - if SpecVersion != "2.7" { - t.Errorf("SpecVersion = %q, want %q", SpecVersion, "2.7") + if SpecVersion != "2.8" { + t.Errorf("SpecVersion = %q, want %q", SpecVersion, "2.8") } } From b9902ca34fc0e786257adf76b68ef130e4c441b2 Mon Sep 17 00:00:00 2001 From: Matt <47545907+SoundMatt@users.noreply.github.com> Date: Fri, 21 Aug 2026 06:44:38 -0700 Subject: [PATCH 2/2] fix: cover the new interop --baseline error path and regression-rendering branches CI's 90% coverage gate caught 3 genuinely-uncovered branches from the previous commit: malformed --baseline JSON (distinct from the file-not-found path, which was already covered), and the markdown/text renderers' new Regressions sections (only the JSON format's doc.Regressions content was exercised before). 3 new tests: TestInteropBaselineInvalidJSON, TestRenderInteropRegressions (both text and markdown). Total coverage 89.9% -> 90.3%. Signed-off-by: Matt <47545907+SoundMatt@users.noreply.github.com> --- cmd/relay/interop_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/cmd/relay/interop_test.go b/cmd/relay/interop_test.go index d443903..1729263 100644 --- a/cmd/relay/interop_test.go +++ b/cmd/relay/interop_test.go @@ -313,6 +313,44 @@ func TestInteropRegressionsHelper(t *testing.T) { } } +//fusa:test REQ-RELAY-101 +func TestInteropBaselineInvalidJSON(t *testing.T) { + bad := filepath.Join(t.TempDir(), "bad.json") + if err := os.WriteFile(bad, []byte("not json"), 0o644); err != nil { + t.Fatal(err) + } + bin := buildTestBinary(t) + var out, errb bytes.Buffer + err := runInterop(&out, &errb, []string{"--protocol", "CAN", "--baseline", bad, bin}) + var code exitCode + if !errors.As(err, &code) || int(code) != 2 { + t.Errorf("malformed --baseline JSON must exit 2, got %v", err) + } + if !strings.Contains(errb.String(), "not a relay-interop-matrix/1 document") { + t.Errorf("expected a parse-error message, got %q", errb.String()) + } +} + +//fusa:test REQ-RELAY-101 +func TestRenderInteropRegressions(t *testing.T) { + doc := interopDoc{ + Kind: "relay-interop-matrix", MatrixVersion: "relay-interop-matrix/1", + Reference: "relay (reference)", Participants: []string{"go-can"}, + Result: "FAIL", + Vectors: []interopVectorResult{{Vector: "v1", Protocol: "CAN", Cells: []interopCell{{Participant: "go-can", OK: true, Equivalent: false}}}}, + Regressions: []string{"v1: go-can regressed from EQUIVALENT (payload differs)"}, + } + for _, format := range []string{"text", "markdown"} { + var out bytes.Buffer + if err := renderInterop(&out, doc, format); err != nil { + t.Fatalf("renderInterop(%s): %v", format, err) + } + if !strings.Contains(out.String(), "v1: go-can regressed from EQUIVALENT") { + t.Errorf("%s output missing regression line:\n%s", format, out.String()) + } + } +} + //fusa:test REQ-RELAY-083 func TestInteropBrokenConvertFails(t *testing.T) { // A spoke that ADVERTISES convert but whose convert errors is a conformance