From f4dab7eb25a455198b6451dc257ffecfd9391048 Mon Sep 17 00:00:00 2001 From: Justin Kaseman Date: Sat, 14 Nov 2026 11:43:50 -0800 Subject: [PATCH 1/2] (test): Add vault OCR fuzzing --- .../ocr2/plugins/vault/plugin_fuzz_test.go | 187 ++++++++++++ core/services/ring/plugin_fuzz_test.go | 273 ++++++++++++++++++ 2 files changed, 460 insertions(+) create mode 100644 core/services/ocr2/plugins/vault/plugin_fuzz_test.go create mode 100644 core/services/ring/plugin_fuzz_test.go diff --git a/core/services/ocr2/plugins/vault/plugin_fuzz_test.go b/core/services/ocr2/plugins/vault/plugin_fuzz_test.go new file mode 100644 index 00000000000..3212d88ce28 --- /dev/null +++ b/core/services/ocr2/plugins/vault/plugin_fuzz_test.go @@ -0,0 +1,187 @@ +package vault + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "testing" + "unicode/utf8" + + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/smartcontractkit/libocr/commontypes" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3_1types" + "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + "github.com/smartcontractkit/tdh2/go/tdh2/tdh2easy" + + vaultcommon "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault" + "github.com/smartcontractkit/chainlink-common/pkg/logger" +) + +const ( + fuzzN = 4 + fuzzF = 1 + fuzzOwner = "0x1111111111111111111111111111111111111111" + fuzzRequest = "request-1" + fuzzByzantineID = 3 +) + +// unavailableBlobs lets a Byzantine observation reference blob handles that no oracle ever broadcast. +type unavailableBlobs struct{} + +func (unavailableBlobs) FetchBlob(context.Context, ocr3_1types.BlobHandle) ([]byte, error) { + return nil, errors.New("blob unavailable") +} + +func fuzzPendingItem(t *testing.T, pk *tdh2easy.PublicKey, kind uint8, owner, namespace, key string, secret []byte, corrupt bool) proto.Message { + t.Helper() + id := &vaultcommon.SecretIdentifier{Owner: owner, Namespace: namespace, Key: key} + + encrypt := func() string { + var label [32]byte + copy(label[12:], common.HexToAddress(owner).Bytes()) + ct, err := tdh2easy.EncryptWithLabel(pk, secret, label) + require.NoError(t, err) + raw, err := ct.Marshal() + require.NoError(t, err) + if corrupt && len(raw) > 0 { + raw[len(raw)/2] ^= 0xff + } + return hex.EncodeToString(raw) + } + + switch kind % 5 { + case 0: + return &vaultcommon.GetSecretsRequest{Requests: []*vaultcommon.SecretRequest{{Id: id}}} + case 1: + return &vaultcommon.CreateSecretsRequest{EncryptedSecrets: []*vaultcommon.EncryptedSecret{{Id: id, EncryptedValue: encrypt()}}} + case 2: + return &vaultcommon.UpdateSecretsRequest{EncryptedSecrets: []*vaultcommon.EncryptedSecret{{Id: id, EncryptedValue: encrypt()}}} + case 3: + return &vaultcommon.DeleteSecretsRequest{Ids: []*vaultcommon.SecretIdentifier{id}} + default: + return &vaultcommon.ListSecretIdentifiersRequest{Owner: owner, Namespace: namespace} + } +} + +func FuzzReportingPlugin_StateTransitionAcrossOracles(f *testing.F) { + _, pk, shares, err := tdh2easy.GenerateKeys(1, 3) + require.NoError(f, err) + + f.Add(uint64(1), uint8(0), true, fuzzOwner, "main", "secret", []byte("value"), false, false, false, []byte{}) + f.Add(uint64(1), uint8(1), false, fuzzOwner, "main", "secret", []byte("value"), false, false, false, []byte{}) + f.Add(uint64(2), uint8(2), true, fuzzOwner, "main", "secret", []byte("value"), true, false, false, []byte{0x0a, 0x00}) + f.Add(uint64(3), uint8(3), true, fuzzOwner, "main", "secret", []byte{}, false, false, false, []byte{0xff}) + f.Add(uint64(4), uint8(4), true, "owner", "", "", []byte{}, false, false, false, []byte{}) + // A Byzantine oracle sends a well-formed observation computed from the opposite KV state, so the oracles disagree; + // with a second oracle doing the same, two different observations each have F+1 supporters. + for kind := range uint8(5) { + for _, preexisting := range []bool{true, false} { + f.Add(uint64(6), kind, preexisting, fuzzOwner, "main", "secret", []byte("value"), false, true, false, []byte{}) + f.Add(uint64(6), kind, preexisting, fuzzOwner, "main", "secret", []byte("value"), false, true, true, []byte{}) + } + } + danglingBlob, err := proto.Marshal(&vaultcommon.Observations{ + SortNonce: bytes.Repeat([]byte{'a'}, sortNonceLength), + PendingQueueItems: [][]byte{{0x00}}, + }) + require.NoError(f, err) + f.Add(uint64(5), uint8(0), true, fuzzOwner, "main", "secret", []byte("value"), false, false, false, danglingBlob) + + f.Fuzz(func(t *testing.T, seqNr uint64, kind uint8, preexisting bool, owner, namespace, key string, secret []byte, corrupt, byzantineFlipsState, honestOracleFlipsState bool, byzantine []byte) { + if !utf8.ValidString(owner) || !utf8.ValidString(namespace) || !utf8.ValidString(key) { + t.Skip("proto3 strings must be valid UTF-8") + } + if len(secret) == 0 { + secret = []byte("x") + } + item := fuzzPendingItem(t, pk, kind, owner, namespace, key, secret, corrupt) + anyItem, err := anypb.New(item) + require.NoError(t, err) + + id := &vaultcommon.SecretIdentifier{Owner: owner, Namespace: namespace, Key: key} + kvWith := func(secretExists bool) *kv { + rdr := &kv{m: make(map[string]response)} + w := newTestWriteStore(t, rdr) + require.NoError(t, w.WritePendingQueue(t.Context(), []*vaultcommon.StoredPendingQueueItem{{Id: fuzzRequest, Item: anyItem}})) + if secretExists { + require.NoError(t, w.WriteSecret(t.Context(), id, &vaultcommon.StoredSecret{EncryptedSecret: []byte("ciphertext")})) + require.NoError(t, w.WriteMetadata(t.Context(), owner, &vaultcommon.StoredMetadata{SecretIdentifiers: []*vaultcommon.SecretIdentifier{id}})) + } + return rdr + } + freshKV := func() *kv { return kvWith(preexisting) } + newPlugin := func(i int) *ReportingPlugin { + return newTestReportingPlugin(t, withKeys(pk, shares[i]), withOnchainCfg(fuzzN, fuzzF), withLggr(logger.Nop()), withBatchSize(10)) + } + + aos := make([]types.AttributedObservation, 0, fuzzN) + for i := range fuzzN - 1 { + state := freshKV() + if honestOracleFlipsState && i == fuzzN-2 { + state = kvWith(!preexisting) + } + data, err := newPlugin(i).Observation(t.Context(), seqNr, types.AttributedQuery{}, state, &blobber{}) + require.NoError(t, err, "an honest oracle must be able to observe a well-formed pending request") + aos = append(aos, types.AttributedObservation{Observer: commontypes.OracleID(i), Observation: data}) + } + if byzantineFlipsState { + data, err := newPlugin(fuzzByzantineID-1).Observation(t.Context(), seqNr, types.AttributedQuery{}, kvWith(!preexisting), &blobber{}) + require.NoError(t, err) + byzantine = data + } + aos = append(aos, types.AttributedObservation{Observer: fuzzByzantineID, Observation: byzantine}) + + validator := newPlugin(0) + var validated []types.AttributedObservation + for i, ao := range aos { + err := validator.ValidateObservation(t.Context(), seqNr, types.AttributedQuery{}, ao, freshKV(), unavailableBlobs{}) + if i < fuzzN-1 && !honestOracleFlipsState { + require.NoError(t, err, "honest observation must validate") + } + if err == nil { + validated = append(validated, ao) + } + } + + type result struct { + precursor ocr3_1types.ReportsPlusPrecursor + errMsg string + kv map[string]response + } + run := func(oracle int) result { + rdr := freshKV() + p, err := newPlugin(oracle).StateTransition(t.Context(), seqNr, types.AttributedQuery{}, validated, rdr, unavailableBlobs{}) + res := result{precursor: p, kv: rdr.m} + if err != nil { + res.errMsg = err.Error() + } + return res + } + + first := run(0) + // Several runs per oracle so a dependence on map iteration order has a chance to show. + for range 3 { + for oracle := range fuzzN - 1 { + next := run(oracle) + require.Equal(t, first.errMsg, next.errMsg) + require.Equal(t, first.precursor, next.precursor, "StateTransition must be deterministic") + require.Equal(t, first.kv, next.kv, "KV writes must be deterministic") + } + } + if first.errMsg != "" { + return + } + + want, wantErr := newPlugin(0).Reports(t.Context(), seqNr, first.precursor) + for oracle := range fuzzN - 1 { + got, err := newPlugin(oracle).Reports(t.Context(), seqNr, first.precursor) + require.Equal(t, wantErr == nil, err == nil) + require.Equal(t, want, got, "Reports must not depend on the oracle's private key share") + } + }) +} diff --git a/core/services/ring/plugin_fuzz_test.go b/core/services/ring/plugin_fuzz_test.go new file mode 100644 index 00000000000..8de56ac2cc8 --- /dev/null +++ b/core/services/ring/plugin_fuzz_test.go @@ -0,0 +1,273 @@ +package ring + +import ( + "maps" + "math" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/smartcontractkit/libocr/commontypes" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + ringpb "github.com/smartcontractkit/chainlink-protos/ring/go" +) + +const ( + fuzzN = 4 + fuzzF = 1 + // An honest arbiter reports a small shard set; consistent hashing panics beyond its 997 partitions, which a + // healthy-shard quorum (needing an honest vote) can never reach. + maxFuzzShardIDs = 64 +) + +func newFuzzPlugin(t *testing.T) *Plugin { + t.Helper() + cfg := ocr3types.ReportingPluginConfig{N: fuzzN, F: fuzzF} + p, err := NewPlugin(NewStore(), &mockArbiter{}, cfg, logger.Nop(), nil) + require.NoError(t, err) + return p +} + +func seedObservation(t testing.TB, wantShards uint32, workflows []string) []byte { + b, err := proto.Marshal(&ringpb.Observation{ + ShardStatus: toShardStatus(map[uint32]bool{0: true, 1: true, 2: false}), + WorkflowIds: workflows, + Now: timestamppb.New(time.Unix(1_700_000_000, 0)), + WantShards: wantShards, + }) + require.NoError(t, err) + return b +} + +func seedObservationWith(t testing.TB, obs *ringpb.Observation) []byte { + b, err := proto.Marshal(obs) + require.NoError(t, err) + return b +} + +func priorOutcome(t *testing.T, seqNr uint64, shards uint32, transition bool, safeAfterUnix int64) ocr3types.Outcome { + t.Helper() + if shards == 0 { + return nil + } + state := &ringpb.RoutingState{Id: seqNr, State: &ringpb.RoutingState_RoutableShards{RoutableShards: shards}} + if transition { + state.State = &ringpb.RoutingState_Transition{Transition: &ringpb.Transition{ + WantShards: shards, + LastStableCount: shards + 1, + ChangesSafeAfter: timestamppb.New(time.Unix(safeAfterUnix, 0)), + }} + } + b, err := proto.MarshalOptions{Deterministic: true}.Marshal(&ringpb.Outcome{State: state}) + require.NoError(t, err) + return b +} + +func parseObservations(aos []types.AttributedObservation) []*ringpb.Observation { + out := make([]*ringpb.Observation, 0, len(aos)) + for _, ao := range aos { + obs := &ringpb.Observation{} + if proto.Unmarshal(ao.Observation, obs) == nil { + out = append(out, obs) + } + } + return out +} + +func distinctShardIDs(observations []*ringpb.Observation) int { + ids := map[uint32]struct{}{} + for _, obs := range observations { + for id := range obs.ShardStatus { + ids[id] = struct{}{} + } + } + return len(ids) +} + +// healthyShards is the set of shards reported healthy by more than F oracles. +func healthyShards(observations []*ringpb.Observation) []uint32 { + votes := map[uint32]int{} + for _, obs := range observations { + for id, status := range obs.ShardStatus { + if status != nil && status.IsHealthy { + votes[id]++ + } + } + } + var healthy []uint32 + for id, n := range votes { + if n > fuzzF { + healthy = append(healthy, id) + } + } + slices.Sort(healthy) + return healthy +} + +func medianWantShards(observations []*ringpb.Observation) uint32 { + votes := make([]uint32, 0, len(observations)) + for _, obs := range observations { + votes = append(votes, obs.WantShards) + } + slices.Sort(votes) + return votes[len(votes)/2] +} + +func medianNow(observations []*ringpb.Observation) time.Time { + nows := make([]time.Time, 0, len(observations)) + for _, obs := range observations { + nows = append(nows, obs.Now.AsTime()) + } + slices.SortFunc(nows, time.Time.Compare) + return nows[len(nows)/2] +} + +// expectedNextState spells out the routing state machine: a steady state moves to a transition when the agreed shard +// count changes, and a transition settles only once its safe time has passed. +func expectedNextState(prior *ringpb.RoutingState, wantShards uint32, now time.Time) *ringpb.RoutingState { + switch s := prior.State.(type) { + case *ringpb.RoutingState_RoutableShards: + if s.RoutableShards == wantShards { + return &ringpb.RoutingState{Id: prior.Id, State: &ringpb.RoutingState_RoutableShards{RoutableShards: wantShards}} + } + return &ringpb.RoutingState{Id: prior.Id + 1, State: &ringpb.RoutingState_Transition{Transition: &ringpb.Transition{ + WantShards: wantShards, + LastStableCount: s.RoutableShards, + ChangesSafeAfter: timestamppb.New(now.Add(DefaultTimeToSync)), + }}} + case *ringpb.RoutingState_Transition: + if now.Before(s.Transition.ChangesSafeAfter.AsTime()) { + return prior + } + return &ringpb.RoutingState{Id: prior.Id + 1, State: &ringpb.RoutingState_RoutableShards{RoutableShards: s.Transition.WantShards}} + } + return nil +} + +func requireOutcomeInvariants(t *testing.T, raw []byte, prev ocr3types.Outcome, seqNr uint64, observations []*ringpb.Observation) { + t.Helper() + outcome := &ringpb.Outcome{} + require.NoError(t, proto.Unmarshal(raw, outcome)) + require.NotNil(t, outcome.State) + + workflows := map[string]struct{}{} + for _, obs := range observations { + for _, id := range obs.WorkflowIds { + workflows[id] = struct{}{} + } + } + require.Equal(t, slices.Sorted(maps.Keys(workflows)), slices.Sorted(maps.Keys(outcome.Routes)), "every observed workflow must be routed, and only those") + + healthy := healthyShards(observations) + for wfID, route := range outcome.Routes { + if len(healthy) == 0 { + require.Zero(t, route.DonId, "workflow %q must fall back to shard 0 when no shard is healthy", wfID) + continue + } + want, err := locateShard(newShardRing(healthy), wfID) + require.NoError(t, err) + require.Equal(t, want, route.DonId, "workflow %q must be placed by consistent hashing over the healthy shards", wfID) + } + + wantShards := medianWantShards(observations) + prior := &ringpb.Outcome{State: &ringpb.RoutingState{Id: seqNr, State: &ringpb.RoutingState_RoutableShards{RoutableShards: wantShards}}} + if prev != nil { + prior = &ringpb.Outcome{} + require.NoError(t, proto.Unmarshal(prev, prior)) + } + expected := expectedNextState(prior.State, wantShards, medianNow(observations)) + require.True(t, proto.Equal(expected, outcome.State), "routing state must follow the transition rules: want %v, got %v", expected, outcome.State) +} + +func FuzzPlugin_OutcomeInvariants(f *testing.F) { + obsA := seedObservation(f, 2, []string{"a", "b"}) + obsB := seedObservation(f, 3, []string{"b", "c"}) + obsC := seedObservation(f, 2, nil) + obsWant3 := seedObservation(f, 3, []string{"a"}) + f.Add(uint64(1), uint32(2), false, int64(0), obsA, obsB, obsC, []byte{}) + f.Add(uint64(7), uint32(3), true, int64(1_700_000_000+3600), obsA, obsB, obsC, obsA) + f.Add(uint64(7), uint32(3), true, int64(1_600_000_000), obsA, obsB, obsC, obsA) + f.Add(uint64(9), uint32(2), false, int64(0), obsWant3, obsWant3, obsWant3, obsWant3) + // shard 5 is healthy according to a single oracle, which is not enough to route to it + obsShard5 := seedObservationWith(f, &ringpb.Observation{ + ShardStatus: toShardStatus(map[uint32]bool{0: true, 1: true, 5: true}), + WorkflowIds: []string{"a", "b", "c", "d"}, + Now: timestamppb.New(time.Unix(1_700_000_000, 0)), + WantShards: 2, + }) + f.Add(uint64(3), uint32(2), false, int64(0), obsShard5, obsA, obsA, obsC) + f.Add(uint64(0), uint32(0), false, int64(0), []byte{}, []byte{0xff, 0xff}, []byte{0x0a}, []byte{0x12, 0x00}) + + f.Fuzz(func(t *testing.T, seqNr uint64, priorShards uint32, transition bool, safeAfterUnix int64, o0, o1, o2, o3 []byte) { + var aos []types.AttributedObservation + validator := newFuzzPlugin(t) + for i, raw := range [][]byte{o0, o1, o2, o3} { + ao := types.AttributedObservation{Observation: raw, Observer: commontypes.OracleID(i)} + if validator.ValidateObservation(t.Context(), ocr3types.OutcomeContext{}, nil, ao) == nil { + aos = append(aos, ao) + } + } + if len(aos) == 0 { + return + } + observations := parseObservations(aos) + if distinctShardIDs(observations) > maxFuzzShardIDs { + t.Skip("more shards than an arbiter reports") + } + + prev := priorOutcome(t, seqNr, priorShards, transition, safeAfterUnix) + outctx := ocr3types.OutcomeContext{SeqNr: seqNr, PreviousOutcome: prev} + + first, err1 := newFuzzPlugin(t).Outcome(t.Context(), outctx, nil, aos) + second, err2 := newFuzzPlugin(t).Outcome(t.Context(), outctx, nil, aos) + require.Equal(t, err1 == nil, err2 == nil) + require.Equal(t, first, second, "same inputs must yield identical outcome") + if err1 != nil { + return + } + + if seqNr != math.MaxUint64 { + requireOutcomeInvariants(t, first, prev, seqNr, observations) + } + + reports, err := newFuzzPlugin(t).Reports(t.Context(), seqNr, first) + require.NoError(t, err) + require.Len(t, reports, 1) + require.Equal(t, first, ocr3types.Outcome(reports[0].ReportWithInfo.Report), "report must carry the outcome verbatim") + again, err := newFuzzPlugin(t).Reports(t.Context(), seqNr, first) + require.NoError(t, err) + require.Equal(t, reports, again, "same outcome must yield identical reports") + }) +} + +func FuzzPlugin_ValidatedObservationYieldsOutcome(f *testing.F) { + f.Add(seedObservation(f, 2, []string{"a"})) + f.Add([]byte{}) + f.Add([]byte{0x10, 0x01}) + f.Add(seedObservationWith(f, &ringpb.Observation{WantShards: 2})) // no timestamp + f.Add(seedObservationWith(f, &ringpb.Observation{Now: timestamppb.New(time.Unix(1, 0))})) // no shard count + + f.Fuzz(func(t *testing.T, raw []byte) { + p := newFuzzPlugin(t) + ao := types.AttributedObservation{Observation: raw} + if err := p.ValidateObservation(t.Context(), ocr3types.OutcomeContext{}, nil, ao); err != nil { + return + } + + observation := &ringpb.Observation{} + require.NoError(t, proto.Unmarshal(raw, observation)) + require.NotNil(t, observation.Now, "a validated observation must carry a timestamp") + require.NotZero(t, observation.WantShards, "a validated observation must carry a shard count") + + out, err := p.Outcome(t.Context(), ocr3types.OutcomeContext{SeqNr: 1}, nil, []types.AttributedObservation{ao}) + require.NoError(t, err, "a validated observation must produce an outcome") + requireOutcomeInvariants(t, out, nil, 1, []*ringpb.Observation{observation}) + }) +} From 5d8d80747d077226b7fab14bd671575d6960f1ec Mon Sep 17 00:00:00 2001 From: Justin Kaseman Date: Thu, 19 Nov 2026 14:12:31 -0800 Subject: [PATCH 2/2] (test): Close vault plugins in fuzz harness to stop leaking limiters --- core/services/ocr2/plugins/vault/plugin_fuzz_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/services/ocr2/plugins/vault/plugin_fuzz_test.go b/core/services/ocr2/plugins/vault/plugin_fuzz_test.go index 3212d88ce28..9bfd1efabde 100644 --- a/core/services/ocr2/plugins/vault/plugin_fuzz_test.go +++ b/core/services/ocr2/plugins/vault/plugin_fuzz_test.go @@ -116,7 +116,10 @@ func FuzzReportingPlugin_StateTransitionAcrossOracles(f *testing.F) { } freshKV := func() *kv { return kvWith(preexisting) } newPlugin := func(i int) *ReportingPlugin { - return newTestReportingPlugin(t, withKeys(pk, shares[i]), withOnchainCfg(fuzzN, fuzzF), withLggr(logger.Nop()), withBatchSize(10)) + p := newTestReportingPlugin(t, withKeys(pk, shares[i]), withOnchainCfg(fuzzN, fuzzF), withLggr(logger.Nop()), withBatchSize(10)) + // Each plugin owns limiters with background updaters; leaking them exhausts memory within seconds of fuzzing. + t.Cleanup(func() { require.NoError(t, p.Close()) }) + return p } aos := make([]types.AttributedObservation, 0, fuzzN)