diff --git a/.changeset/clever-towns-tan.md b/.changeset/clever-towns-tan.md new file mode 100644 index 00000000000..1f21b72f0f2 --- /dev/null +++ b/.changeset/clever-towns-tan.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +#updated observation cache TTL and bridge cache-fallback test coverage diff --git a/.github/workflows/cre-system-tests.yaml b/.github/workflows/cre-system-tests.yaml index 43c1152970c..9c77b222255 100644 --- a/.github/workflows/cre-system-tests.yaml +++ b/.github/workflows/cre-system-tests.yaml @@ -126,6 +126,9 @@ jobs: "Test_CRE_V2_ShardManualAssignment": [ {"topology":"workflow-gateway-sharded-manual","configs":"configs/workflow-gateway-sharded-manual.toml"} ], + "Test_CRE_V2_ShardedCapabilitiesManualEVMLogTrigger": [ + {"topology":"workflow-sharded-capabilities","configs":"configs/workflow-sharded-capabilities-don.toml"} + ], "Test_CRE_V2_ShardRingOCROverrides": [ {"topology":"workflow-gateway-sharded-ringocr-overrides","configs":"configs/workflow-gateway-sharded-ringocr-overrides.toml"} ], diff --git a/core/config/toml/types.go b/core/config/toml/types.go index c2949270b50..dac2de6244f 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -180,8 +180,9 @@ func (s *SolKeys) SetFrom(f *SolKeys) error { if f == nil || len(f.Keys) == 0 { return nil } - s.Keys = make([]*SolKey, len(f.Keys)) - copy(s.Keys, f.Keys) + // Replacing would discard the keys contributed by earlier -s files; + // validateMerge has already rejected any duplicated across them. + s.Keys = append(s.Keys, f.Keys...) return nil } @@ -189,9 +190,15 @@ func (s *SolKeys) validateMerge(f *SolKeys) (err error) { have := make(map[string]struct{}) if s != nil && f != nil { for _, solKey := range s.Keys { + if solKey.ID == nil { + continue + } have[*solKey.ID] = struct{}{} } for _, solKey := range f.Keys { + if solKey.ID == nil { + continue // a missing ID is a validation error, not a merge conflict + } if _, ok := have[*solKey.ID]; ok { err = errors.Join(err, configutils.ErrOverride{Name: "SolKeys: " + *solKey.ID}) } @@ -240,7 +247,7 @@ func (e *SolKey) validateMerge(f *SolKey) (err error) { } func (e *SolKey) ValidateConfig() (err error) { - if (e.JSON != nil) != (e.Password != nil) && (e.Password != nil) != (e.ID != nil) { + if (e.JSON != nil) != (e.Password != nil) || (e.Password != nil) != (e.ID != nil) { err = errors.Join(err, configutils.ErrInvalid{Name: "SolKey", Value: e.JSON, Msg: "all fields must be nil or non-nil"}) } // require valid id @@ -271,8 +278,9 @@ func (a *AptosKeys) SetFrom(f *AptosKeys) error { if f == nil || len(f.Keys) == 0 { return nil } - a.Keys = make([]*AptosKey, len(f.Keys)) - copy(a.Keys, f.Keys) + // Replacing would discard the keys contributed by earlier -s files; + // validateMerge has already rejected any duplicated across them. + a.Keys = append(a.Keys, f.Keys...) return nil } @@ -280,9 +288,15 @@ func (a *AptosKeys) validateMerge(f *AptosKeys) (err error) { have := make(map[uint64]struct{}) if a != nil && f != nil { for _, aptosKey := range a.Keys { + if aptosKey.ID == nil { + continue + } have[*aptosKey.ID] = struct{}{} } for _, aptosKey := range f.Keys { + if aptosKey.ID == nil { + continue // a missing ID is a validation error, not a merge conflict + } if _, ok := have[*aptosKey.ID]; ok { err = errors.Join(err, configutils.ErrOverride{Name: fmt.Sprintf("AptosKeys: %d", *aptosKey.ID)}) } @@ -360,8 +374,9 @@ func (s *StellarKeys) SetFrom(f *StellarKeys) error { if f == nil || len(f.Keys) == 0 { return nil } - s.Keys = make([]*StellarKey, len(f.Keys)) - copy(s.Keys, f.Keys) + // Replacing would discard the keys contributed by earlier -s files; + // validateMerge has already rejected any duplicated across them. + s.Keys = append(s.Keys, f.Keys...) return nil } @@ -369,9 +384,15 @@ func (s *StellarKeys) validateMerge(f *StellarKeys) (err error) { have := make(map[string]struct{}) if s != nil && f != nil { for _, stellarKey := range s.Keys { + if stellarKey.ID == nil { + continue + } have[*stellarKey.ID] = struct{}{} } for _, stellarKey := range f.Keys { + if stellarKey.ID == nil { + continue // a missing ID is a validation error, not a merge conflict + } if _, ok := have[*stellarKey.ID]; ok { err = errors.Join(err, configutils.ErrOverride{Name: "StellarKeys: " + *stellarKey.ID}) } @@ -443,8 +464,9 @@ func (e *EthKeys) SetFrom(f *EthKeys) error { if f == nil || len(f.Keys) == 0 { return nil } - e.Keys = make([]*EthKey, len(f.Keys)) - copy(e.Keys, f.Keys) + // Replacing would discard the keys contributed by earlier -s files; + // validateMerge has already rejected any duplicated across them. + e.Keys = append(e.Keys, f.Keys...) return nil } @@ -452,9 +474,15 @@ func (e *EthKeys) validateMerge(f *EthKeys) (err error) { have := make(map[int]struct{}) if e != nil && f != nil { for _, ethKey := range e.Keys { + if ethKey.ID == nil { + continue + } have[*ethKey.ID] = struct{}{} } for _, ethKey := range f.Keys { + if ethKey.ID == nil { + continue // a missing ID is a validation error, not a merge conflict + } if _, ok := have[*ethKey.ID]; ok { err = errors.Join(err, configutils.ErrOverride{Name: fmt.Sprintf("EthKeys: %d", *ethKey.ID)}) } @@ -612,7 +640,7 @@ func (e *EthKey) validateMerge(f *EthKey) (err error) { } func (e *EthKey) ValidateConfig() (err error) { - if (e.JSON != nil) != (e.Password != nil) && (e.Password != nil) != (e.ID != nil) { + if (e.JSON != nil) != (e.Password != nil) || (e.Password != nil) != (e.ID != nil) { err = errors.Join(err, configutils.ErrInvalid{Name: "EthKey", Value: e.JSON, Msg: "all fields must be nil or non-nil"}) } // require valid id diff --git a/core/config/toml/types_test.go b/core/config/toml/types_test.go index 89c19b3f1d4..a5252a2552e 100644 --- a/core/config/toml/types_test.go +++ b/core/config/toml/types_test.go @@ -698,6 +698,165 @@ func TestEthKeys_SetFrom(t *testing.T) { assert.Equal(t, ethKeysWrapper2, *ethKeysWrapper1) } +func TestEthKeys_SetFrom_multipleSecretsFiles(t *testing.T) { + t.Parallel() + // Secrets files are applied in order and must union: a key from an earlier + // -s file has to survive a later file that only carries other chains' keys. + base := &EthKeys{Keys: []*EthKey{ + {JSON: new(models.Secret("key1")), Password: new(models.Secret("pass1")), ID: new(1)}, + }} + disjoint := &EthKeys{Keys: []*EthKey{ + {JSON: new(models.Secret("key56")), Password: new(models.Secret("pass56")), ID: new(56)}, + }} + + require.NoError(t, base.SetFrom(disjoint)) + + ids := make([]int, 0, len(base.Keys)) + for _, k := range base.Keys { + ids = append(ids, *k.ID) + } + assert.Equal(t, []int{1, 56}, ids, "keys from earlier secrets files must not be discarded") + + // Union must not weaken the no-overrides guarantee the -s flag documents. + dupe := &EthKeys{Keys: []*EthKey{ + {JSON: new(models.Secret("other")), Password: new(models.Secret("otherpass")), ID: new(1)}, + }} + require.Error(t, base.SetFrom(dupe)) + assert.Len(t, base.Keys, 2) +} + +func TestEthKeys_validateMerge_nilID(t *testing.T) { + t.Parallel() + // A secrets file may omit ID, and merging must survive it: a missing field + // is a validation error, not a crash. + base := &EthKeys{} + noID := &EthKeys{Keys: []*EthKey{ + {JSON: new(models.Secret("key1")), Password: new(models.Secret("pass1"))}, + }} + require.NotPanics(t, func() { + require.NoError(t, base.SetFrom(noID)) + }) +} + +func TestSolKeys_SetFrom_multipleSecretsFiles(t *testing.T) { + t.Parallel() + base := &SolKeys{Keys: []*SolKey{ + {JSON: new(models.Secret("key1")), Password: new(models.Secret("pass1")), ID: new("devnet")}, + }} + disjoint := &SolKeys{Keys: []*SolKey{ + {JSON: new(models.Secret("key2")), Password: new(models.Secret("pass2")), ID: new("mainnet")}, + }} + + require.NoError(t, base.SetFrom(disjoint)) + + ids := make([]string, 0, len(base.Keys)) + for _, k := range base.Keys { + ids = append(ids, *k.ID) + } + assert.Equal(t, []string{"devnet", "mainnet"}, ids, "keys from earlier secrets files must not be discarded") + + dupe := &SolKeys{Keys: []*SolKey{ + {JSON: new(models.Secret("other")), Password: new(models.Secret("otherpass")), ID: new("devnet")}, + }} + require.Error(t, base.SetFrom(dupe)) + assert.Len(t, base.Keys, 2) + + require.NotPanics(t, func() { + noID := &SolKeys{Keys: []*SolKey{{JSON: new(models.Secret("k"))}}} + require.NoError(t, (&SolKeys{}).SetFrom(noID)) + }) +} + +func TestAptosKeys_SetFrom_multipleSecretsFiles(t *testing.T) { + t.Parallel() + base := &AptosKeys{Keys: []*AptosKey{ + {JSON: new(models.Secret("key1")), Password: new(models.Secret("pass1")), ID: new(uint64(1))}, + }} + disjoint := &AptosKeys{Keys: []*AptosKey{ + {JSON: new(models.Secret("key2")), Password: new(models.Secret("pass2")), ID: new(uint64(2))}, + }} + + require.NoError(t, base.SetFrom(disjoint)) + + ids := make([]uint64, 0, len(base.Keys)) + for _, k := range base.Keys { + ids = append(ids, *k.ID) + } + assert.Equal(t, []uint64{1, 2}, ids, "keys from earlier secrets files must not be discarded") + + dupe := &AptosKeys{Keys: []*AptosKey{ + {JSON: new(models.Secret("other")), Password: new(models.Secret("otherpass")), ID: new(uint64(1))}, + }} + require.Error(t, base.SetFrom(dupe)) + assert.Len(t, base.Keys, 2) + + require.NotPanics(t, func() { + noID := &AptosKeys{Keys: []*AptosKey{{JSON: new(models.Secret("k"))}}} + require.NoError(t, (&AptosKeys{}).SetFrom(noID)) + }) +} + +func TestStellarKeys_SetFrom_multipleSecretsFiles(t *testing.T) { + t.Parallel() + base := &StellarKeys{Keys: []*StellarKey{ + {JSON: new(commonconfig.SecretString("key1")), Password: new(commonconfig.SecretString("pass1")), ID: new("testnet")}, + }} + disjoint := &StellarKeys{Keys: []*StellarKey{ + {JSON: new(commonconfig.SecretString("key2")), Password: new(commonconfig.SecretString("pass2")), ID: new("pubnet")}, + }} + + require.NoError(t, base.SetFrom(disjoint)) + + ids := make([]string, 0, len(base.Keys)) + for _, k := range base.Keys { + ids = append(ids, *k.ID) + } + assert.Equal(t, []string{"testnet", "pubnet"}, ids, "keys from earlier secrets files must not be discarded") + + dupe := &StellarKeys{Keys: []*StellarKey{ + {JSON: new(commonconfig.SecretString("other")), Password: new(commonconfig.SecretString("otherpass")), ID: new("testnet")}, + }} + require.Error(t, base.SetFrom(dupe)) + assert.Len(t, base.Keys, 2) + + require.NotPanics(t, func() { + noID := &StellarKeys{Keys: []*StellarKey{{JSON: new(commonconfig.SecretString("k"))}}} + require.NoError(t, (&StellarKeys{}).SetFrom(noID)) + }) +} + +// A key with only some of JSON/Password/ID set must be rejected, not silently +// accepted. All four key types share the "all fields must be nil or non-nil" +// rule. +func TestKeys_ValidateConfig_partialFields(t *testing.T) { + t.Parallel() + secret := new(models.Secret("s")) + stellarSecret := new(commonconfig.SecretString("s")) + + for _, tt := range []struct { + name string + cfg interface{ ValidateConfig() error } + }{ + {"EthKey missing ID", &EthKey{JSON: secret, Password: secret}}, + {"EthKey missing Password", &EthKey{JSON: secret, ID: new(1)}}, + {"SolKey missing ID", &SolKey{JSON: secret, Password: secret}}, + {"SolKey missing Password", &SolKey{JSON: secret, ID: new("devnet")}}, + {"AptosKey missing ID", &AptosKey{JSON: secret, Password: secret}}, + {"StellarKey missing ID", &StellarKey{JSON: stellarSecret, Password: stellarSecret}}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + require.Error(t, tt.cfg.ValidateConfig()) + }) + } + + // All-nil remains valid: an absent key section is not an error. + require.NoError(t, (&EthKey{}).ValidateConfig()) + require.NoError(t, (&SolKey{}).ValidateConfig()) + require.NoError(t, (&AptosKey{}).ValidateConfig()) + require.NoError(t, (&StellarKey{}).ValidateConfig()) +} + func TestBridgeStatusReporter_ValidateConfig(t *testing.T) { testCases := []struct { name string diff --git a/core/scripts/cre/environment/configs/workflow-sharded-capabilities-don.toml b/core/scripts/cre/environment/configs/workflow-sharded-capabilities-don.toml index db9c20447b7..5f7a340d742 100644 --- a/core/scripts/cre/environment/configs/workflow-sharded-capabilities-don.toml +++ b/core/scripts/cre/environment/configs/workflow-sharded-capabilities-don.toml @@ -1,9 +1,12 @@ # Sharded capability routing via DON families. # -# The same capability (evm-2337 write target) is hosted on multiple capability DON shards. -# Each workflow shard is routed to exactly one capability shard by sharing a per-shard DON -# family, while a common family ("zone-a") keeps every workflow shard connected to the -# gateway and to shared (non-sharded) capability DONs. +# The same capability (evm-1337) is hosted on multiple capability DON shards. Each workflow +# shard is routed to exactly one capability shard by sharing a per-shard DON family, while a +# common family ("zone-a") keeps every workflow shard connected to the gateway and to shared +# (non-sharded) capability DONs. +# +# The workflow shards deliberately host no EVM capability of their own: if they did, the +# launcher would serve evm-1337 locally and family routing would never be exercised. # # Family layout (see don_families): # workflow-shard-0 : don_families=["zone-a", "zone-a_shard-0"] @@ -14,7 +17,7 @@ # bootstrap-gateway: don_families=["zone-a"] # # The launcher's family-overlap filter therefore routes workflow-shard-0 -> cap-shard-0 and -# workflow-shard-1 -> cap-shard-1 for evm-2337, while both shards reach shared-cap via "zone-a". +# workflow-shard-1 -> cap-shard-1 for evm-1337, while both shards reach shared-cap via "zone-a". # Gateway pairing uses the primary (first) don_families entry ("zone-a") only. [chip_router] @@ -58,7 +61,7 @@ supported_evm_chains = [1337, 2337] env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' } - capabilities = ["cron", "http-action", "http-trigger", "consensus", "don-time", "evm-1337"] + capabilities = ["cron", "http-action", "http-trigger", "consensus", "don-time"] registry_based_launch_allowlist = ["cron-trigger@1.0.0", "dontime@1.0.0"] [nodesets.db] @@ -113,7 +116,7 @@ supported_evm_chains = [1337, 2337] env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' } - capabilities = ["cron", "http-action", "http-trigger", "consensus", "don-time", "evm-1337"] + capabilities = ["cron", "http-action", "http-trigger", "consensus", "don-time"] registry_based_launch_allowlist = ["cron-trigger@1.0.0", "dontime@1.0.0"] [nodesets.db] @@ -140,7 +143,7 @@ supported_evm_chains = [1337, 2337] env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' } - capabilities = ["evm-2337"] + capabilities = ["evm-1337"] [nodesets.db] image = "postgres:15.0" @@ -166,7 +169,7 @@ supported_evm_chains = [1337, 2337] env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' } - capabilities = ["evm-2337"] + capabilities = ["evm-1337"] [nodesets.db] image = "postgres:15.0" diff --git a/core/services/pipeline/bridgeconn/bridge_conn_manager.go b/core/services/pipeline/bridgeconn/bridge_conn_manager.go index e00680b379e..a2e649b556c 100644 --- a/core/services/pipeline/bridgeconn/bridge_conn_manager.go +++ b/core/services/pipeline/bridgeconn/bridge_conn_manager.go @@ -31,7 +31,7 @@ var ( // observationTTL bounds how long a cached observation may be served before it is // treated as stale. Hardcoded for now; may become configurable later. -const observationTTL = 60 * time.Second +const observationTTL = 90 * time.Second // cacheEntry pairs a cached observation with the time it was stored, so // GetObservation can reject entries older than observationTTL. diff --git a/core/services/pipeline/bridgeconn/eaconn.go b/core/services/pipeline/bridgeconn/eaconn.go index 18ba878e70b..eee7ab9c606 100644 --- a/core/services/pipeline/bridgeconn/eaconn.go +++ b/core/services/pipeline/bridgeconn/eaconn.go @@ -3,6 +3,7 @@ package bridgeconn import ( "context" "encoding/hex" + stdErrors "errors" "net/url" "sync" "time" @@ -182,6 +183,9 @@ func (c *eaConn) run(ctx context.Context) { for { stream, err := c.dial(ctx, c.target, c.useTLS) if err != nil { + if stdErrors.Is(err, errStreamDialingDisabledForTest) { + return + } c.lggr.Errorw("EAConn: dial failed", "target", c.target, "err", err) backoff = c.sleepBackoff(backoff) continue diff --git a/core/services/pipeline/task.bridge_test.go b/core/services/pipeline/task.bridge_test.go index 8b23ecd9b18..cabb3cc6dae 100644 --- a/core/services/pipeline/task.bridge_test.go +++ b/core/services/pipeline/task.bridge_test.go @@ -363,6 +363,7 @@ func TestBridgeTask_UsesBridgeConnManagerCacheFallback(t *testing.T) { manager := bridgeconn.NewBridgeConnManager(logger.TestLogger(t)) seedable, ok := manager.(interface { + SeedObservation(bridge bridges.BridgeType, requestData map[string]any, observation []byte) error DisableEAConnDialingForTest() }) require.True(t, ok) @@ -381,25 +382,47 @@ func TestBridgeTask_UsesBridgeConnManagerCacheFallback(t *testing.T) { task.HelperSetDependencies(cfg.JobPipeline(), cfg.WebServer(), orm, specID, uuid.UUID{}, c) task.HelperSetBridgeConnManager(manager) - telemCh := make(chan any, 1) + telemCh := make(chan any, 2) ctx := pipeline.WithTelemetryCh(t.Context(), telemCh) - // No observation in the connection manager, so the live lookup will fail. - // Seed the bridge cache directly so the task can fall back to it. - require.NoError(t, orm.UpsertBridgeResponse(ctx, task.DotID(), specID, []byte(`{"data":{"result":"9700"}}`))) + cachedResponse := []byte(`{"data":{"result":"9700"}}`) - result, runInfo := task.Run(ctx, logger.TestLogger(t), pipeline.NewVarsFrom(nil), nil) + // Prime the connection manager so the first live lookup succeeds and its + // response is persisted to the bridge cache by the task. + require.NoError(t, seedable.SeedObservation(*bridge, utils.MustUnmarshalToMap(btcUSDPairing), cachedResponse)) + result, runInfo := task.Run(ctx, logger.TestLogger(t), pipeline.NewVarsFrom(nil), nil) assert.False(t, runInfo.IsPending) assert.False(t, runInfo.IsRetryable) require.NoError(t, result.Error) - assert.JSONEq(t, `{"data":{"result":"9700"}}`, result.Value.(string)) + assert.JSONEq(t, string(cachedResponse), result.Value.(string)) assert.Equal(t, int32(0), httpCalls.Load()) telem := <-telemCh require.IsType(t, &pipeline.BridgeTelemetry{}, telem) btelem := telem.(*pipeline.BridgeTelemetry) - assert.True(t, btelem.LocalCacheHit) + assert.False(t, btelem.LocalCacheHit) + // The telemetry body carries the live observation. + assert.JSONEq(t, string(cachedResponse), string(btelem.ResponseData)) + + // Point the task at a different request payload. The connection manager only + // holds an observation for the original payload, so this second lookup misses + // and falls back to the bridge cache persisted by the first run. + task.RequestData = ethUSDPairing + + result2, runInfo2 := task.Run(ctx, logger.TestLogger(t), pipeline.NewVarsFrom(nil), nil) + assert.False(t, runInfo2.IsPending) + assert.False(t, runInfo2.IsRetryable) + require.NoError(t, result2.Error) + assert.JSONEq(t, string(cachedResponse), result2.Value.(string)) + assert.Equal(t, int32(0), httpCalls.Load()) + + telem2 := <-telemCh + require.IsType(t, &pipeline.BridgeTelemetry{}, telem2) + btelem2 := telem2.(*pipeline.BridgeTelemetry) + assert.True(t, btelem2.LocalCacheHit) + // The cache-fallback body is served from the bridge cache persisted above. + assert.JSONEq(t, string(cachedResponse), string(btelem2.ResponseData)) } func TestBridgeTask_HandlesIntermittentFailure(t *testing.T) { diff --git a/system-tests/tests/smoke/cre/capability_helpers.go b/system-tests/tests/smoke/cre/capability_helpers.go index 783acbde1d8..49ade6a2d6c 100644 --- a/system-tests/tests/smoke/cre/capability_helpers.go +++ b/system-tests/tests/smoke/cre/capability_helpers.go @@ -22,8 +22,12 @@ import ( evmreadcontracts "github.com/smartcontractkit/chainlink/system-tests/tests/smoke/cre/evm/evmread/contracts" ) -// chainlink-common/pkg/capabilities/base_trigger.go logs this when AckEvent is called. -var triggerEventACKLogPattern = regexp.MustCompile(`Event ACK`) +// triggerEventACKLogNeedle is what chainlink-common/pkg/capabilities/base_trigger.go logs when +// AckEvent is called. It is written by the node hosting the trigger capability, so its container +// identifies which DON served the trigger. +const triggerEventACKLogNeedle = "Event ACK" + +var triggerEventACKLogPattern = regexp.MustCompile(triggerEventACKLogNeedle) // verifyTriggerEventACKLogs starts parallel readers (one per container stream) that scan for BaseTrigger // Event ACK lines. Call the returned cleanup after the test step finishes to cancel scanners and close diff --git a/system-tests/tests/smoke/cre/cre_suite_test.go b/system-tests/tests/smoke/cre/cre_suite_test.go index 6f4c7b35847..c451d29c8e8 100644 --- a/system-tests/tests/smoke/cre/cre_suite_test.go +++ b/system-tests/tests/smoke/cre/cre_suite_test.go @@ -371,6 +371,15 @@ func Test_CRE_V2_ShardManualAssignment(t *testing.T) { ExecuteManualShardAssignmentTest(t, testEnv) } +//nolint:paralleltest // the test owns the sharded topology it runs on +func Test_CRE_V2_ShardedCapabilitiesManualEVMLogTrigger(t *testing.T) { + testEnv := t_helpers.SetupTestEnvironmentWithConfig( + t, + t_helpers.GetTestConfig(t, "/configs/workflow-sharded-capabilities-don.toml"), + ) + ExecuteManualShardAssignmentWithEVMLogTriggerTest(t, testEnv) +} + //nolint:paralleltest // subtests share the same sharding config func Test_CRE_V2_ShardRingOCROverrides(t *testing.T) { testEnv := t_helpers.SetupTestEnvironmentWithConfig( diff --git a/system-tests/tests/smoke/cre/shard_assignment_test.go b/system-tests/tests/smoke/cre/shard_assignment_test.go index 1feaca187c9..4834fcf9ec5 100644 --- a/system-tests/tests/smoke/cre/shard_assignment_test.go +++ b/system-tests/tests/smoke/cre/shard_assignment_test.go @@ -4,10 +4,12 @@ import ( "context" "fmt" "net/url" + "slices" "strings" "testing" "time" + "github.com/ethereum/go-ethereum/crypto" "github.com/rs/zerolog" "github.com/stretchr/testify/require" @@ -21,6 +23,8 @@ import ( "github.com/smartcontractkit/chainlink/deployment/cre/pkg/offchain" "github.com/smartcontractkit/chainlink/system-tests/lib/cre" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/jobs" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment/blockchains/evm" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/sharding" stvault "github.com/smartcontractkit/chainlink/system-tests/lib/cre/vault" t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers" @@ -105,6 +109,218 @@ hashed_default_assignment = false testLogger.Info().Int("executedCount", len(executedWorkflows)).Msg("All workflows executed on correct shards (manual-only mode)") } +// ExecuteManualShardAssignmentWithEVMLogTriggerTest exercises manual-only shard assignment on a +// topology where the same capability is hosted by several capability DON shards and routed by DON +// family (configs/workflow-sharded-capabilities-don.toml). +// +// It differs from ExecuteManualShardAssignmentTest in three ways: +// - the workflow is pinned with per_owner_assignment instead of per_org_assignment, so no linking +// service is involved and the highest-precedence branch of the manual resolver is covered; +// - the workflow is driven by an EVM log trigger instead of cron, and the workflow shards host no +// EVM capability of their own, so the trigger must be served by a remote capability DON; +// - it asserts which capability DON served that trigger: only the capability shard sharing a DON +// family with the workflow shard may ack the trigger event, the other one must stay idle. +// +// There is no Ring OCR here: SetupSharding is never called, so no ShardConfig contract, no Ring +// OCR3 contract and no Ring jobs exist. Nodes fall back to the default "manual-only" assignment +// mode (core/services/chainlink/config_sharding.go), which resolves ownership purely from the +// shard-assignment job spec. +func ExecuteManualShardAssignmentWithEVMLogTriggerTest(t *testing.T, testEnv *ttypes.TestEnvironment) { + testLogger := framework.L + + shardDONs := testEnv.Dons.DonsWithFlag(cre.ShardDON) + require.GreaterOrEqual(t, len(shardDONs), 2, "Expected at least 2 shard DONs for manual assignment log trigger test") + + shardLeaderDON := getShardZeroDon(t, testEnv) + shardLeaderDonID := uint32(shardLeaderDON.ID) //nolint:gosec // G115: overflow is unrealistic + + // Pin to a non-leader shard on purpose. The leader is where the static default below sends + // anything the per-owner entry does not match, so an assignment that silently did not take + // effect cannot look like a pass. + nonLeaderDONs := slices.DeleteFunc(slices.Clone(shardDONs), func(don *cre.Don) bool { + return don.ID == shardLeaderDON.ID + }) + require.NotEmpty(t, nonLeaderDONs, "Expected to find a non-leader shard DON") + targetDON := nonLeaderDONs[0] + targetDonID := uint32(targetDON.ID) //nolint:gosec // G115: overflow is unrealistic + + // Both capability shards host the same EVM chain and the workflow shards host none, so the + // only thing that can decide who serves the log trigger is the DON family the workflow shard + // shares with one of them. servingCapDON is that shard; idleCapDONs must never be involved. + targetNodeSet := nodeSetForDON(t, testEnv, targetDON) + servingCapDON, idleCapDONs := evmCapabilityDONsByFamily(t, testEnv, targetNodeSet) + logTriggerChain := evmChainEnabledOnNodeSet(t, testEnv, servingCapDON) + logTriggerChainID := logTriggerChain.CtfOutput().ChainID + + workflowOwner := testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient.MustGetRootPrivateKey() + workflowOwnerAddress := strings.ToLower(crypto.PubkeyToAddress(workflowOwner.PublicKey).Hex()) + + testLogger.Info(). + Str("shardLeaderDON", shardLeaderDON.Name). + Uint32("shardLeaderDonID", shardLeaderDonID). + Str("targetDON", targetDON.Name). + Uint32("targetDonID", targetDonID). + Str("logTriggerChainID", logTriggerChainID). + Str("workflowOwner", workflowOwnerAddress). + Str("servingCapDON", servingCapDON.Name). + Strs("idleCapDONs", nodeSetNames(idleCapDONs)). + Msg("Manual shard assignment with EVM log trigger") + + workflowConfig, msgEmitter := configureEVMLogTriggerWorkflow(t, testLogger, logTriggerChain) + expectedUserLog := "Data for manual shard assignment log trigger chain " + logTriggerChainID + + emitCtx, emitCancelFn := context.WithCancel(t.Context()) + defer emitCancelFn() + startEVMLogTriggerEventEmitter(emitCtx, t, testLogger, logTriggerChainID, logTriggerChain, msgEmitter, expectedUserLog) + + // per_owner_assignment takes precedence over both per_org_assignment and the static default, + // so the static default points at the leader to prove the per-owner entry is what routed. + shardAssignmentTOML := fmt.Sprintf(` +static_default_assignment = [%d] +hashed_default_assignment = false + +[per_owner_assignment] + %q = [%d] +`, shardLeaderDonID, workflowOwnerAddress, targetDonID) + + // Every shard resolves ownership from its own copy of the spec, so both the shard that must + // run the workflow and the shard that must not need it. + for _, don := range shardDONs { + proposeAndApproveShardAssignmentJob(t, testEnv, don, shardAssignmentTOML, testLogger) + } + + // One workflow is enough: manual resolution keys off the owner alone, ignoring the workflow ID + // (resolveManual in core/services/workflows/shardownership/resolver.go), so extra workflows + // would re-evaluate the same per_owner_assignment branch at the cost of another WASM compile. + workflowID := t_helpers.CompileAndDeployWorkflow(t, testEnv, testLogger, "manualshard-evmlogtrigger", &workflowConfig, "./evm/logtrigger/main.go") + workflowIDs := []string{workflowID} + workflowToShardIndex := map[string]uint32{workflowID: targetDonID} + testLogger.Info().Str("workflowID", workflowID).Msg("Deployed workflow for manual shard assignment log trigger test") + + nodeP2PIDToShardIndex := buildNodeP2PIDToShardIndex(t, testEnv) + + userLogsCh := make(chan *workflowevents.UserLogs, 1000) + baseMessageCh := make(chan *commonevents.BaseMessage, 1000) + server := t_helpers.StartChipTestSink(t, t_helpers.GetPublishFn(testLogger, userLogsCh, baseMessageCh)) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + t_helpers.ShutdownChipSinkWithDrain(ctx, server, userLogsCh, baseMessageCh) + }) + + execTimeout := 4 * time.Minute + timeoutCtx, cancelTimeout := context.WithTimeout(t.Context(), execTimeout) + defer cancelTimeout() + execCtx, cancelCause := context.WithCancelCause(timeoutCtx) + defer cancelCause(nil) + go t_helpers.FailOnBaseMessage(execCtx, cancelCause, t, testLogger, baseMessageCh, t_helpers.WorkflowEngineInitErrorLog) + + // waitForAllWorkflowsExecuted only counts a workflow once it is seen on its expected shard, + // so a log from the leader shard is reported as a mismatch rather than accepted. + executedWorkflows := waitForAllWorkflowsExecuted(execCtx, t, testLogger, userLogsCh, workflowIDs, workflowToShardIndex, nodeP2PIDToShardIndex, expectedUserLog, execTimeout) + require.Len(t, executedWorkflows, len(workflowIDs), "Workflow did not execute on the manually assigned shard") + testLogger.Info().Msg("EVM log trigger workflow executed on the manually assigned shard") + + // The ack is logged by the node that hosts the trigger capability, so the container it comes + // from names the capability DON that served the trigger. The ack can trail the user log, hence + // the retry on the positive side; by the time it lands, an ack on the out-of-family shard would + // already be in its logs. + t_helpers.RequireContainerLogsForNodesetEventually(t, testEnv, servingCapDON.Name, triggerEventACKLogNeedle, 2*time.Minute, 5*time.Second) + for _, idle := range idleCapDONs { + t_helpers.AssertContainerLogsAbsentForNodeset(t, testEnv, idle.Name, triggerEventACKLogNeedle) + } + testLogger.Info().Str("servingCapDON", servingCapDON.Name).Msg("Log trigger was served by the in-family capability DON only") +} + +// nodeSetForDON returns the node set that backs don. Node sets carry the DON family and +// chain-capability information the Don snapshot does not. +func nodeSetForDON(t *testing.T, testEnv *ttypes.TestEnvironment, don *cre.Don) *cre.NodeSet { + t.Helper() + + for _, ns := range testEnv.Config.NodeSets { + if ns.Name == don.Name { + return ns + } + } + + require.FailNowf(t, "node set not found", "failed to find the node set for DON %s", don.Name) + return nil +} + +// evmCapabilityDONsByFamily splits the capability DONs that host an EVM capability into the single +// one sharing a DON family with workflowNodeSet - the one the launcher's family-overlap filter must +// route to - and the rest, which must never serve that workflow's triggers. +func evmCapabilityDONsByFamily(t *testing.T, testEnv *ttypes.TestEnvironment, workflowNodeSet *cre.NodeSet) (inFamily *cre.NodeSet, outOfFamily []*cre.NodeSet) { + t.Helper() + + for _, ns := range testEnv.Config.NodeSets { + if !slices.Contains(ns.DONTypes, cre.CapabilitiesDON) { + continue + } + + enabledChainIDs, err := ns.GetEnabledChainIDsForCapability(cre.EVMCapability) + require.NoErrorf(t, err, "failed to get EVM-enabled chain IDs for DON %s", ns.Name) + if len(enabledChainIDs) == 0 { + continue + } + + if sharesDonFamily(workflowNodeSet, ns) { + require.Nilf(t, inFamily, "DONs %s and %s both host EVM in a family of %s, routing would be ambiguous", ns.Name, nodeSetName(inFamily), workflowNodeSet.Name) + inFamily = ns + continue + } + outOfFamily = append(outOfFamily, ns) + } + + require.NotNilf(t, inFamily, "no capability DON hosts an EVM capability in a DON family of %s", workflowNodeSet.Name) + require.NotEmptyf(t, outOfFamily, "every EVM capability DON shares a family with %s, so routing cannot be told apart from a broadcast", workflowNodeSet.Name) + return inFamily, outOfFamily +} + +func sharesDonFamily(a, b *cre.NodeSet) bool { + return slices.ContainsFunc(a.DonFamilies, func(family string) bool { + return slices.Contains(b.DonFamilies, family) + }) +} + +func nodeSetName(ns *cre.NodeSet) string { + if ns == nil { + return "" + } + return ns.Name +} + +func nodeSetNames(nodeSets []*cre.NodeSet) []string { + names := make([]string, 0, len(nodeSets)) + for _, ns := range nodeSets { + names = append(names, ns.Name) + } + return names +} + +// evmChainEnabledOnNodeSet returns a deployed blockchain whose EVM capability is enabled on the +// given node set. It reads the node set rather than Don.GetEnabledChainIDsForCapability, because +// the node set builds its chain-capability index on demand while the Don only carries a snapshot. +func evmChainEnabledOnNodeSet(t *testing.T, testEnv *ttypes.TestEnvironment, nodeSet *cre.NodeSet) blockchains.Blockchain { + t.Helper() + + enabledChainIDs, err := nodeSet.GetEnabledChainIDsForCapability(cre.EVMCapability) + require.NoErrorf(t, err, "failed to get EVM-enabled chain IDs for DON %s", nodeSet.Name) + require.NotEmptyf(t, enabledChainIDs, "DON %s has no EVM chain enabled, it cannot serve a log trigger", nodeSet.Name) + + for _, chainID := range enabledChainIDs { + for _, bcOutput := range testEnv.CreEnvironment.Blockchains { + if bcOutput.ChainID() == chainID { + return bcOutput + } + } + } + + require.FailNowf(t, "no deployed blockchain matches the EVM chains enabled on the DON", + "DON %s has EVM chains %v enabled, none of which is deployed in this environment", nodeSet.Name, enabledChainIDs) + return nil +} + func ExecuteRingOCROverridesTest(t *testing.T, testEnv *ttypes.TestEnvironment) { testLogger := framework.L diff --git a/system-tests/tests/test-helpers/container_logs.go b/system-tests/tests/test-helpers/container_logs.go index eae0d1c1947..5cb48b98568 100644 --- a/system-tests/tests/test-helpers/container_logs.go +++ b/system-tests/tests/test-helpers/container_logs.go @@ -8,6 +8,7 @@ import ( "slices" "strings" "testing" + "time" "github.com/moby/moby/client" "github.com/stretchr/testify/assert" @@ -62,8 +63,9 @@ func nodesetContainerNames(t *testing.T, testEnv *ttypes.TestEnvironment, nodese return names } -// assertContainerLogs scans stdout/stderr of containerNames and checks whether needle appears. -func assertContainerLogs(t *testing.T, containerNames []string, needle string, wantFound bool) { +// containerLogsContain reports whether needle appears in the logs of any of containerNames, +// as they stand at the moment of the call. +func containerLogsContain(t *testing.T, containerNames []string, needle string) bool { t.Helper() targetNames := make(map[string]struct{}, len(containerNames)) @@ -90,10 +92,17 @@ func assertContainerLogs(t *testing.T, containerNames []string, needle string, w } if strings.Contains(content, needle) { found = true - framework.L.Info().Str("container", containerName).Str("needle", needle).Bool("want_found", wantFound).Msg("container log match") + framework.L.Info().Str("container", containerName).Str("needle", needle).Msg("container log match") } } + return found +} +// assertContainerLogs scans stdout/stderr of containerNames and checks whether needle appears. +func assertContainerLogs(t *testing.T, containerNames []string, needle string, wantFound bool) { + t.Helper() + + found := containerLogsContain(t, containerNames, needle) if wantFound { assert.True(t, found, "expected at least one of %v to contain %q", containerNames, needle) return @@ -119,6 +128,18 @@ func AssertContainerLogsAbsentForNodeset(t *testing.T, testEnv *ttypes.TestEnvir assertContainerLogs(t, nodesetContainerNames(t, testEnv, nodesetName), needle, false) } +// RequireContainerLogsForNodesetEventually waits for needle to appear in at least one container +// log of nodesetName, rescanning until timeout. Use it instead of AssertContainerLogsForNodeset +// when the line is written asynchronously and may still be in flight when the test reaches it. +func RequireContainerLogsForNodesetEventually(t *testing.T, testEnv *ttypes.TestEnvironment, nodesetName, needle string, timeout, interval time.Duration) { + t.Helper() + + containerNames := nodesetContainerNames(t, testEnv, nodesetName) + require.Eventually(t, func() bool { + return containerLogsContain(t, containerNames, needle) + }, timeout, interval, "expected at least one of %v to contain %q within %s", containerNames, needle, timeout) +} + // readContainerLogs decodes a Docker multiplexed log stream into plain text. // framework.StreamContainerLogs returns this format; the framework decoder is not exported. func readContainerLogs(r io.ReadCloser) (string, error) {