From cfc7e3ffd99575403d85932ebf59e231a7dd533b Mon Sep 17 00:00:00 2001 From: Dmitriy Tymoshchenko Date: Wed, 18 Nov 2026 00:37:11 +0100 Subject: [PATCH] [CRE-7129] Make sharding test deterministic by picking workflow names upfront --- .changeset/ring-locate-shard.md | 5 ++ core/services/ring/utils.go | 4 ++ core/services/ring/utils_test.go | 21 ++++++ system-tests/lib/cre/workflow/workflow.go | 39 +++++++---- system-tests/tests/smoke/cre/sharding_test.go | 64 +++++++++++-------- system-tests/tests/test-helpers/t_helpers.go | 28 ++++++++ 6 files changed, 121 insertions(+), 40 deletions(-) create mode 100644 .changeset/ring-locate-shard.md diff --git a/.changeset/ring-locate-shard.md b/.changeset/ring-locate-shard.md new file mode 100644 index 00000000000..70bfecd581d --- /dev/null +++ b/.changeset/ring-locate-shard.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +#internal Export ring.LocateShard so the sharding system test can pick workflow names that land on both shards diff --git a/core/services/ring/utils.go b/core/services/ring/utils.go index 8cc944252f6..b058da6ce21 100644 --- a/core/services/ring/utils.go +++ b/core/services/ring/utils.go @@ -51,6 +51,10 @@ func newShardRing(healthyShards []uint32) *consistent.Consistent { return consistent.New(members, consistentHashConfig()) } +func LocateShard(healthyShards []uint32, workflowID string) (uint32, error) { + return locateShard(newShardRing(healthyShards), workflowID) +} + func locateShard(ring *consistent.Consistent, workflowID string) (uint32, error) { if ring == nil { return 0, ErrNoHealthyShards diff --git a/core/services/ring/utils_test.go b/core/services/ring/utils_test.go index 65f1c61a625..dd9f359b3f3 100644 --- a/core/services/ring/utils_test.go +++ b/core/services/ring/utils_test.go @@ -1,6 +1,7 @@ package ring import ( + "strconv" "testing" "github.com/stretchr/testify/require" @@ -10,3 +11,23 @@ func TestUniqueSorted(t *testing.T) { got := uniqueSorted([]string{"c", "a", "b", "a", "c"}) require.Equal(t, []string{"a", "b", "c"}, got) } + +func TestLocateShard(t *testing.T) { + t.Parallel() + + _, err := LocateShard(nil, "wf") + require.ErrorIs(t, err, ErrNoHealthyShards) + + counts := map[uint32]int{} + for i := range 1000 { + wfID := "workflow-" + strconv.Itoa(i) + shard, err := LocateShard([]uint32{1, 2}, wfID) + require.NoError(t, err) + reversed, err := LocateShard([]uint32{2, 1}, wfID) + require.NoError(t, err) + require.Equal(t, shard, reversed, "Shard order should not change the assignment") + counts[shard]++ + } + require.Greater(t, counts[1], 350, "Shard 1 should get a fair share of workflows") + require.Greater(t, counts[2], 350, "Shard 2 should get a fair share of workflows") +} diff --git a/system-tests/lib/cre/workflow/workflow.go b/system-tests/lib/cre/workflow/workflow.go index 9cf82d64285..921399e4eb6 100644 --- a/system-tests/lib/cre/workflow/workflow.go +++ b/system-tests/lib/cre/workflow/workflow.go @@ -88,23 +88,12 @@ func RegisterWithContract( attributes []byte, artifactsDirInContainer *string, ) (string, error) { - // Download and decode workflow binary - workflowData, err := libnet.DownloadAndDecodeBase64(ctx, binaryURL) - if err != nil { - return "", errors.Wrap(err, "failed to download and decode workflow binary") - } - // Construct binary URL for container if needed binaryURLToUse := constructArtifactURL(binaryURL, artifactsDirInContainer) // Handle config URL if provided - var configData []byte configURLToUse := "" if configURL != nil && *configURL != "" { - configData, err = libnet.Download(ctx, *configURL) - if err != nil { - return "", errors.Wrap(err, "failed to download workflow config") - } configURLToUse = constructArtifactURL(*configURL, artifactsDirInContainer) } @@ -115,9 +104,9 @@ func RegisterWithContract( } // Generate workflow ID - workflowID, err := generateWorkflowIDFromStrings(sc.MustGetRootKeyAddress().Hex(), workflowName, workflowData, configData, secretsURLToUse) + workflowID, err := ComputeWorkflowID(ctx, sc.MustGetRootKeyAddress().Hex(), workflowName, binaryURL, configURL, secretsURLToUse) if err != nil { - return "", errors.Wrap(err, "failed to generate workflow ID") + return "", err } if version == nil || version.Major() != 2 { @@ -131,6 +120,30 @@ func RegisterWithContract( return workflowID, nil } +// ComputeWorkflowID returns the ID a workflow gets when registered with the given artifacts. +func ComputeWorkflowID(ctx context.Context, owner, workflowName, binaryURL string, configURL *string, secretsURL string) (string, error) { + // Download and decode workflow binary + workflowData, err := libnet.DownloadAndDecodeBase64(ctx, binaryURL) + if err != nil { + return "", errors.Wrap(err, "failed to download and decode workflow binary") + } + + // Download config if provided + var configData []byte + if configURL != nil && *configURL != "" { + configData, err = libnet.Download(ctx, *configURL) + if err != nil { + return "", errors.Wrap(err, "failed to download workflow config") + } + } + + workflowID, err := generateWorkflowIDFromStrings(owner, workflowName, workflowData, configData, secretsURL) + if err != nil { + return "", errors.Wrap(err, "failed to generate workflow ID") + } + return workflowID, nil +} + // LinkOwner links the seth client's root key (key 0) as a workflow owner. func LinkOwner(sc *seth.Client, workflowRegistryAddr common.Address, version *semver.Version) error { return LinkOwnerKey(sc, 0, workflowRegistryAddr, version) diff --git a/system-tests/tests/smoke/cre/sharding_test.go b/system-tests/tests/smoke/cre/sharding_test.go index 3765154e722..b1ea06d7037 100644 --- a/system-tests/tests/smoke/cre/sharding_test.go +++ b/system-tests/tests/smoke/cre/sharding_test.go @@ -35,6 +35,7 @@ import ( libcrypto "github.com/smartcontractkit/chainlink/system-tests/lib/crypto" t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers" ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration" + "github.com/smartcontractkit/chainlink/v2/core/services/ring" ) /* @@ -60,27 +61,8 @@ func ExecuteShardingTestWithCronTrigger(t *testing.T, testEnv *ttypes.TestEnviro workflowConfig := crontypes.WorkflowConfig{ Schedule: "*/30 * * * * *", } - // TODO: 16 workflows is a statistical workaround, not a fix. Step 8 requires the mapping - // to span both shards, but placement is consistent hashing over workflow IDs - // (core/services/ring/utils.go), and the IDs embed the compiled WASM, so they re-roll - // with the Go toolchain and checkout path (system-tests/lib/cre/workflow/compile.go - // builds without -trimpath). With 5 workflows all-on-one-shard happens ~1 in 15 runs; - // with 16 it is ~1 in 15,000-40,000. The flake is rare, not gone, and the count adds - // deploy/registry/cleanup runtime to every run. - // Real fixes, in order of preference: - // 1. Expose healthy shard/ring membership on the shard orchestrator API and assert - // shard 1 joined the ring after scale-up. That checks the actual regression - // ("second shard never receives auto-sharded work") with zero hash-ring luck, and - // would let step 8 drop the both-shards requirement entirely. - // 2. Add -trimpath -buildvcs=false to the workflow build so IDs stop depending on the - // checkout path (toolchain re-rolls would still happen). - // 3. Cover the distribution property in ring unit tests (deterministic, in-package) - // instead of sampling a hash ring through a Docker system test. - const numWorkflows = 16 - workflowNames := make([]string, numWorkflows) - for i := range numWorkflows { - workflowNames[i] = fmt.Sprintf("shardtest-cron-%d", i) - } + const numWorkflows = 5 + workflowNames := pickShardSpanningWorkflowNames(t, testEnv, workflowFileLocation, &workflowConfig, "shardtest-cron", numWorkflows) ExecuteShardingTemplate(t, testEnv, workflowFileLocation, &workflowConfig, workflowNames, "Amazing workflow user log") } @@ -111,10 +93,7 @@ func ExecuteShardingTestWithEVMLogTrigger(t *testing.T, testEnv *ttypes.TestEnvi workflowFileLocation := "./evm/logtrigger/main.go" const numWorkflows = 10 - workflowNames := make([]string, numWorkflows) - for i := range numWorkflows { - workflowNames[i] = fmt.Sprintf("shardtest-evmlogtrigger-%d", i) - } + workflowNames := pickShardSpanningWorkflowNames(t, testEnv, workflowFileLocation, &workflowConfig, "shardtest-evmlogtrigger", numWorkflows) ExecuteShardingTemplate(t, testEnv, workflowFileLocation, &workflowConfig, workflowNames, expectedMessage) requireTriggerEventACKLog(t, testLogger, singleAckFound) @@ -158,9 +137,8 @@ func ExecuteShardingTestWithHTTPTrigger(t *testing.T, testEnv *ttypes.TestEnviro workflowOwnerAddress := strings.ToLower(crypto.PubkeyToAddress(workflowOwner.PublicKey).Hex()) const numWorkflows = 5 - workflowNames := make([]string, numWorkflows) + workflowNames := pickShardSpanningWorkflowNames(t, testEnv, workflowFileLocation, &httpWorkflowConfig, "shard-httptrigger", numWorkflows) for i := range numWorkflows { - workflowNames[i] = fmt.Sprintf("shard-httptrigger-%d", i) go executeHTTPTriggerRequest(t, testEnv, gatewayURL, workflowNames[i], "", signingKey, workflowOwnerAddress) //nolint:testifylint // require.Eventually inside a goroutine is unsafe } @@ -169,6 +147,38 @@ func ExecuteShardingTestWithHTTPTrigger(t *testing.T, testEnv *ttypes.TestEnviro validateHTTPWorkflowRequest(t, testEnv) } +// pickShardSpanningWorkflowNames returns count workflow names split evenly between the first +// two shards. Shard placement hashes the workflow ID, which depends on the name, so fixed +// names occasionally put every workflow on one shard and fail the scale-up check. +func pickShardSpanningWorkflowNames[T t_helpers.WorkflowConfig](t *testing.T, testEnv *ttypes.TestEnvironment, workflowFileLocation string, workflowConfig *T, namePrefix string, count int) []string { + t.Helper() + testLogger := framework.L + + shardDONs := testEnv.Dons.DonsWithFlag(cre.ShardDON) + require.GreaterOrEqual(t, len(shardDONs), 2, "expected at least two shard DONs") + shardDONIDs := []uint32{uint32(shardDONs[0].ID), uint32(shardDONs[1].ID)} //nolint:gosec // G115: overflow is unrealistic + predictWorkflowID := t_helpers.WorkflowIDPredictor(t, testEnv, testLogger, workflowConfig, workflowFileLocation) + + const maxCandidates = 100 + perShard := map[uint32]int{} + names := make([]string, 0, count) + for i := 0; len(names) < count; i++ { + require.Less(t, i, maxCandidates, "could not pick %d workflow names spanning shards %v", count, shardDONIDs) + name := fmt.Sprintf("%s-%d", namePrefix, i) + shard, err := ring.LocateShard(shardDONIDs, predictWorkflowID(name)) + require.NoError(t, err) + if perShard[shard] < (count+1)/2 { + perShard[shard]++ + names = append(names, name) + } + } + testLogger.Info(). + Strs("workflowNames", names). + Interface("distribution", perShard). + Msg("Picked workflow names spanning both shards") + return names +} + func ExecuteShardingTemplate[T t_helpers.WorkflowConfig](t *testing.T, testEnv *ttypes.TestEnvironment, workflowFileLocation string, workflowConfig *T, workflowNames []string, expectedMessage string) { testLogger := framework.L diff --git a/system-tests/tests/test-helpers/t_helpers.go b/system-tests/tests/test-helpers/t_helpers.go index 18dbef522bc..5bb54bd7ef1 100644 --- a/system-tests/tests/test-helpers/t_helpers.go +++ b/system-tests/tests/test-helpers/t_helpers.go @@ -866,6 +866,34 @@ func CompileAndDeployWorkflow[T WorkflowConfig](t *testing.T, return workflowID } +// WorkflowIDPredictor compiles the workflow once and returns a function that computes the ID +// CompileAndDeployWorkflow would register it under for a given name. +func WorkflowIDPredictor[T WorkflowConfig](t *testing.T, + testEnv *ttypes.TestEnvironment, testLogger zerolog.Logger, + workflowConfig *T, workflowFileLocation string, +) func(workflowName string) string { + t.Helper() + + require.IsType(t, &evm.Blockchain{}, testEnv.CreEnvironment.Blockchains[0], "expected EVM blockchain type") + owner := testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient.MustGetRootKeyAddress().Hex() + + artifactDir := t.TempDir() + compressedWorkflowWasmPath, compileErr := creworkflow.CompileWorkflowToDir(t.Context(), workflowFileLocation, "workflow-id-predictor", artifactDir) + require.NoError(t, compileErr, "failed to compile workflow '%s'", workflowFileLocation) + binaryURL := "file://" + compressedWorkflowWasmPath + + return func(workflowName string) string { + var configURL *string + if workflowConfigPath := workflowConfigFactory(t, testLogger, workflowName, workflowConfig, artifactDir); workflowConfigPath != "" { + u := "file://" + workflowConfigPath + configURL = &u + } + workflowID, idErr := creworkflow.ComputeWorkflowID(t.Context(), owner, workflowName, binaryURL, configURL, "") + require.NoError(t, idErr, "failed to compute workflow ID for '%s'", workflowName) + return workflowID + } +} + func envVarOrDefault(envVar string, defaultValue int) int { v := strings.TrimSpace(os.Getenv(envVar)) if v == "" {