Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/ring-locate-shard.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"chainlink": patch
---

#internal Export ring.LocateShard so the sharding system test can pick workflow names that land on both shards
4 changes: 4 additions & 0 deletions core/services/ring/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ func newShardRing(healthyShards []uint32) *consistent.Consistent {
return consistent.New(members, consistentHashConfig())
}

func LocateShard(healthyShards []uint32, workflowID string) (uint32, error) {
return locateShard(newShardRing(healthyShards), workflowID)
}

func locateShard(ring *consistent.Consistent, workflowID string) (uint32, error) {
if ring == nil {
return 0, ErrNoHealthyShards
Expand Down
21 changes: 21 additions & 0 deletions core/services/ring/utils_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package ring

import (
"strconv"
"testing"

"github.com/stretchr/testify/require"
Expand All @@ -10,3 +11,23 @@ func TestUniqueSorted(t *testing.T) {
got := uniqueSorted([]string{"c", "a", "b", "a", "c"})
require.Equal(t, []string{"a", "b", "c"}, got)
}

func TestLocateShard(t *testing.T) {
t.Parallel()

_, err := LocateShard(nil, "wf")
require.ErrorIs(t, err, ErrNoHealthyShards)

counts := map[uint32]int{}
for i := range 1000 {
wfID := "workflow-" + strconv.Itoa(i)
shard, err := LocateShard([]uint32{1, 2}, wfID)
require.NoError(t, err)
reversed, err := LocateShard([]uint32{2, 1}, wfID)
require.NoError(t, err)
require.Equal(t, shard, reversed, "Shard order should not change the assignment")
counts[shard]++
}
require.Greater(t, counts[1], 350, "Shard 1 should get a fair share of workflows")
require.Greater(t, counts[2], 350, "Shard 2 should get a fair share of workflows")
}
39 changes: 26 additions & 13 deletions system-tests/lib/cre/workflow/workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,23 +88,12 @@ func RegisterWithContract(
attributes []byte,
artifactsDirInContainer *string,
) (string, error) {
// Download and decode workflow binary
workflowData, err := libnet.DownloadAndDecodeBase64(ctx, binaryURL)
if err != nil {
return "", errors.Wrap(err, "failed to download and decode workflow binary")
}

// Construct binary URL for container if needed
binaryURLToUse := constructArtifactURL(binaryURL, artifactsDirInContainer)

// Handle config URL if provided
var configData []byte
configURLToUse := ""
if configURL != nil && *configURL != "" {
configData, err = libnet.Download(ctx, *configURL)
if err != nil {
return "", errors.Wrap(err, "failed to download workflow config")
}
configURLToUse = constructArtifactURL(*configURL, artifactsDirInContainer)
}

Expand All @@ -115,9 +104,9 @@ func RegisterWithContract(
}

// Generate workflow ID
workflowID, err := generateWorkflowIDFromStrings(sc.MustGetRootKeyAddress().Hex(), workflowName, workflowData, configData, secretsURLToUse)
workflowID, err := ComputeWorkflowID(ctx, sc.MustGetRootKeyAddress().Hex(), workflowName, binaryURL, configURL, secretsURLToUse)
if err != nil {
return "", errors.Wrap(err, "failed to generate workflow ID")
return "", err
}

if version == nil || version.Major() != 2 {
Expand All @@ -131,6 +120,30 @@ func RegisterWithContract(
return workflowID, nil
}

// ComputeWorkflowID returns the ID a workflow gets when registered with the given artifacts.
func ComputeWorkflowID(ctx context.Context, owner, workflowName, binaryURL string, configURL *string, secretsURL string) (string, error) {
// Download and decode workflow binary
workflowData, err := libnet.DownloadAndDecodeBase64(ctx, binaryURL)
if err != nil {
return "", errors.Wrap(err, "failed to download and decode workflow binary")
}

// Download config if provided
var configData []byte
if configURL != nil && *configURL != "" {
configData, err = libnet.Download(ctx, *configURL)
if err != nil {
return "", errors.Wrap(err, "failed to download workflow config")
}
}

workflowID, err := generateWorkflowIDFromStrings(owner, workflowName, workflowData, configData, secretsURL)
if err != nil {
return "", errors.Wrap(err, "failed to generate workflow ID")
}
return workflowID, nil
}

// LinkOwner links the seth client's root key (key 0) as a workflow owner.
func LinkOwner(sc *seth.Client, workflowRegistryAddr common.Address, version *semver.Version) error {
return LinkOwnerKey(sc, 0, workflowRegistryAddr, version)
Expand Down
64 changes: 37 additions & 27 deletions system-tests/tests/smoke/cre/sharding_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
libcrypto "github.com/smartcontractkit/chainlink/system-tests/lib/crypto"
t_helpers "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers"
ttypes "github.com/smartcontractkit/chainlink/system-tests/tests/test-helpers/configuration"
"github.com/smartcontractkit/chainlink/v2/core/services/ring"
)

/*
Expand All @@ -60,27 +61,8 @@ func ExecuteShardingTestWithCronTrigger(t *testing.T, testEnv *ttypes.TestEnviro
workflowConfig := crontypes.WorkflowConfig{
Schedule: "*/30 * * * * *",
}
// TODO: 16 workflows is a statistical workaround, not a fix. Step 8 requires the mapping
// to span both shards, but placement is consistent hashing over workflow IDs
// (core/services/ring/utils.go), and the IDs embed the compiled WASM, so they re-roll
// with the Go toolchain and checkout path (system-tests/lib/cre/workflow/compile.go
// builds without -trimpath). With 5 workflows all-on-one-shard happens ~1 in 15 runs;
// with 16 it is ~1 in 15,000-40,000. The flake is rare, not gone, and the count adds
// deploy/registry/cleanup runtime to every run.
// Real fixes, in order of preference:
// 1. Expose healthy shard/ring membership on the shard orchestrator API and assert
// shard 1 joined the ring after scale-up. That checks the actual regression
// ("second shard never receives auto-sharded work") with zero hash-ring luck, and
// would let step 8 drop the both-shards requirement entirely.
// 2. Add -trimpath -buildvcs=false to the workflow build so IDs stop depending on the
// checkout path (toolchain re-rolls would still happen).
// 3. Cover the distribution property in ring unit tests (deterministic, in-package)
// instead of sampling a hash ring through a Docker system test.
const numWorkflows = 16
workflowNames := make([]string, numWorkflows)
for i := range numWorkflows {
workflowNames[i] = fmt.Sprintf("shardtest-cron-%d", i)
}
const numWorkflows = 5
workflowNames := pickShardSpanningWorkflowNames(t, testEnv, workflowFileLocation, &workflowConfig, "shardtest-cron", numWorkflows)
ExecuteShardingTemplate(t, testEnv, workflowFileLocation, &workflowConfig, workflowNames, "Amazing workflow user log")
}

Expand Down Expand Up @@ -111,10 +93,7 @@ func ExecuteShardingTestWithEVMLogTrigger(t *testing.T, testEnv *ttypes.TestEnvi
workflowFileLocation := "./evm/logtrigger/main.go"

const numWorkflows = 10
workflowNames := make([]string, numWorkflows)
for i := range numWorkflows {
workflowNames[i] = fmt.Sprintf("shardtest-evmlogtrigger-%d", i)
}
workflowNames := pickShardSpanningWorkflowNames(t, testEnv, workflowFileLocation, &workflowConfig, "shardtest-evmlogtrigger", numWorkflows)
ExecuteShardingTemplate(t, testEnv, workflowFileLocation, &workflowConfig, workflowNames, expectedMessage)

requireTriggerEventACKLog(t, testLogger, singleAckFound)
Expand Down Expand Up @@ -158,9 +137,8 @@ func ExecuteShardingTestWithHTTPTrigger(t *testing.T, testEnv *ttypes.TestEnviro
workflowOwnerAddress := strings.ToLower(crypto.PubkeyToAddress(workflowOwner.PublicKey).Hex())

const numWorkflows = 5
workflowNames := make([]string, numWorkflows)
workflowNames := pickShardSpanningWorkflowNames(t, testEnv, workflowFileLocation, &httpWorkflowConfig, "shard-httptrigger", numWorkflows)
for i := range numWorkflows {
workflowNames[i] = fmt.Sprintf("shard-httptrigger-%d", i)
go executeHTTPTriggerRequest(t, testEnv, gatewayURL, workflowNames[i], "", signingKey, workflowOwnerAddress) //nolint:testifylint // require.Eventually inside a goroutine is unsafe
}

Expand All @@ -169,6 +147,38 @@ func ExecuteShardingTestWithHTTPTrigger(t *testing.T, testEnv *ttypes.TestEnviro
validateHTTPWorkflowRequest(t, testEnv)
}

// pickShardSpanningWorkflowNames returns count workflow names split evenly between the first
// two shards. Shard placement hashes the workflow ID, which depends on the name, so fixed
// names occasionally put every workflow on one shard and fail the scale-up check.
func pickShardSpanningWorkflowNames[T t_helpers.WorkflowConfig](t *testing.T, testEnv *ttypes.TestEnvironment, workflowFileLocation string, workflowConfig *T, namePrefix string, count int) []string {
t.Helper()
testLogger := framework.L

shardDONs := testEnv.Dons.DonsWithFlag(cre.ShardDON)
require.GreaterOrEqual(t, len(shardDONs), 2, "expected at least two shard DONs")
shardDONIDs := []uint32{uint32(shardDONs[0].ID), uint32(shardDONs[1].ID)} //nolint:gosec // G115: overflow is unrealistic
predictWorkflowID := t_helpers.WorkflowIDPredictor(t, testEnv, testLogger, workflowConfig, workflowFileLocation)

const maxCandidates = 100
perShard := map[uint32]int{}
names := make([]string, 0, count)
for i := 0; len(names) < count; i++ {
require.Less(t, i, maxCandidates, "could not pick %d workflow names spanning shards %v", count, shardDONIDs)
name := fmt.Sprintf("%s-%d", namePrefix, i)
shard, err := ring.LocateShard(shardDONIDs, predictWorkflowID(name))
require.NoError(t, err)
if perShard[shard] < (count+1)/2 {
perShard[shard]++
names = append(names, name)
}
}
testLogger.Info().
Strs("workflowNames", names).
Interface("distribution", perShard).
Msg("Picked workflow names spanning both shards")
return names
}

func ExecuteShardingTemplate[T t_helpers.WorkflowConfig](t *testing.T, testEnv *ttypes.TestEnvironment, workflowFileLocation string, workflowConfig *T, workflowNames []string, expectedMessage string) {
testLogger := framework.L

Expand Down
28 changes: 28 additions & 0 deletions system-tests/tests/test-helpers/t_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -866,6 +866,34 @@ func CompileAndDeployWorkflow[T WorkflowConfig](t *testing.T,
return workflowID
}

// WorkflowIDPredictor compiles the workflow once and returns a function that computes the ID
// CompileAndDeployWorkflow would register it under for a given name.
func WorkflowIDPredictor[T WorkflowConfig](t *testing.T,
testEnv *ttypes.TestEnvironment, testLogger zerolog.Logger,
workflowConfig *T, workflowFileLocation string,
) func(workflowName string) string {
t.Helper()

require.IsType(t, &evm.Blockchain{}, testEnv.CreEnvironment.Blockchains[0], "expected EVM blockchain type")
owner := testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient.MustGetRootKeyAddress().Hex()

artifactDir := t.TempDir()
compressedWorkflowWasmPath, compileErr := creworkflow.CompileWorkflowToDir(t.Context(), workflowFileLocation, "workflow-id-predictor", artifactDir)
require.NoError(t, compileErr, "failed to compile workflow '%s'", workflowFileLocation)
binaryURL := "file://" + compressedWorkflowWasmPath

return func(workflowName string) string {
var configURL *string
if workflowConfigPath := workflowConfigFactory(t, testLogger, workflowName, workflowConfig, artifactDir); workflowConfigPath != "" {
u := "file://" + workflowConfigPath
configURL = &u
}
workflowID, idErr := creworkflow.ComputeWorkflowID(t.Context(), owner, workflowName, binaryURL, configURL, "")
require.NoError(t, idErr, "failed to compute workflow ID for '%s'", workflowName)
return workflowID
}
}

func envVarOrDefault(envVar string, defaultValue int) int {
v := strings.TrimSpace(os.Getenv(envVar))
if v == "" {
Expand Down
Loading